3 # Copyright (C) 2015, 2016 Red Hat <contact@redhat.com>
4 # Copyright (C) 2014 Inktank <info@inktank.com>
5 # Copyright (C) 2014 Cloudwatt <libre.licensing@cloudwatt.com>
6 # Copyright (C) 2014 Catalyst.net Ltd
8 # Author: Loic Dachary <loic@dachary.org>
10 # This program is free software; you can redistribute it and/or modify
11 # it under the terms of the GNU Library Public License as published by
12 # the Free Software Foundation; either version 2, or (at your option)
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU Library Public License for more details.
21 from __future__
import print_function
44 CEPH_OSD_ONDISK_MAGIC
= 'ceph osd volume v026'
45 CEPH_LOCKBOX_ONDISK_MAGIC
= 'ceph lockbox volume v001'
47 KEY_MANAGEMENT_MODE_V1
= 'ceph-mon v1'
52 # identical because creating a journal is atomic
53 'ready': '45b0969e-9b03-4f30-b4c6-b4b80ceff106',
54 'tobe': '45b0969e-9b03-4f30-b4c6-b4b80ceff106',
57 # identical because creating a block is atomic
58 'ready': 'cafecafe-9b03-4f30-b4c6-b4b80ceff106',
59 'tobe': 'cafecafe-9b03-4f30-b4c6-b4b80ceff106',
62 # identical because creating a block is atomic
63 'ready': '30cd0809-c2b2-499c-8879-2d6b78529876',
64 'tobe': '30cd0809-c2b2-499c-8879-2d6b785292be',
67 # identical because creating a block is atomic
68 'ready': '5ce17fce-4087-4169-b7ff-056cc58473f9',
69 'tobe': '5ce17fce-4087-4169-b7ff-056cc58472be',
72 'ready': '4fbd7e29-9d25-41b8-afd0-062c0ceff05d',
73 'tobe': '89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be',
76 'ready': 'fb3aabf9-d25f-47cc-bf5e-721d1816496b',
77 'tobe': 'fb3aabf9-d25f-47cc-bf5e-721d181642be',
82 'ready': '45b0969e-9b03-4f30-b4c6-35865ceff106',
83 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
86 'ready': 'cafecafe-9b03-4f30-b4c6-35865ceff106',
87 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
90 'ready': '166418da-c469-4022-adf4-b30afd37f176',
91 'tobe': '7521c784-4626-4260-bc8d-ba77a0f5f2be',
94 'ready': '86a32090-3647-40b9-bbbd-38d8c573aa86',
95 'tobe': '92dad30f-175b-4d40-a5b0-5c0a258b42be',
98 'ready': '4fbd7e29-9d25-41b8-afd0-35865ceff05d',
99 'tobe': '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be',
104 'ready': '45b0969e-9b03-4f30-b4c6-5ec00ceff106',
105 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
108 'ready': 'cafecafe-9b03-4f30-b4c6-5ec00ceff106',
109 'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
112 'ready': '93b0052d-02d9-4d8a-a43b-33a3ee4dfbc3',
113 'tobe': '69d17c68-3e58-4399-aff0-b68265f2e2be',
116 'ready': '306e8683-4fe2-4330-b7c0-00a917c16966',
117 'tobe': 'f2d89683-a621-4063-964a-eb1f7863a2be',
120 'ready': '4fbd7e29-9d25-41b8-afd0-5ec00ceff05d',
121 'tobe': '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be',
126 'ready': '45b0969e-8ae0-4982-bf9d-5a8d867af560',
127 'tobe': '45b0969e-8ae0-4982-bf9d-5a8d867af560',
130 'ready': 'cafecafe-8ae0-4982-bf9d-5a8d867af560',
131 'tobe': 'cafecafe-8ae0-4982-bf9d-5a8d867af560',
134 'ready': 'ec6d6385-e346-45dc-be91-da2a7c8b3261',
135 'tobe': 'ec6d6385-e346-45dc-be91-da2a7c8b32be',
138 'ready': '01b41e1b-002a-453c-9f17-88793989ff8f',
139 'tobe': '01b41e1b-002a-453c-9f17-88793989f2be',
142 'ready': '4fbd7e29-8ae0-4982-bf9d-5a8d867af560',
143 'tobe': '89c57f98-8ae0-4982-bf9d-5a8d867af560',
146 'ready': '7f4a666a-16f3-47a2-8445-152ef4d03f6c',
147 'tobe': '7f4a666a-16f3-47a2-8445-152ef4d032be',
156 def get_ready_by_type(what
):
157 return [x
['ready'] for x
in PTYPE
[what
].values()]
160 def get_ready_by_name(name
):
161 return [x
[name
]['ready'] for x
in PTYPE
.values() if name
in x
]
164 def is_regular_space(ptype
):
165 return Ptype
.is_what_space('regular', ptype
)
168 def is_mpath_space(ptype
):
169 return Ptype
.is_what_space('mpath', ptype
)
172 def is_plain_space(ptype
):
173 return Ptype
.is_what_space('plain', ptype
)
176 def is_luks_space(ptype
):
177 return Ptype
.is_what_space('luks', ptype
)
180 def is_what_space(what
, ptype
):
181 for name
in Space
.NAMES
:
182 if ptype
== PTYPE
[what
][name
]['ready']:
187 def space_ptype_to_name(ptype
):
188 for what
in PTYPE
.values():
189 for name
in Space
.NAMES
:
190 if ptype
== what
[name
]['ready']:
192 raise ValueError('ptype ' + ptype
+ ' not found')
195 def is_dmcrypt_space(ptype
):
196 for name
in Space
.NAMES
:
197 if Ptype
.is_dmcrypt(ptype
, name
):
202 def is_dmcrypt(ptype
, name
):
203 for what
in ('plain', 'luks'):
204 if ptype
== PTYPE
[what
][name
]['ready']:
211 if platform
.system() == 'FreeBSD':
213 DEFAULT_FS_TYPE
= 'zfs'
214 PROCDIR
= '/compat/linux/proc'
215 # FreeBSD does not have blockdevices any more
219 DEFAULT_FS_TYPE
= 'xfs'
221 BLOCKDIR
= '/sys/block'
224 OSD STATUS Definition
226 OSD_STATUS_OUT_DOWN
= 0
227 OSD_STATUS_OUT_UP
= 1
228 OSD_STATUS_IN_DOWN
= 2
231 MOUNT_OPTIONS
= dict(
232 btrfs
='noatime,user_subvol_rm_allowed',
233 # user_xattr is default ever since linux 2.6.39 / 3.0, but we'll
234 # delay a moment before removing it fully because we did have some
235 # issues with ext4 before the xatts-in-leveldb work, and it seemed
236 # that user_xattr helped
237 ext4
='noatime,user_xattr',
238 xfs
='noatime,inode64',
243 # btrfs requires -f, for the same reason as xfs (see comment below)
250 # xfs insists on not overwriting previous fs; even if we wipe
251 # partition table, we often recreate it exactly the same way,
252 # so we'll see ghosts of filesystems past
271 STATEDIR
= '/var/lib/ceph'
273 SYSCONFDIR
= '/etc/ceph'
277 SUPPRESS_PREFIX
= None
279 # only warn once about some things
282 # Nuke the TERM variable to avoid confusing any subprocesses we call.
283 # For example, libreadline will print weird control sequences for some
285 if 'TERM' in os
.environ
:
286 del os
.environ
['TERM']
289 if LOG_NAME
== '__main__':
290 LOG_NAME
= os
.path
.basename(sys
.argv
[0])
291 LOG
= logging
.getLogger(LOG_NAME
)
293 # Allow user-preferred values for subprocess user and group
294 CEPH_PREF_USER
= None
295 CEPH_PREF_GROUP
= None
298 class FileLock(object):
299 def __init__(self
, fn
):
305 self
.fd
= os
.open(self
.fn
, os
.O_WRONLY | os
.O_CREAT
)
306 fcntl
.lockf(self
.fd
, fcntl
.LOCK_EX
)
308 def __exit__(self
, exc_type
, exc_val
, exc_tb
):
310 fcntl
.lockf(self
.fd
, fcntl
.LOCK_UN
)
315 class Error(Exception):
321 doc
= _bytes2str(self
.__doc
__.strip())
323 str_type
= basestring
326 args
= [a
if isinstance(a
, str_type
) else str(a
) for a
in self
.args
]
327 return ': '.join([doc
] + [_bytes2str(a
) for a
in args
])
330 class MountError(Error
):
332 Mounting filesystem failed
336 class UnmountError(Error
):
338 Unmounting filesystem failed
342 class BadMagicError(Error
):
344 Does not look like a Ceph OSD, or incompatible version
348 class TruncatedLineError(Error
):
354 class TooManyLinesError(Error
):
360 class FilesystemTypeError(Error
):
362 Cannot discover filesystem type
366 class CephDiskException(Exception):
368 A base exception for ceph-disk to provide custom (ad-hoc) messages that
369 will be caught and dealt with when main() is executed
374 class ExecutableNotFound(CephDiskException
):
376 Exception to report on executables not available in PATH
383 Detect whether systemd is running
385 with
open(PROCDIR
+ '/1/comm', 'r') as f
:
386 return 'systemd' in f
.read()
391 Detect whether upstart is running
393 (out
, err
, _
) = command(['init', '--version'])
394 return 'upstart' in out
397 def maybe_mkdir(*a
, **kw
):
399 Creates a new directory if it doesn't exist, removes
400 existing symlink before creating the directory.
402 # remove any symlink, if it is there..
403 if os
.path
.exists(*a
) and stat
.S_ISLNK(os
.lstat(*a
).st_mode
):
404 LOG
.debug('Removing old symlink at %s', *a
)
409 if e
.errno
== errno
.EEXIST
:
415 def which(executable
):
416 """find the location of an executable"""
417 envpath
= os
.environ
.get('PATH') or os
.defpath
418 PATH
= envpath
.split(os
.pathsep
)
429 for location
in locations
:
430 executable_path
= os
.path
.join(location
, executable
)
431 if (os
.path
.isfile(executable_path
) and
432 os
.access(executable_path
, os
.X_OK
)):
433 return executable_path
436 def _get_command_executable(arguments
):
438 Return the full path for an executable, raise if the executable is not
439 found. If the executable has already a full path do not perform any checks.
441 if os
.path
.isabs(arguments
[0]): # an absolute path
443 executable
= which(arguments
[0])
445 command_msg
= 'Could not run command: %s' % ' '.join(arguments
)
446 executable_msg
= '%s not in path.' % arguments
[0]
447 raise ExecutableNotFound('%s %s' % (executable_msg
, command_msg
))
449 # swap the old executable for the new one
450 arguments
[0] = executable
454 def command(arguments
, **kwargs
):
456 Safely execute a ``subprocess.Popen`` call making sure that the
457 executable exists and raising a helpful error message
460 .. note:: This should be the preferred way of calling ``subprocess.Popen``
461 since it provides the caller with the safety net of making sure that
462 executables *will* be found and will error nicely otherwise.
464 This returns the output of the command and the return code of the
465 process in a tuple: (stdout, stderr, returncode).
468 arguments
= list(map(_bytes2str
, _get_command_executable(arguments
)))
470 LOG
.info('Running command: %s' % ' '.join(arguments
))
471 process
= subprocess
.Popen(
473 stdout
=subprocess
.PIPE
,
474 stderr
=subprocess
.PIPE
,
476 out
, err
= process
.communicate()
478 return _bytes2str(out
), _bytes2str(err
), process
.returncode
481 def _bytes2str(string
):
482 return string
.decode('utf-8') if isinstance(string
, bytes
) else string
485 def command_init(arguments
, **kwargs
):
487 Safely execute a non-blocking ``subprocess.Popen`` call
488 making sure that the executable exists and raising a helpful
489 error message if it does not.
491 .. note:: This should be the preferred way of calling ``subprocess.Popen``
492 since it provides the caller with the safety net of making sure that
493 executables *will* be found and will error nicely otherwise.
495 This returns the process.
498 arguments
= list(map(_bytes2str
, _get_command_executable(arguments
)))
500 LOG
.info('Running command: %s' % ' '.join(arguments
))
501 process
= subprocess
.Popen(
503 stdout
=subprocess
.PIPE
,
504 stderr
=subprocess
.PIPE
,
509 def command_wait(process
):
511 Wait for the process finish and parse its output.
514 out
, err
= process
.communicate()
516 return _bytes2str(out
), _bytes2str(err
), process
.returncode
519 def command_check_call(arguments
, exit
=False):
521 Safely execute a ``subprocess.check_call`` call making sure that the
522 executable exists and raising a helpful error message if it does not.
524 When ``exit`` is set to ``True`` this helper will do a clean (sans
525 traceback) system exit.
526 .. note:: This should be the preferred way of calling
527 ``subprocess.check_call`` since it provides the caller with the safety net
528 of making sure that executables *will* be found and will error nicely
531 arguments
= _get_command_executable(arguments
)
532 command
= ' '.join(arguments
)
533 LOG
.info('Running command: %s', command
)
535 return subprocess
.check_call(arguments
)
536 except subprocess
.CalledProcessError
as error
:
539 LOG
.error(error
.output
)
541 "'{cmd}' failed with status code {returncode}".format(
543 returncode
=error
.returncode
,
550 # An alternative block_path implementation would be
552 # name = basename(dev)
553 # return /sys/devices/virtual/block/$name
555 # It is however more fragile because it relies on the fact
556 # that the basename of the device the user will use always
557 # matches the one the driver will use. On Ubuntu 14.04, for
558 # instance, when multipath creates a partition table on
560 # /dev/mapper/353333330000007d0 -> ../dm-0
562 # it will create partition devices named
564 # /dev/mapper/353333330000007d0-part1
566 # which is the same device as /dev/dm-1 but not a symbolic
569 # ubuntu@other:~$ ls -l /dev/mapper /dev/dm-1
570 # brw-rw---- 1 root disk 252, 1 Aug 15 17:52 /dev/dm-1
571 # lrwxrwxrwx 1 root root 7 Aug 15 17:52 353333330000007d0 -> ../dm-0
572 # brw-rw---- 1 root disk 252, 1 Aug 15 17:52 353333330000007d0-part1
574 # Using the basename in this case fails.
581 path
= os
.path
.realpath(dev
)
582 rdev
= os
.stat(path
).st_rdev
583 (M
, m
) = (os
.major(rdev
), os
.minor(rdev
))
584 return "{sysfs}/dev/block/{M}:{m}".format(sysfs
=SYSFS
, M
=M
, m
=m
)
587 def get_dm_uuid(dev
):
588 uuid_path
= os
.path
.join(block_path(dev
), 'dm', 'uuid')
589 LOG
.debug("get_dm_uuid " + dev
+ " uuid path is " + uuid_path
)
590 if not os
.path
.exists(uuid_path
):
592 uuid
= open(uuid_path
, 'r').read()
593 LOG
.debug("get_dm_uuid " + dev
+ " uuid is " + uuid
)
599 True if the path is managed by multipath
603 uuid
= get_dm_uuid(dev
)
605 (re
.match('part\d+-mpath-', uuid
) or
606 re
.match('mpath-', uuid
)))
609 def get_dev_name(path
):
611 get device name from path. e.g.::
613 /dev/sda -> sda, /dev/cciss/c0d1 -> cciss!c0d1
615 a device "name" is something like::
621 assert path
.startswith('/dev/')
623 return base
.replace('/', '!')
626 def get_dev_path(name
):
628 get a path (/dev/...) from a name (cciss!c0d1)
629 a device "path" is something like::
635 return '/dev/' + name
.replace('!', '/')
638 def get_dev_relpath(name
):
640 get a relative path to /dev from a name (cciss!c0d1)
642 return name
.replace('!', '/')
645 def get_dev_size(dev
, size
='megabytes'):
647 Attempt to get the size of a device so that we can prevent errors
648 from actions to devices that are smaller, and improve error reporting.
650 Because we want to avoid breakage in case this approach is not robust, we
651 will issue a warning if we failed to get the size.
653 :param size: bytes or megabytes
654 :param dev: the device to calculate the size
656 fd
= os
.open(dev
, os
.O_RDONLY
)
657 dividers
= {'bytes': 1, 'megabytes': 1024 * 1024}
659 device_size
= os
.lseek(fd
, 0, os
.SEEK_END
)
660 divider
= dividers
.get(size
, 1024 * 1024) # default to megabytes
661 return device_size
// divider
662 except Exception as error
:
663 LOG
.warning('failed to get size of %s: %s' % (dev
, str(error
)))
668 def get_partition_mpath(dev
, pnum
):
669 part_re
= "part{pnum}-mpath-".format(pnum
=pnum
)
670 partitions
= list_partitions_mpath(dev
, part_re
)
677 def get_partition_dev(dev
, pnum
):
679 get the device name for a partition
681 assume that partitions are named like the base dev,
682 with a number, and optionally
683 some intervening characters (like 'p'). e.g.,
686 cciss/c0d1 1 -> cciss!c0d1p1
689 for retry
in range(0, max_retry
+ 1):
693 partname
= get_partition_mpath(dev
, pnum
)
695 name
= get_dev_name(os
.path
.realpath(dev
))
696 sys_entry
= os
.path
.join(BLOCKDIR
, name
)
697 error_msg
= " in %s" % sys_entry
698 for f
in os
.listdir(sys_entry
):
699 if f
.startswith(name
) and f
.endswith(str(pnum
)):
700 # we want the shortest name that starts with the base name
701 # and ends with the partition number
702 if not partname
or len(f
) < len(partname
):
706 LOG
.info('Found partition %d for %s after %d tries' %
708 return get_dev_path(partname
)
710 if retry
< max_retry
:
711 LOG
.info('Try %d/%d : partition %d for %s does not exist%s' %
712 (retry
+ 1, max_retry
, pnum
, dev
, error_msg
))
716 raise Error('partition %d for %s does not appear to exist%s' %
717 (pnum
, dev
, error_msg
))
720 def list_all_partitions():
722 Return a list of devices and partitions
725 names
= os
.listdir(BLOCKDIR
)
728 # /dev/fd0 may hang http://tracker.ceph.com/issues/6827
729 if re
.match(r
'^fd\d$', name
):
731 dev_part_list
[name
] = list_partitions(get_dev_path(name
))
733 with
open(os
.path
.join(PROCDIR
, "partitions")) as partitions
:
734 for line
in partitions
:
735 columns
= line
.split()
736 if len(columns
) >= 4:
738 dev_part_list
[name
] = list_partitions(get_dev_path(name
))
742 def list_partitions(dev
):
743 dev
= os
.path
.realpath(dev
)
745 return list_partitions_mpath(dev
)
747 return list_partitions_device(dev
)
750 def list_partitions_mpath(dev
, part_re
="part\d+-mpath-"):
753 holders
= os
.path
.join(p
, 'holders')
754 for holder
in os
.listdir(holders
):
755 uuid_path
= os
.path
.join(holders
, holder
, 'dm', 'uuid')
756 uuid
= open(uuid_path
, 'r').read()
757 LOG
.debug("list_partitions_mpath: " + uuid_path
+ " uuid = " + uuid
)
758 if re
.match(part_re
, uuid
):
759 partitions
.append(holder
)
763 def list_partitions_device(dev
):
765 Return a list of partitions on the given device name
768 basename
= get_dev_name(dev
)
769 for name
in os
.listdir(block_path(dev
)):
770 if name
.startswith(basename
):
771 partitions
.append(name
)
775 def get_partition_base(dev
):
777 Get the base device for a partition
779 dev
= os
.path
.realpath(dev
)
780 if not stat
.S_ISBLK(os
.lstat(dev
).st_mode
):
781 raise Error('not a block device', dev
)
783 name
= get_dev_name(dev
)
784 if os
.path
.exists(os
.path
.join('/sys/block', name
)):
785 raise Error('not a partition', dev
)
788 for basename
in os
.listdir('/sys/block'):
789 if os
.path
.exists(os
.path
.join('/sys/block', basename
, name
)):
790 return get_dev_path(basename
)
791 raise Error('no parent device for partition', dev
)
794 def is_partition_mpath(dev
):
795 uuid
= get_dm_uuid(dev
)
796 return bool(re
.match('part\d+-mpath-', uuid
))
799 def partnum_mpath(dev
):
800 uuid
= get_dm_uuid(dev
)
801 return re
.findall('part(\d+)-mpath-', uuid
)[0]
804 def get_partition_base_mpath(dev
):
805 slave_path
= os
.path
.join(block_path(dev
), 'slaves')
806 slaves
= os
.listdir(slave_path
)
808 name_path
= os
.path
.join(slave_path
, slaves
[0], 'dm', 'name')
809 name
= open(name_path
, 'r').read().strip()
810 return os
.path
.join('/dev/mapper', name
)
813 def is_partition(dev
):
815 Check whether a given device path is a partition or a full disk.
818 return is_partition_mpath(dev
)
820 dev
= os
.path
.realpath(dev
)
822 if not stat
.S_ISBLK(st
.st_mode
):
823 raise Error('not a block device', dev
)
825 name
= get_dev_name(dev
)
826 if os
.path
.exists(os
.path
.join(BLOCKDIR
, name
)):
829 # make sure it is a partition of something else
830 major
= os
.major(st
.st_rdev
)
831 minor
= os
.minor(st
.st_rdev
)
832 if os
.path
.exists('/sys/dev/block/%d:%d/partition' % (major
, minor
)):
835 raise Error('not a disk or partition', dev
)
840 Check if the given device is mounted.
842 dev
= os
.path
.realpath(dev
)
843 with
open(PROCDIR
+ '/mounts', 'rb') as proc_mounts
:
844 for line
in proc_mounts
:
845 fields
= line
.split()
848 mounts_dev
= fields
[0]
850 if os
.path
.isabs(mounts_dev
) and os
.path
.exists(mounts_dev
):
851 mounts_dev
= os
.path
.realpath(mounts_dev
)
852 if mounts_dev
== dev
:
853 return _bytes2str(path
)
859 Check if a device is held by another device (e.g., a dm-crypt mapping)
861 assert os
.path
.exists(dev
)
865 dev
= os
.path
.realpath(dev
)
866 base
= get_dev_name(dev
)
869 directory
= '/sys/block/{base}/holders'.format(base
=base
)
870 if os
.path
.exists(directory
):
871 return os
.listdir(directory
)
876 directory
= '/sys/block/{base}/{part}/holders'.format(
877 part
=part
, base
=base
)
878 if os
.path
.exists(directory
):
879 return os
.listdir(directory
)
884 def verify_not_in_use(dev
, check_partitions
=False):
886 Verify if a given device (path) is in use (e.g. mounted or
887 in use by device-mapper).
889 :raises: Error if device is in use.
891 assert os
.path
.exists(dev
)
893 raise Error('Device is mounted', dev
)
894 holders
= is_held(dev
)
896 raise Error('Device %s is in use by a device-mapper '
897 'mapping (dm-crypt?)' % dev
, ','.join(holders
))
899 if check_partitions
and not is_partition(dev
):
900 for partname
in list_partitions(dev
):
901 partition
= get_dev_path(partname
)
902 if is_mounted(partition
):
903 raise Error('Device is mounted', partition
)
904 holders
= is_held(partition
)
906 raise Error('Device %s is in use by a device-mapper '
907 'mapping (dm-crypt?)'
908 % partition
, ','.join(holders
))
911 def must_be_one_line(line
):
913 Checks if given line is really one single line.
915 :raises: TruncatedLineError or TooManyLinesError
916 :return: Content of the line, or None if line isn't valid.
918 line
= _bytes2str(line
)
920 if line
[-1:] != '\n':
921 raise TruncatedLineError(line
)
924 raise TooManyLinesError(line
)
928 def read_one_line(parent
, name
):
930 Read a file whose sole contents are a single line.
934 :return: Contents of the line, or None if file did not exist.
936 path
= os
.path
.join(parent
, name
)
938 line
= open(path
, 'rb').read()
940 if e
.errno
== errno
.ENOENT
:
946 line
= must_be_one_line(line
)
947 except (TruncatedLineError
, TooManyLinesError
) as e
:
949 'File is corrupt: {path}: {msg}'.format(
957 def write_one_line(parent
, name
, text
):
959 Write a file whose sole contents are a single line.
963 path
= os
.path
.join(parent
, name
)
964 tmp
= '{path}.{pid}.tmp'.format(path
=path
, pid
=os
.getpid())
965 with
open(tmp
, 'wb') as tmp_file
:
966 tmp_file
.write(text
.encode('utf-8') + b
'\n')
967 os
.fsync(tmp_file
.fileno())
968 path_set_context(tmp
)
974 Get a init system using 'ceph-detect-init'
976 init
= _check_output(
979 '--default', 'sysvinit',
982 init
= must_be_one_line(init
)
986 def check_osd_magic(path
):
988 Check that this path has the Ceph OSD magic.
990 :raises: BadMagicError if this does not look like a Ceph OSD data
993 magic
= read_one_line(path
, 'magic')
995 # probably not mkfs'ed yet
996 raise BadMagicError(path
)
997 if magic
!= CEPH_OSD_ONDISK_MAGIC
:
998 raise BadMagicError(path
)
1001 def check_osd_id(osd_id
):
1003 Ensures osd id is numeric.
1005 if not re
.match(r
'^[0-9]+$', osd_id
):
1006 raise Error('osd id is not numeric', osd_id
)
1009 def allocate_osd_id(
1015 Accocates an OSD id on the given cluster.
1017 :raises: Error if the call to allocate the OSD id fails.
1018 :return: The allocated OSD id.
1021 LOG
.debug('Allocating OSD id...')
1023 osd_id
= _check_output(
1026 '--cluster', cluster
,
1027 '--name', 'client.bootstrap-osd',
1028 '--keyring', keyring
,
1029 'osd', 'create', '--concise',
1033 except subprocess
.CalledProcessError
as e
:
1034 raise Error('ceph osd create failed', e
, e
.output
)
1035 osd_id
= must_be_one_line(osd_id
)
1036 check_osd_id(osd_id
)
1040 def get_osd_id(path
):
1042 Gets the OSD id of the OSD at the given path.
1044 osd_id
= read_one_line(path
, 'whoami')
1045 if osd_id
is not None:
1046 check_osd_id(osd_id
)
1050 def get_ceph_user():
1051 global CEPH_PREF_USER
1053 if CEPH_PREF_USER
is not None:
1055 pwd
.getpwnam(CEPH_PREF_USER
)
1056 return CEPH_PREF_USER
1058 print("No such user:", CEPH_PREF_USER
)
1062 pwd
.getpwnam('ceph')
1068 def get_ceph_group():
1069 global CEPH_PREF_GROUP
1071 if CEPH_PREF_GROUP
is not None:
1073 grp
.getgrnam(CEPH_PREF_GROUP
)
1074 return CEPH_PREF_GROUP
1076 print("No such group:", CEPH_PREF_GROUP
)
1080 grp
.getgrnam('ceph')
1086 def path_set_context(path
):
1087 # restore selinux context to default policy values
1088 if which('restorecon'):
1089 command(['restorecon', '-R', path
])
1091 # if ceph user exists, set owner to ceph
1092 if get_ceph_user() == 'ceph':
1093 command(['chown', '-R', 'ceph:ceph', path
])
1096 def _check_output(args
=None, **kwargs
):
1097 out
, err
, ret
= command(args
, **kwargs
)
1100 error
= subprocess
.CalledProcessError(ret
, cmd
)
1101 error
.output
= out
+ err
1103 return _bytes2str(out
)
1106 def get_conf(cluster
, variable
):
1108 Get the value of the given configuration variable from the
1111 :raises: Error if call to ceph-conf fails.
1112 :return: The variable value or None.
1115 out
, err
, ret
= command(
1118 '--cluster={cluster}'.format(
1127 except OSError as e
:
1128 raise Error('error executing ceph-conf', e
, err
)
1130 # config entry not found
1133 raise Error('getting variable from configuration failed')
1134 value
= out
.split('\n', 1)[0]
1135 # don't differentiate between "var=" and no var set
1141 def get_conf_with_default(cluster
, variable
):
1143 Get a config value that is known to the C++ code.
1145 This will fail if called on variables that are not defined in
1146 common config options.
1149 out
= _check_output(
1152 '--cluster={cluster}'.format(
1155 '--show-config-value={variable}'.format(
1161 except subprocess
.CalledProcessError
as e
:
1163 'getting variable from configuration failed',
1167 value
= str(out
).split('\n', 1)[0]
1171 def get_fsid(cluster
):
1173 Get the fsid of the cluster.
1175 :return: The fsid or raises Error.
1177 fsid
= get_conf_with_default(cluster
=cluster
, variable
='fsid')
1179 raise Error('getting cluster uuid from configuration failed')
1183 def get_dmcrypt_key_path(
1189 Get path to dmcrypt key file.
1191 :return: Path to the dmcrypt key file, callers should check for existence.
1194 path
= os
.path
.join(key_dir
, _uuid
+ ".luks.key")
1196 path
= os
.path
.join(key_dir
, _uuid
)
1201 def get_dmcrypt_key(
1206 legacy_path
= get_dmcrypt_key_path(_uuid
, key_dir
, luks
)
1207 if os
.path
.exists(legacy_path
):
1208 return (legacy_path
,)
1209 path
= os
.path
.join(STATEDIR
, 'osd-lockbox', _uuid
)
1210 if os
.path
.exists(path
):
1211 mode
= get_oneliner(path
, 'key-management-mode')
1212 osd_uuid
= get_oneliner(path
, 'osd-uuid')
1213 ceph_fsid
= read_one_line(path
, 'ceph_fsid')
1214 if ceph_fsid
is None:
1215 raise Error('No cluster uuid assigned.')
1216 cluster
= find_cluster_by_uuid(ceph_fsid
)
1218 raise Error('No cluster conf found in ' + SYSCONFDIR
+
1219 ' with fsid %s' % ceph_fsid
)
1221 if mode
== KEY_MANAGEMENT_MODE_V1
:
1222 key
, stderr
, ret
= command(
1225 '--cluster', cluster
,
1227 'client.osd-lockbox.' + osd_uuid
,
1229 os
.path
.join(path
, 'keyring'),
1232 'dm-crypt/osd/' + osd_uuid
+ '/luks',
1235 LOG
.debug("stderr " + stderr
)
1237 return base64
.b64decode(key
)
1239 raise Error('unknown key-management-mode ' + str(mode
))
1240 raise Error('unable to read dm-crypt key', path
, legacy_path
)
1247 cryptsetup_parameters
,
1251 dev
= dmcrypt_is_mapped(_uuid
)
1255 if isinstance(key
, tuple):
1256 # legacy, before lockbox
1257 assert os
.path
.exists(key
[0])
1262 dev
= '/dev/mapper/' + _uuid
1270 ] + cryptsetup_parameters
1288 ] + cryptsetup_parameters
1290 def run(args
, stdin
):
1291 LOG
.info(" ".join(args
))
1292 process
= subprocess
.Popen(
1294 stdin
=subprocess
.PIPE
,
1295 stdout
=subprocess
.PIPE
,
1296 stderr
=subprocess
.PIPE
)
1297 out
, err
= process
.communicate(stdin
)
1300 assert process
.returncode
== 0
1305 run(luksFormat_args
, key
)
1306 run(luksOpen_args
, key
)
1308 # Plain mode has no format function, nor any validation
1309 # that the key is correct.
1310 run(create_args
, key
)
1311 # set proper ownership of mapped device
1312 command_check_call(['chown', 'ceph:ceph', dev
])
1315 except subprocess
.CalledProcessError
as e
:
1316 raise Error('unable to map device', rawdev
, e
)
1322 if not os
.path
.exists('/dev/mapper/' + _uuid
):
1327 command_check_call(['cryptsetup', 'remove', _uuid
])
1329 except subprocess
.CalledProcessError
as e
:
1331 raise Error('unable to unmap device', _uuid
, e
)
1333 time
.sleep(0.5 + retries
* 1.0)
1343 Mounts a device with given filessystem type and
1344 mount options to a tempfile path under /var/lib/ceph/tmp.
1346 # sanity check: none of the arguments are None
1348 raise ValueError('dev may not be None')
1350 raise ValueError('fstype may not be None')
1352 # pick best-of-breed mount options based on fs type
1354 options
= MOUNT_OPTIONS
.get(fstype
, '')
1356 myTemp
= STATEDIR
+ '/tmp'
1357 # mkdtemp expect 'dir' to be existing on the system
1358 # Let's be sure it's always the case
1359 if not os
.path
.exists(myTemp
):
1363 path
= tempfile
.mkdtemp(
1368 LOG
.debug('Mounting %s on %s with options %s', dev
, path
, options
)
1379 if which('restorecon'):
1386 except subprocess
.CalledProcessError
as e
:
1389 except (OSError, IOError):
1400 Unmount and removes the given mount point.
1405 LOG
.debug('Unmounting %s', path
)
1414 except subprocess
.CalledProcessError
as e
:
1415 # on failure, retry 3 times with incremental backoff
1417 raise UnmountError(e
)
1419 time
.sleep(0.5 + retries
* 1.0)
1425 ###########################################
1427 def extract_parted_partition_numbers(partitions
):
1428 numbers_as_strings
= re
.findall('^\d+', partitions
, re
.MULTILINE
)
1429 return map(int, numbers_as_strings
)
1432 def get_free_partition_index(dev
):
1434 Get the next free partition index on a given device.
1436 :return: Index number (> 1 if there is already a partition on the device)
1437 or 1 if there is no partition table.
1440 lines
= _check_output(
1449 except subprocess
.CalledProcessError
as e
:
1450 LOG
.info('cannot read partition index; assume it '
1451 'isn\'t present\n (Error: %s)' % e
)
1455 raise Error('parted failed to output anything')
1456 LOG
.debug('get_free_partition_index: analyzing ' + lines
)
1457 if ('CHS;' not in lines
and
1458 'CYL;' not in lines
and
1459 'BYT;' not in lines
):
1460 raise Error('parted output expected to contain one of ' +
1461 'CHH; CYL; or BYT; : ' + lines
)
1462 if os
.path
.realpath(dev
) not in lines
:
1463 raise Error('parted output expected to contain ' + dev
+ ': ' + lines
)
1464 _
, partitions
= lines
.split(os
.path
.realpath(dev
))
1465 partition_numbers
= extract_parted_partition_numbers(partitions
)
1466 if partition_numbers
:
1467 return max(partition_numbers
) + 1
1472 def check_journal_reqs(args
):
1473 _
, _
, allows_journal
= command([
1474 'ceph-osd', '--check-allows-journal',
1476 '--log-file', '$run_dir/$cluster-osd-check.log',
1477 '--cluster', args
.cluster
,
1478 '--setuser', get_ceph_user(),
1479 '--setgroup', get_ceph_group(),
1481 _
, _
, wants_journal
= command([
1482 'ceph-osd', '--check-wants-journal',
1484 '--log-file', '$run_dir/$cluster-osd-check.log',
1485 '--cluster', args
.cluster
,
1486 '--setuser', get_ceph_user(),
1487 '--setgroup', get_ceph_group(),
1489 _
, _
, needs_journal
= command([
1490 'ceph-osd', '--check-needs-journal',
1492 '--log-file', '$run_dir/$cluster-osd-check.log',
1493 '--cluster', args
.cluster
,
1494 '--setuser', get_ceph_user(),
1495 '--setgroup', get_ceph_group(),
1497 return (not allows_journal
, not wants_journal
, not needs_journal
)
1500 def update_partition(dev
, description
):
1502 Must be called after modifying a partition table so the kernel
1503 know about the change and fire udev events accordingly. A side
1504 effect of partprobe is to remove partitions and add them again.
1505 The first udevadm settle waits for ongoing udev events to
1506 complete, just in case one of them rely on an existing partition
1507 on dev. The second udevadm settle guarantees to the caller that
1508 all udev events related to the partition table change have been
1509 processed, i.e. the 95-ceph-osd.rules actions and mode changes,
1510 group changes etc. are complete.
1512 LOG
.debug('Calling partprobe on %s device %s', description
, dev
)
1513 partprobe_ok
= False
1514 error
= 'unknown error'
1515 partprobe
= _get_command_executable(['partprobe'])[0]
1517 command_check_call(['udevadm', 'settle', '--timeout=600'])
1519 _check_output(['flock', '-s', dev
, partprobe
, dev
])
1522 except subprocess
.CalledProcessError
as e
:
1524 if ('unable to inform the kernel' not in error
and
1525 'Device or resource busy' not in error
):
1527 LOG
.debug('partprobe %s failed : %s (ignored, waiting 60s)'
1530 if not partprobe_ok
:
1531 raise Error('partprobe %s failed : %s' % (dev
, error
))
1532 command_check_call(['udevadm', 'settle', '--timeout=600'])
1537 Destroy the partition table and content of a given disk.
1539 dev
= os
.path
.realpath(dev
)
1540 dmode
= os
.stat(dev
).st_mode
1541 if not stat
.S_ISBLK(dmode
) or is_partition(dev
):
1542 raise Error('not full block device; cannot zap', dev
)
1544 # Thoroughly wipe all partitions of any traces of
1545 # Filesystems or OSD Journals
1547 # In addition we need to write 10M of data to each partition
1548 # to make sure that after re-creating the same partition
1549 # there is no trace left of any previous Filesystem or OSD
1552 LOG
.debug('Writing zeros to existing partitions on %s', dev
)
1554 for partname
in list_partitions(dev
):
1555 partition
= get_dev_path(partname
)
1568 'of={path}'.format(path
=partition
),
1574 LOG
.debug('Zapping partition table on %s', dev
)
1576 # try to wipe out any GPT partition table backups. sgdisk
1577 # isn't too thorough.
1579 size
= 33 * lba_size
1580 with
open(dev
, 'wb') as dev_file
:
1581 dev_file
.seek(-size
, os
.SEEK_END
)
1582 dev_file
.write(size
* b
'\0')
1602 update_partition(dev
, 'zapped')
1604 except subprocess
.CalledProcessError
as e
:
1608 def adjust_symlink(target
, path
):
1610 if os
.path
.lexists(path
):
1612 mode
= os
.lstat(path
).st_mode
1613 if stat
.S_ISREG(mode
):
1614 LOG
.debug('Removing old file %s', path
)
1616 elif stat
.S_ISLNK(mode
):
1617 old
= os
.readlink(path
)
1619 LOG
.debug('Removing old symlink %s -> %s', path
, old
)
1624 raise Error('unable to remove (or adjust) old file (symlink)',
1627 LOG
.debug('Creating symlink %s -> %s', path
, target
)
1629 os
.symlink(target
, path
)
1631 raise Error('unable to create symlink %s -> %s' % (path
, target
))
1634 def get_mount_options(cluster
, fs_type
):
1635 mount_options
= get_conf(
1637 variable
='osd_mount_options_{fstype}'.format(
1641 if mount_options
is None:
1642 mount_options
= get_conf(
1644 variable
='osd_fs_mount_options_{fstype}'.format(
1649 # remove whitespaces
1650 mount_options
= "".join(mount_options
.split())
1651 return mount_options
1654 class Device(object):
1656 def __init__(self
, path
, args
):
1659 self
.dev_size
= None
1660 self
.partitions
= {}
1661 self
.ptype_map
= None
1662 assert not is_partition(self
.path
)
1664 def create_partition(self
, uuid
, name
, size
=0, num
=0):
1665 ptype
= self
.ptype_tobe_for_name(name
)
1667 num
= get_free_partition_index(dev
=self
.path
)
1669 new
= '--new={num}:0:+{size}M'.format(num
=num
, size
=size
)
1670 if size
> self
.get_dev_size():
1671 LOG
.error('refusing to create %s on %s' % (name
, self
.path
))
1672 LOG
.error('%s size (%sM) is bigger than device (%sM)'
1673 % (name
, size
, self
.get_dev_size()))
1674 raise Error('%s device size (%sM) is not big enough for %s'
1675 % (self
.path
, self
.get_dev_size(), name
))
1677 new
= '--largest-new={num}'.format(num
=num
)
1679 LOG
.debug('Creating %s partition num %d size %d on %s',
1680 name
, num
, size
, self
.path
)
1685 '--change-name={num}:ceph {name}'.format(num
=num
, name
=name
),
1686 '--partition-guid={num}:{uuid}'.format(num
=num
, uuid
=uuid
),
1687 '--typecode={num}:{uuid}'.format(num
=num
, uuid
=ptype
),
1694 update_partition(self
.path
, 'created')
1697 def ptype_tobe_for_name(self
, name
):
1698 LOG
.debug("name = " + name
)
1701 if name
== 'lockbox':
1702 if is_mpath(self
.path
):
1703 return PTYPE
['mpath']['lockbox']['tobe']
1705 return PTYPE
['regular']['lockbox']['tobe']
1706 if self
.ptype_map
is None:
1707 partition
= DevicePartition
.factory(
1708 path
=self
.path
, dev
=None, args
=self
.args
)
1709 self
.ptype_map
= partition
.ptype_map
1710 return self
.ptype_map
[name
]['tobe']
1712 def get_partition(self
, num
):
1713 if num
not in self
.partitions
:
1714 dev
= get_partition_dev(self
.path
, num
)
1715 partition
= DevicePartition
.factory(
1716 path
=self
.path
, dev
=dev
, args
=self
.args
)
1717 partition
.set_partition_number(num
)
1718 self
.partitions
[num
] = partition
1719 return self
.partitions
[num
]
1721 def get_dev_size(self
):
1722 if self
.dev_size
is None:
1723 self
.dev_size
= get_dev_size(self
.path
)
1724 return self
.dev_size
1727 def factory(path
, args
):
1728 return Device(path
, args
)
1731 class DevicePartition(object):
1733 def __init__(self
, args
):
1739 self
.ptype_map
= None
1741 self
.set_variables_ptype()
1744 if self
.uuid
is None:
1745 self
.uuid
= get_partition_uuid(self
.rawdev
)
1748 def get_ptype(self
):
1749 if self
.ptype
is None:
1750 self
.ptype
= get_partition_type(self
.rawdev
)
1753 def set_partition_number(self
, num
):
1756 def get_partition_number(self
):
1759 def set_dev(self
, dev
):
1766 def get_rawdev(self
):
1769 def set_variables_ptype(self
):
1770 self
.ptype_map
= PTYPE
['regular']
1772 def ptype_for_name(self
, name
):
1773 return self
.ptype_map
[name
]['ready']
1776 def factory(path
, dev
, args
):
1777 dmcrypt_type
= CryptHelpers
.get_dmcrypt_type(args
)
1778 if ((path
is not None and is_mpath(path
)) or
1779 (dev
is not None and is_mpath(dev
))):
1780 partition
= DevicePartitionMultipath(args
)
1781 elif dmcrypt_type
== 'luks':
1782 partition
= DevicePartitionCryptLuks(args
)
1783 elif dmcrypt_type
== 'plain':
1784 partition
= DevicePartitionCryptPlain(args
)
1786 partition
= DevicePartition(args
)
1787 partition
.set_dev(dev
)
1791 class DevicePartitionMultipath(DevicePartition
):
1793 def set_variables_ptype(self
):
1794 self
.ptype_map
= PTYPE
['mpath']
1797 class DevicePartitionCrypt(DevicePartition
):
1799 def __init__(self
, args
):
1800 super(DevicePartitionCrypt
, self
).__init
__(args
)
1801 self
.osd_dm_key
= None
1802 self
.cryptsetup_parameters
= CryptHelpers
.get_cryptsetup_parameters(
1804 self
.dmcrypt_type
= CryptHelpers
.get_dmcrypt_type(self
.args
)
1805 self
.dmcrypt_keysize
= CryptHelpers
.get_dmcrypt_keysize(self
.args
)
1807 def setup_crypt(self
):
1812 self
.dev
= _dmcrypt_map(
1814 key
=self
.osd_dm_key
,
1815 _uuid
=self
.get_uuid(),
1816 cryptsetup_parameters
=self
.cryptsetup_parameters
,
1823 dmcrypt_unmap(self
.get_uuid())
1824 self
.dev
= self
.rawdev
1831 class DevicePartitionCryptPlain(DevicePartitionCrypt
):
1836 def setup_crypt(self
):
1837 if self
.osd_dm_key
is not None:
1840 self
.cryptsetup_parameters
+= ['--key-size', str(self
.dmcrypt_keysize
)]
1842 self
.osd_dm_key
= get_dmcrypt_key(
1843 self
.get_uuid(), self
.args
.dmcrypt_key_dir
,
1846 def set_variables_ptype(self
):
1847 self
.ptype_map
= PTYPE
['plain']
1850 class DevicePartitionCryptLuks(DevicePartitionCrypt
):
1855 def setup_crypt(self
):
1856 if self
.osd_dm_key
is not None:
1859 if self
.dmcrypt_keysize
== 1024:
1860 # We don't force this into the cryptsetup_parameters,
1861 # as we want the cryptsetup defaults
1862 # to prevail for the actual LUKS key lengths.
1865 self
.cryptsetup_parameters
+= ['--key-size',
1866 str(self
.dmcrypt_keysize
)]
1868 self
.osd_dm_key
= get_dmcrypt_key(
1869 self
.get_uuid(), self
.args
.dmcrypt_key_dir
,
1872 def set_variables_ptype(self
):
1873 self
.ptype_map
= PTYPE
['luks']
1876 class Prepare(object):
1878 def __init__(self
, args
):
1883 parser
= argparse
.ArgumentParser(add_help
=False)
1884 parser
.add_argument(
1888 help='cluster name to assign this disk to',
1890 parser
.add_argument(
1893 help='cluster uuid to assign this disk to',
1895 parser
.add_argument(
1898 help='unique OSD uuid to assign this disk to',
1900 parser
.add_argument(
1901 '--crush-device-class',
1902 help='crush device class to assign this disk to',
1904 parser
.add_argument(
1906 action
='store_true', default
=None,
1907 help='encrypt DATA and/or JOURNAL devices with dm-crypt',
1909 parser
.add_argument(
1910 '--dmcrypt-key-dir',
1912 default
='/etc/ceph/dmcrypt-keys',
1913 help='directory where dm-crypt keys are stored',
1915 parser
.add_argument(
1918 help='bootstrap-osd keyring path template (%(default)s)',
1919 default
='{statedir}/bootstrap-osd/{cluster}.keyring',
1920 dest
='prepare_key_template',
1922 parser
.add_argument(
1924 action
='store_true', default
=None,
1925 help='let many prepare\'s run in parallel',
1930 def set_subparser(subparsers
):
1933 PrepareData
.parser(),
1936 parents
.extend(PrepareFilestore
.parent_parsers())
1937 parents
.extend(PrepareBluestore
.parent_parsers())
1938 parser
= subparsers
.add_parser(
1941 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
1942 description
=textwrap
.fill(textwrap
.dedent("""\
1943 If the --bluestore argument is given, a bluestore objectstore
1944 will be created. If --filestore is provided, a legacy FileStore
1945 objectstore will be created. If neither is specified, we default
1948 When an entire device is prepared for bluestore, two
1949 partitions are created. The first partition is for metadata,
1950 the second partition is for blocks that contain data.
1952 Unless explicitly specified with --block.db or
1953 --block.wal, the bluestore DB and WAL data is stored on
1954 the main block device. For instance:
1956 ceph-disk prepare --bluestore /dev/sdc
1960 /dev/sdc1 for osd metadata
1961 /dev/sdc2 for block, db, and wal data (the rest of the disk)
1964 If either --block.db or --block.wal are specified to be
1965 the same whole device, they will be created as partition
1966 three and four respectively. For instance:
1968 ceph-disk prepare --bluestore \\
1969 --block.db /dev/sdc \\
1970 --block.wal /dev/sdc \\
1975 /dev/sdc1 for osd metadata
1976 /dev/sdc2 for block (the rest of the disk)
1981 help='Prepare a directory or disk for a Ceph OSD',
1983 parser
.set_defaults(
1989 if self
.args
.no_locking
:
1998 return PrepareBluestore(args
)
2000 return PrepareFilestore(args
)
2004 Prepare
.factory(args
).prepare()
2007 class PrepareFilestore(Prepare
):
2009 def __init__(self
, args
):
2010 super(PrepareFilestore
, self
).__init
__(args
)
2012 self
.lockbox
= Lockbox(args
)
2013 self
.data
= PrepareFilestoreData(args
)
2014 self
.journal
= PrepareJournal(args
)
2017 def parent_parsers():
2019 PrepareJournal
.parser(),
2023 if self
.data
.args
.dmcrypt
:
2024 self
.lockbox
.prepare()
2025 self
.data
.prepare(self
.journal
)
2028 class PrepareBluestore(Prepare
):
2030 def __init__(self
, args
):
2031 super(PrepareBluestore
, self
).__init
__(args
)
2033 self
.lockbox
= Lockbox(args
)
2034 self
.data
= PrepareBluestoreData(args
)
2035 self
.block
= PrepareBluestoreBlock(args
)
2036 self
.blockdb
= PrepareBluestoreBlockDB(args
)
2037 self
.blockwal
= PrepareBluestoreBlockWAL(args
)
2041 parser
= argparse
.ArgumentParser(add_help
=False)
2042 parser
.add_argument(
2045 action
='store_true', default
=True,
2046 help='bluestore objectstore',
2048 parser
.add_argument(
2051 action
='store_false',
2052 help='filestore objectstore',
2057 def parent_parsers():
2059 PrepareBluestore
.parser(),
2060 PrepareBluestoreBlock
.parser(),
2061 PrepareBluestoreBlockDB
.parser(),
2062 PrepareBluestoreBlockWAL
.parser(),
2066 if self
.data
.args
.dmcrypt
:
2067 self
.lockbox
.prepare()
2068 to_prepare_list
= []
2069 if getattr(self
.data
.args
, 'block.db'):
2070 to_prepare_list
.append(self
.blockdb
)
2071 if getattr(self
.data
.args
, 'block.wal'):
2072 to_prepare_list
.append(self
.blockwal
)
2073 to_prepare_list
.append(self
.block
)
2074 self
.data
.prepare(*to_prepare_list
)
2077 class Space(object):
2079 NAMES
= ('block', 'journal', 'block.db', 'block.wal')
2082 class PrepareSpace(object):
2088 def __init__(self
, args
):
2091 self
.space_size
= self
.get_space_size()
2092 if getattr(self
.args
, self
.name
+ '_uuid') is None:
2093 setattr(self
.args
, self
.name
+ '_uuid', str(uuid
.uuid4()))
2094 self
.space_symlink
= None
2095 self
.space_dmcrypt
= None
2100 dmode
= os
.stat(args
.data
).st_mode
2101 if (self
.wants_space() and
2102 stat
.S_ISBLK(dmode
) and
2103 not is_partition(args
.data
) and
2104 getattr(args
, name
) is None and
2105 getattr(args
, name
+ '_file') is None):
2106 LOG
.info('Will colocate %s with data on %s',
2108 setattr(args
, name
, args
.data
)
2110 if getattr(args
, name
) is None:
2111 if getattr(args
, name
+ '_dev'):
2112 raise Error('%s is unspecified; not a block device' %
2113 name
.capitalize(), getattr(args
, name
))
2114 self
.type = self
.NONE
2117 if not os
.path
.exists(getattr(args
, name
)):
2118 if getattr(args
, name
+ '_dev'):
2119 raise Error('%s does not exist; not a block device' %
2120 name
.capitalize(), getattr(args
, name
))
2121 self
.type = self
.FILE
2124 mode
= os
.stat(getattr(args
, name
)).st_mode
2125 if stat
.S_ISBLK(mode
):
2126 if getattr(args
, name
+ '_file'):
2127 raise Error('%s is not a regular file' % name
.capitalize
,
2128 getattr(args
, name
))
2129 self
.type = self
.DEVICE
2132 if stat
.S_ISREG(mode
):
2133 if getattr(args
, name
+ '_dev'):
2134 raise Error('%s is not a block device' % name
.capitalize
,
2135 getattr(args
, name
))
2136 self
.type = self
.FILE
2139 raise Error('%s %s is neither a block device nor regular file' %
2140 (name
.capitalize
, getattr(args
, name
)))
2143 return self
.type == self
.NONE
2146 return self
.type == self
.FILE
2148 def is_device(self
):
2149 return self
.type == self
.DEVICE
2152 def parser(name
, positional
=True):
2153 parser
= argparse
.ArgumentParser(add_help
=False)
2154 parser
.add_argument(
2157 help='unique uuid to assign to the %s' % name
,
2159 parser
.add_argument(
2161 action
='store_true', default
=None,
2162 help='verify that %s is a file' % name
.upper(),
2164 parser
.add_argument(
2166 action
='store_true', default
=None,
2167 help='verify that %s is a block device' % name
.upper(),
2171 parser
.add_argument(
2173 metavar
=name
.upper(),
2175 help=('path to OSD %s disk block device;' % name
+
2176 ' leave out to store %s in file' % name
),
2180 def wants_space(self
):
2183 def populate_data_path(self
, path
):
2184 if self
.type == self
.DEVICE
:
2185 self
.populate_data_path_device(path
)
2186 elif self
.type == self
.FILE
:
2187 self
.populate_data_path_file(path
)
2188 elif self
.type == self
.NONE
:
2191 raise Error('unexpected type ', self
.type)
2193 def populate_data_path_file(self
, path
):
2194 space_uuid
= self
.name
+ '_uuid'
2195 if getattr(self
.args
, space_uuid
) is not None:
2196 write_one_line(path
, space_uuid
,
2197 getattr(self
.args
, space_uuid
))
2198 if self
.space_symlink
is not None:
2199 adjust_symlink(self
.space_symlink
,
2200 os
.path
.join(path
, self
.name
))
2202 def populate_data_path_device(self
, path
):
2203 self
.populate_data_path_file(path
)
2205 if self
.space_dmcrypt
is not None:
2206 adjust_symlink(self
.space_dmcrypt
,
2207 os
.path
.join(path
, self
.name
+ '_dmcrypt'))
2210 os
.unlink(os
.path
.join(path
, self
.name
+ '_dmcrypt'))
2215 if self
.type == self
.DEVICE
:
2216 self
.prepare_device()
2217 elif self
.type == self
.FILE
:
2219 elif self
.type == self
.NONE
:
2222 raise Error('unexpected type ', self
.type)
2224 def prepare_file(self
):
2225 space_filename
= getattr(self
.args
, self
.name
)
2226 if not os
.path
.exists(space_filename
):
2227 LOG
.debug('Creating %s file %s with size 0'
2228 ' (ceph-osd will resize and allocate)',
2231 space_file
= open(space_filename
, 'wb')
2233 path_set_context(space_filename
)
2235 LOG
.debug('%s is file %s',
2236 self
.name
.capitalize(),
2238 LOG
.warning('OSD will not be hot-swappable if %s is '
2239 'not the same device as the osd data' %
2241 self
.space_symlink
= space_filename
2243 def prepare_device(self
):
2244 reusing_partition
= False
2246 if is_partition(getattr(self
.args
, self
.name
)):
2247 LOG
.debug('%s %s is a partition',
2248 self
.name
.capitalize(), getattr(self
.args
, self
.name
))
2249 partition
= DevicePartition
.factory(
2250 path
=None, dev
=getattr(self
.args
, self
.name
), args
=self
.args
)
2251 if isinstance(partition
, DevicePartitionCrypt
):
2252 raise Error(getattr(self
.args
, self
.name
) +
2253 ' partition already exists'
2254 ' and --dmcrypt specified')
2255 LOG
.warning('OSD will not be hot-swappable' +
2256 ' if ' + self
.name
+ ' is not' +
2257 ' the same device as the osd data')
2258 if partition
.get_ptype() == partition
.ptype_for_name(self
.name
):
2259 LOG
.debug('%s %s was previously prepared with '
2260 'ceph-disk. Reusing it.',
2261 self
.name
.capitalize(),
2262 getattr(self
.args
, self
.name
))
2263 reusing_partition
= True
2264 # Read and reuse the partition uuid from this journal's
2265 # previous life. We reuse the uuid instead of changing it
2266 # because udev does not reliably notice changes to an
2267 # existing partition's GUID. See
2268 # http://tracker.ceph.com/issues/10146
2269 setattr(self
.args
, self
.name
+ '_uuid', partition
.get_uuid())
2270 LOG
.debug('Reusing %s with uuid %s',
2272 getattr(self
.args
, self
.name
+ '_uuid'))
2274 LOG
.warning('%s %s was not prepared with '
2275 'ceph-disk. Symlinking directly.',
2276 self
.name
.capitalize(),
2277 getattr(self
.args
, self
.name
))
2278 self
.space_symlink
= getattr(self
.args
, self
.name
)
2281 self
.space_symlink
= '/dev/disk/by-partuuid/{uuid}'.format(
2282 uuid
=getattr(self
.args
, self
.name
+ '_uuid'))
2284 if self
.args
.dmcrypt
:
2285 self
.space_dmcrypt
= self
.space_symlink
2286 self
.space_symlink
= '/dev/mapper/{uuid}'.format(
2287 uuid
=getattr(self
.args
, self
.name
+ '_uuid'))
2289 if reusing_partition
:
2290 # confirm that the space_symlink exists. It should since
2291 # this was an active space
2292 # in the past. Continuing otherwise would be futile.
2293 assert os
.path
.exists(self
.space_symlink
)
2296 num
= self
.desired_partition_number()
2299 LOG
.warning('OSD will not be hot-swappable if %s '
2300 'is not the same device as the osd data',
2303 device
= Device
.factory(getattr(self
.args
, self
.name
), self
.args
)
2304 num
= device
.create_partition(
2305 uuid
=getattr(self
.args
, self
.name
+ '_uuid'),
2307 size
=self
.space_size
,
2310 partition
= device
.get_partition(num
)
2312 LOG
.debug('%s is GPT partition %s',
2313 self
.name
.capitalize(),
2316 if isinstance(partition
, DevicePartitionCrypt
):
2323 '--typecode={num}:{uuid}'.format(
2325 uuid
=partition
.ptype_for_name(self
.name
),
2328 getattr(self
.args
, self
.name
),
2331 update_partition(getattr(self
.args
, self
.name
), 'prepared')
2333 LOG
.debug('%s is GPT partition %s',
2334 self
.name
.capitalize(),
2338 class PrepareJournal(PrepareSpace
):
2340 def __init__(self
, args
):
2341 self
.name
= 'journal'
2342 (self
.allows_journal
,
2344 self
.needs_journal
) = check_journal_reqs(args
)
2346 if args
.journal
and not self
.allows_journal
:
2347 raise Error('journal specified but not allowed by osd backend')
2349 super(PrepareJournal
, self
).__init
__(args
)
2351 def wants_space(self
):
2352 return self
.wants_journal
2354 def get_space_size(self
):
2355 return int(get_conf_with_default(
2356 cluster
=self
.args
.cluster
,
2357 variable
='osd_journal_size',
2360 def desired_partition_number(self
):
2361 if self
.args
.journal
== self
.args
.data
:
2362 # we're sharing the disk between osd data and journal;
2363 # make journal be partition number 2
2371 return PrepareSpace
.parser('journal')
2374 class PrepareBluestoreBlock(PrepareSpace
):
2376 def __init__(self
, args
):
2378 super(PrepareBluestoreBlock
, self
).__init
__(args
)
2380 def get_space_size(self
):
2381 block_size
= get_conf(
2382 cluster
=self
.args
.cluster
,
2383 variable
='bluestore_block_size',
2386 if block_size
is None:
2387 return 0 # get as much space as possible
2389 return int(block_size
) / 1048576 # MB
2391 def desired_partition_number(self
):
2392 if self
.args
.block
== self
.args
.data
:
2400 return PrepareSpace
.parser('block')
2403 class PrepareBluestoreBlockDB(PrepareSpace
):
2405 def __init__(self
, args
):
2406 self
.name
= 'block.db'
2407 super(PrepareBluestoreBlockDB
, self
).__init
__(args
)
2409 def get_space_size(self
):
2410 block_db_size
= get_conf(
2411 cluster
=self
.args
.cluster
,
2412 variable
='bluestore_block_db_size',
2415 if block_db_size
is None or int(block_db_size
) == 0:
2416 block_size
= get_conf(
2417 cluster
=self
.args
.cluster
,
2418 variable
='bluestore_block_size',
2420 if block_size
is None:
2422 size
= int(block_size
) / 100 / 1048576
2423 return max(size
, 1024) # MB
2425 return int(block_db_size
) / 1048576 # MB
2427 def desired_partition_number(self
):
2428 if getattr(self
.args
, 'block.db') == self
.args
.data
:
2434 def wants_space(self
):
2439 parser
= PrepareSpace
.parser('block.db', positional
=False)
2440 parser
.add_argument(
2443 help='path to the device or file for bluestore block.db',
2448 class PrepareBluestoreBlockWAL(PrepareSpace
):
2450 def __init__(self
, args
):
2451 self
.name
= 'block.wal'
2452 super(PrepareBluestoreBlockWAL
, self
).__init
__(args
)
2454 def get_space_size(self
):
2455 block_size
= get_conf(
2456 cluster
=self
.args
.cluster
,
2457 variable
='bluestore_block_wal_size',
2460 if block_size
is None:
2461 return 576 # MB, default value
2463 return int(block_size
) / 1048576 # MB
2465 def desired_partition_number(self
):
2466 if getattr(self
.args
, 'block.wal') == self
.args
.data
:
2472 def wants_space(self
):
2477 parser
= PrepareSpace
.parser('block.wal', positional
=False)
2478 parser
.add_argument(
2481 help='path to the device or file for bluestore block.wal',
2486 class CryptHelpers(object):
2489 def get_cryptsetup_parameters(args
):
2490 cryptsetup_parameters_str
= get_conf(
2491 cluster
=args
.cluster
,
2492 variable
='osd_cryptsetup_parameters',
2494 if cryptsetup_parameters_str
is None:
2497 return shlex
.split(cryptsetup_parameters_str
)
2500 def get_dmcrypt_keysize(args
):
2501 dmcrypt_keysize_str
= get_conf(
2502 cluster
=args
.cluster
,
2503 variable
='osd_dmcrypt_key_size',
2505 dmcrypt_type
= CryptHelpers
.get_dmcrypt_type(args
)
2506 if dmcrypt_type
== 'luks':
2507 if dmcrypt_keysize_str
is None:
2508 # As LUKS will hash the 'passphrase' in .luks.key
2509 # into a key, set a large default
2510 # so if not updated for some time, it is still a
2515 return int(dmcrypt_keysize_str
)
2516 elif dmcrypt_type
== 'plain':
2517 if dmcrypt_keysize_str
is None:
2518 # This value is hard-coded in the udev script
2521 LOG
.warning('ensure the 95-ceph-osd.rules file has '
2522 'been copied to /etc/udev/rules.d '
2523 'and modified to call cryptsetup '
2524 'with --key-size=%s' % dmcrypt_keysize_str
)
2525 return int(dmcrypt_keysize_str
)
2530 def get_dmcrypt_type(args
):
2531 if hasattr(args
, 'dmcrypt') and args
.dmcrypt
:
2532 dmcrypt_type
= get_conf(
2533 cluster
=args
.cluster
,
2534 variable
='osd_dmcrypt_type',
2537 if dmcrypt_type
is None or dmcrypt_type
== 'luks':
2539 elif dmcrypt_type
== 'plain':
2542 raise Error('invalid osd_dmcrypt_type parameter '
2543 '(must be luks or plain): ', dmcrypt_type
)
2548 class Lockbox(object):
2550 def __init__(self
, args
):
2552 self
.partition
= None
2555 if hasattr(self
.args
, 'lockbox') and self
.args
.lockbox
is None:
2556 self
.args
.lockbox
= self
.args
.data
2558 def set_partition(self
, partition
):
2559 self
.partition
= partition
2563 parser
= argparse
.ArgumentParser(add_help
=False)
2564 parser
.add_argument(
2566 help='path to the device to store the lockbox',
2568 parser
.add_argument(
2571 help='unique lockbox uuid',
2575 def create_partition(self
):
2576 self
.device
= Device
.factory(self
.args
.lockbox
, argparse
.Namespace())
2577 partition_number
= 3
2578 self
.device
.create_partition(uuid
=self
.args
.lockbox_uuid
,
2580 num
=partition_number
,
2582 return self
.device
.get_partition(partition_number
)
2584 def set_or_create_partition(self
):
2585 if is_partition(self
.args
.lockbox
):
2586 LOG
.debug('OSD lockbox device %s is a partition',
2588 self
.partition
= DevicePartition
.factory(
2589 path
=None, dev
=self
.args
.lockbox
, args
=self
.args
)
2590 ptype
= self
.partition
.get_ptype()
2591 ready
= Ptype
.get_ready_by_name('lockbox')
2592 if ptype
not in ready
:
2593 LOG
.warning('incorrect partition UUID: %s, expected %s'
2594 % (ptype
, str(ready
)))
2596 LOG
.debug('Creating osd partition on %s',
2598 self
.partition
= self
.create_partition()
2600 def create_key(self
):
2601 key_size
= CryptHelpers
.get_dmcrypt_keysize(self
.args
)
2602 key
= open('/dev/urandom', 'rb').read(key_size
/ 8)
2603 base64_key
= base64
.b64encode(key
)
2604 cluster
= self
.args
.cluster
2605 bootstrap
= self
.args
.prepare_key_template
.format(cluster
=cluster
,
2610 '--cluster', cluster
,
2611 '--name', 'client.bootstrap-osd',
2612 '--keyring', bootstrap
,
2615 'dm-crypt/osd/' + self
.args
.osd_uuid
+ '/luks',
2619 keyring
, stderr
, ret
= command(
2622 '--cluster', cluster
,
2623 '--name', 'client.bootstrap-osd',
2624 '--keyring', bootstrap
,
2627 'client.osd-lockbox.' + self
.args
.osd_uuid
,
2629 ('allow command "config-key get" with key="dm-crypt/osd/' +
2630 self
.args
.osd_uuid
+ '/luks"'),
2633 LOG
.debug("stderr " + stderr
)
2635 path
= self
.get_mount_point()
2636 open(os
.path
.join(path
, 'keyring'), 'w').write(keyring
)
2637 write_one_line(path
, 'key-management-mode', KEY_MANAGEMENT_MODE_V1
)
2639 def symlink_spaces(self
, path
):
2640 target
= self
.get_mount_point()
2641 for name
in Space
.NAMES
:
2642 if (hasattr(self
.args
, name
+ '_uuid') and
2643 getattr(self
.args
, name
+ '_uuid')):
2644 uuid
= getattr(self
.args
, name
+ '_uuid')
2645 symlink
= os
.path
.join(STATEDIR
, 'osd-lockbox', uuid
)
2646 adjust_symlink(target
, symlink
)
2647 write_one_line(path
, name
+ '-uuid', uuid
)
2650 maybe_mkdir(os
.path
.join(STATEDIR
, 'osd-lockbox'))
2651 args
= ['mkfs', '-t', 'ext4', self
.partition
.get_dev()]
2652 LOG
.debug('Creating lockbox fs on %s: ' + str(" ".join(args
)))
2653 command_check_call(args
)
2654 path
= self
.get_mount_point()
2656 args
= ['mount', '-t', 'ext4', self
.partition
.get_dev(), path
]
2657 LOG
.debug('Mounting lockbox ' + str(" ".join(args
)))
2658 command_check_call(args
)
2659 write_one_line(path
, 'osd-uuid', self
.args
.osd_uuid
)
2660 if self
.args
.cluster_uuid
is None:
2661 self
.args
.cluster_uuid
= get_fsid(cluster
=self
.args
.cluster
)
2662 write_one_line(path
, 'ceph_fsid', self
.args
.cluster_uuid
)
2664 self
.symlink_spaces(path
)
2665 write_one_line(path
, 'magic', CEPH_LOCKBOX_ONDISK_MAGIC
)
2666 if self
.device
is not None:
2670 '--typecode={num}:{uuid}'.format(
2671 num
=self
.partition
.get_partition_number(),
2672 uuid
=self
.partition
.ptype_for_name('lockbox'),
2675 get_partition_base(self
.partition
.get_dev()),
2679 def get_mount_point(self
):
2680 return os
.path
.join(STATEDIR
, 'osd-lockbox', self
.args
.osd_uuid
)
2682 def get_osd_uuid(self
):
2683 return self
.args
.osd_uuid
2686 path
= is_mounted(self
.partition
.get_dev())
2688 LOG
.info("Lockbox already mounted at " + path
)
2691 path
= tempfile
.mkdtemp(
2693 dir=STATEDIR
+ '/tmp',
2695 args
= ['mount', '-t', 'ext4', '-o', 'ro',
2696 self
.partition
.get_dev(),
2698 LOG
.debug('Mounting lockbox temporarily ' + str(" ".join(args
)))
2699 command_check_call(args
)
2700 self
.args
.osd_uuid
= get_oneliner(path
, 'osd-uuid')
2701 command_check_call(['umount', path
])
2702 LOG
.debug('Mounting lockbox readonly ' + str(" ".join(args
)))
2703 args
= ['mount', '-t', 'ext4', '-o', 'ro',
2704 self
.partition
.get_dev(),
2705 self
.get_mount_point()]
2706 command_check_call(args
)
2707 for name
in Space
.NAMES
+ ('osd',):
2708 uuid_path
= os
.path
.join(self
.get_mount_point(), name
+ '-uuid')
2709 if os
.path
.exists(uuid_path
):
2710 uuid
= get_oneliner(self
.get_mount_point(), name
+ '-uuid')
2711 dev
= os
.path
.join('/dev/disk/by-partuuid/', uuid
.lower())
2712 args
= ['ceph-disk', 'trigger', dev
]
2713 command_check_call(args
)
2716 verify_not_in_use(self
.args
.lockbox
, check_partitions
=True)
2717 self
.set_or_create_partition()
2721 class PrepareData(object):
2726 def __init__(self
, args
):
2729 self
.partition
= None
2731 if self
.args
.cluster_uuid
is None:
2732 self
.args
.cluster_uuid
= get_fsid(cluster
=self
.args
.cluster
)
2734 if self
.args
.osd_uuid
is None:
2735 self
.args
.osd_uuid
= str(uuid
.uuid4())
2738 dmode
= os
.stat(self
.args
.data
).st_mode
2740 if stat
.S_ISDIR(dmode
):
2741 self
.type = self
.FILE
2742 elif stat
.S_ISBLK(dmode
):
2743 self
.type = self
.DEVICE
2745 raise Error('not a dir or block device', self
.args
.data
)
2748 return self
.type == self
.FILE
2750 def is_device(self
):
2751 return self
.type == self
.DEVICE
2755 parser
= argparse
.ArgumentParser(add_help
=False)
2756 parser
.add_argument(
2758 help='file system type to use (e.g. "ext4")',
2760 parser
.add_argument(
2762 action
='store_true', default
=None,
2763 help='destroy the partition table (and content) of a disk',
2765 parser
.add_argument(
2767 action
='store_true', default
=None,
2768 help='verify that DATA is a dir',
2770 parser
.add_argument(
2772 action
='store_true', default
=None,
2773 help='verify that DATA is a block device',
2775 parser
.add_argument(
2778 help='path to OSD data (a disk block device or directory)',
2782 def populate_data_path_file(self
, path
, *to_prepare_list
):
2783 self
.populate_data_path(path
, *to_prepare_list
)
2785 def populate_data_path(self
, path
, *to_prepare_list
):
2786 if os
.path
.exists(os
.path
.join(path
, 'magic')):
2787 LOG
.debug('Data dir %s already exists', path
)
2790 LOG
.debug('Preparing osd data dir %s', path
)
2792 if self
.args
.osd_uuid
is None:
2793 self
.args
.osd_uuid
= str(uuid
.uuid4())
2795 write_one_line(path
, 'ceph_fsid', self
.args
.cluster_uuid
)
2796 write_one_line(path
, 'fsid', self
.args
.osd_uuid
)
2797 if self
.args
.crush_device_class
:
2798 write_one_line(path
, 'crush_device_class',
2799 self
.args
.crush_device_class
)
2800 write_one_line(path
, 'magic', CEPH_OSD_ONDISK_MAGIC
)
2802 for to_prepare
in to_prepare_list
:
2803 to_prepare
.populate_data_path(path
)
2805 def prepare(self
, *to_prepare_list
):
2806 if self
.type == self
.DEVICE
:
2807 self
.prepare_device(*to_prepare_list
)
2808 elif self
.type == self
.FILE
:
2809 self
.prepare_file(*to_prepare_list
)
2811 raise Error('unexpected type ', self
.type)
2813 def prepare_file(self
, *to_prepare_list
):
2815 if not os
.path
.exists(self
.args
.data
):
2816 raise Error('data path for directory does not exist',
2819 if self
.args
.data_dev
:
2820 raise Error('data path is not a block device', self
.args
.data
)
2822 for to_prepare
in to_prepare_list
:
2823 to_prepare
.prepare()
2825 self
.populate_data_path_file(self
.args
.data
, *to_prepare_list
)
2827 def sanity_checks(self
):
2828 if not os
.path
.exists(self
.args
.data
):
2829 raise Error('data path for device does not exist',
2831 verify_not_in_use(self
.args
.data
,
2832 check_partitions
=not self
.args
.dmcrypt
)
2834 def set_variables(self
):
2835 if self
.args
.fs_type
is None:
2836 self
.args
.fs_type
= get_conf(
2837 cluster
=self
.args
.cluster
,
2838 variable
='osd_mkfs_type',
2840 if self
.args
.fs_type
is None:
2841 self
.args
.fs_type
= get_conf(
2842 cluster
=self
.args
.cluster
,
2843 variable
='osd_fs_type',
2845 if self
.args
.fs_type
is None:
2846 self
.args
.fs_type
= DEFAULT_FS_TYPE
2848 self
.mkfs_args
= get_conf(
2849 cluster
=self
.args
.cluster
,
2850 variable
='osd_mkfs_options_{fstype}'.format(
2851 fstype
=self
.args
.fs_type
,
2854 if self
.mkfs_args
is None:
2855 self
.mkfs_args
= get_conf(
2856 cluster
=self
.args
.cluster
,
2857 variable
='osd_fs_mkfs_options_{fstype}'.format(
2858 fstype
=self
.args
.fs_type
,
2862 self
.mount_options
= get_mount_options(cluster
=self
.args
.cluster
,
2863 fs_type
=self
.args
.fs_type
)
2865 if self
.args
.osd_uuid
is None:
2866 self
.args
.osd_uuid
= str(uuid
.uuid4())
2868 def prepare_device(self
, *to_prepare_list
):
2869 self
.sanity_checks()
2870 self
.set_variables()
2871 if self
.args
.zap_disk
is not None:
2874 def create_data_partition(self
):
2875 device
= Device
.factory(self
.args
.data
, self
.args
)
2876 partition_number
= 1
2877 device
.create_partition(uuid
=self
.args
.osd_uuid
,
2879 num
=partition_number
,
2880 size
=self
.get_space_size())
2881 return device
.get_partition(partition_number
)
2883 def set_data_partition(self
):
2884 if is_partition(self
.args
.data
):
2885 LOG
.debug('OSD data device %s is a partition',
2887 self
.partition
= DevicePartition
.factory(
2888 path
=None, dev
=self
.args
.data
, args
=self
.args
)
2889 ptype
= self
.partition
.get_ptype()
2890 ready
= Ptype
.get_ready_by_name('osd')
2891 if ptype
not in ready
:
2892 LOG
.warning('incorrect partition UUID: %s, expected %s'
2893 % (ptype
, str(ready
)))
2895 LOG
.debug('Creating osd partition on %s',
2897 self
.partition
= self
.create_data_partition()
2899 def populate_data_path_device(self
, *to_prepare_list
):
2900 partition
= self
.partition
2902 if isinstance(partition
, DevicePartitionCrypt
):
2911 if self
.mkfs_args
is not None:
2912 args
.extend(self
.mkfs_args
.split())
2913 if self
.args
.fs_type
== 'xfs':
2914 args
.extend(['-f']) # always force
2916 args
.extend(MKFS_ARGS
.get(self
.args
.fs_type
, []))
2919 partition
.get_dev(),
2921 LOG
.debug('Creating %s fs on %s',
2922 self
.args
.fs_type
, partition
.get_dev())
2923 command_check_call(args
, exit
=True)
2925 path
= mount(dev
=partition
.get_dev(),
2926 fstype
=self
.args
.fs_type
,
2927 options
=self
.mount_options
)
2930 self
.populate_data_path(path
, *to_prepare_list
)
2932 path_set_context(path
)
2935 if isinstance(partition
, DevicePartitionCrypt
):
2938 if not is_partition(self
.args
.data
):
2942 '--typecode=%d:%s' % (partition
.get_partition_number(),
2943 partition
.ptype_for_name('osd')),
2949 update_partition(self
.args
.data
, 'prepared')
2950 command_check_call(['udevadm', 'trigger',
2953 os
.path
.basename(partition
.rawdev
)])
2956 class PrepareFilestoreData(PrepareData
):
2958 def get_space_size(self
):
2959 return 0 # get as much space as possible
2961 def prepare_device(self
, *to_prepare_list
):
2962 super(PrepareFilestoreData
, self
).prepare_device(*to_prepare_list
)
2963 for to_prepare
in to_prepare_list
:
2964 to_prepare
.prepare()
2965 self
.set_data_partition()
2966 self
.populate_data_path_device(*to_prepare_list
)
2968 def populate_data_path(self
, path
, *to_prepare_list
):
2969 super(PrepareFilestoreData
, self
).populate_data_path(path
,
2971 write_one_line(path
, 'type', 'filestore')
2974 class PrepareBluestoreData(PrepareData
):
2976 def get_space_size(self
):
2979 def prepare_device(self
, *to_prepare_list
):
2980 super(PrepareBluestoreData
, self
).prepare_device(*to_prepare_list
)
2981 self
.set_data_partition()
2982 for to_prepare
in to_prepare_list
:
2983 to_prepare
.prepare()
2984 self
.populate_data_path_device(*to_prepare_list
)
2986 def populate_data_path(self
, path
, *to_prepare_list
):
2987 super(PrepareBluestoreData
, self
).populate_data_path(path
,
2989 write_one_line(path
, 'type', 'bluestore')
2993 # Temporary workaround: if ceph-osd --mkfs does not
2994 # complete within 5 minutes, assume it is blocked
2995 # because of http://tracker.ceph.com/issues/13522
2996 # and retry a few times.
2998 # Remove this function calls with command_check_call
2999 # when http://tracker.ceph.com/issues/13522 is fixed
3001 def ceph_osd_mkfs(arguments
):
3002 timeout
= _get_command_executable(['timeout'])
3004 error
= 'unknown error'
3005 for delay
in os
.environ
.get('CEPH_OSD_MKFS_DELAYS',
3006 '300 300 300 300 300').split():
3008 _check_output(timeout
+ [delay
] + arguments
)
3011 except subprocess
.CalledProcessError
as e
:
3013 if e
.returncode
== 124: # timeout fired, retry
3014 LOG
.debug('%s timed out : %s (retry)'
3015 % (str(arguments
), error
))
3019 raise Error('%s failed : %s' % (str(arguments
), error
))
3029 monmap
= os
.path
.join(path
, 'activate.monmap')
3033 '--cluster', cluster
,
3034 '--name', 'client.bootstrap-osd',
3035 '--keyring', keyring
,
3036 'mon', 'getmap', '-o', monmap
,
3040 osd_type
= read_one_line(path
, 'type')
3042 if osd_type
== 'bluestore':
3046 '--cluster', cluster
,
3053 '--keyring', os
.path
.join(path
, 'keyring'),
3054 '--setuser', get_ceph_user(),
3055 '--setgroup', get_ceph_group(),
3058 elif osd_type
== 'filestore':
3062 '--cluster', cluster
,
3068 '--osd-journal', os
.path
.join(path
, 'journal'),
3070 '--keyring', os
.path
.join(path
, 'keyring'),
3071 '--setuser', get_ceph_user(),
3072 '--setgroup', get_ceph_group(),
3076 raise Error('unrecognized objectstore type %s' % osd_type
)
3086 # try dumpling+ cap scheme
3090 '--cluster', cluster
,
3091 '--name', 'client.bootstrap-osd',
3092 '--keyring', keyring
,
3093 'auth', 'add', 'osd.{osd_id}'.format(osd_id
=osd_id
),
3094 '-i', os
.path
.join(path
, 'keyring'),
3096 'mon', 'allow profile osd',
3099 except subprocess
.CalledProcessError
as err
:
3100 if err
.returncode
== errno
.EINVAL
:
3101 # try old cap scheme
3105 '--cluster', cluster
,
3106 '--name', 'client.bootstrap-osd',
3107 '--keyring', keyring
,
3108 'auth', 'add', 'osd.{osd_id}'.format(osd_id
=osd_id
),
3109 '-i', os
.path
.join(path
, 'keyring'),
3118 def get_mount_point(cluster
, osd_id
):
3119 parent
= STATEDIR
+ '/osd'
3120 return os
.path
.join(
3122 '{cluster}-{osd_id}'.format(cluster
=cluster
, osd_id
=osd_id
),
3134 LOG
.debug('Moving mount to final location...')
3135 osd_data
= get_mount_point(cluster
, osd_id
)
3136 maybe_mkdir(osd_data
)
3138 # pick best-of-breed mount options based on fs type
3139 if mount_options
is None:
3140 mount_options
= MOUNT_OPTIONS
.get(fstype
, '')
3142 # we really want to mount --move, but that is not supported when
3143 # the parent mount is shared, as it is by default on RH, Fedora,
3144 # and probably others. Also, --bind doesn't properly manipulate
3145 # /etc/mtab, which *still* isn't a symlink to /proc/mounts despite
3146 # this being 2013. Instead, mount the original device at the final
3161 '-l', # lazy, in case someone else is peeking at the
3170 # For upgrade purposes, to make sure there are no competing units,
3171 # both --runtime unit and the default should be disabled. There can be
3172 # two units at the same time: one with --runtime and another without
3173 # it. If, for any reason (manual or ceph-disk) the two units co-exist
3174 # they will compete with each other.
3176 def systemd_disable(
3180 # ensure there is no duplicate ceph-osd@.service
3181 for style
in ([], ['--runtime']):
3186 'ceph-osd@{osd_id}'.format(osd_id
=osd_id
),
3195 systemd_disable(path
, osd_id
)
3196 if is_mounted(path
):
3197 style
= ['--runtime']
3204 'ceph-osd@{osd_id}'.format(osd_id
=osd_id
),
3211 'ceph-osd@{osd_id}'.format(osd_id
=osd_id
),
3220 systemd_disable(path
, osd_id
)
3225 'ceph-osd@{osd_id}'.format(osd_id
=osd_id
),
3234 LOG
.debug('Starting %s osd.%s...', cluster
, osd_id
)
3236 path
= (STATEDIR
+ '/osd/{cluster}-{osd_id}').format(
3237 cluster
=cluster
, osd_id
=osd_id
)
3240 if os
.path
.exists(os
.path
.join(path
, 'upstart')):
3244 # use emit, not start, because start would fail if the
3245 # instance was already running
3247 # since the daemon starting doesn't guarantee much about
3248 # the service being operational anyway, don't bother
3253 'cluster={cluster}'.format(cluster
=cluster
),
3254 'id={osd_id}'.format(osd_id
=osd_id
),
3257 elif os
.path
.exists(os
.path
.join(path
, 'sysvinit')):
3258 if os
.path
.exists('/usr/sbin/service'):
3259 svc
= '/usr/sbin/service'
3261 svc
= '/sbin/service'
3267 '{cluster}'.format(cluster
=cluster
),
3269 'osd.{osd_id}'.format(osd_id
=osd_id
),
3272 elif os
.path
.exists(os
.path
.join(path
, 'systemd')):
3273 systemd_start(path
, osd_id
)
3274 elif os
.path
.exists(os
.path
.join(path
, 'openrc')):
3275 base_script
= '/etc/init.d/ceph-osd'
3276 osd_script
= '{base}.{osd_id}'.format(
3280 if not os
.path
.exists(osd_script
):
3281 os
.symlink(base_script
, osd_script
)
3288 elif os
.path
.exists(os
.path
.join(path
, 'bsdrc')):
3291 '/usr/sbin/service', 'ceph', 'start',
3292 'osd.{osd_id}'.format(osd_id
=osd_id
),
3296 raise Error('{cluster} osd.{osd_id} '
3297 'is not tagged with an init system'
3302 except subprocess
.CalledProcessError
as e
:
3303 raise Error('ceph osd start failed', e
)
3310 LOG
.debug('Stoping %s osd.%s...', cluster
, osd_id
)
3312 path
= (STATEDIR
+ '/osd/{cluster}-{osd_id}').format(
3313 cluster
=cluster
, osd_id
=osd_id
)
3316 if os
.path
.exists(os
.path
.join(path
, 'upstart')):
3322 'cluster={cluster}'.format(cluster
=cluster
),
3323 'id={osd_id}'.format(osd_id
=osd_id
),
3326 elif os
.path
.exists(os
.path
.join(path
, 'sysvinit')):
3327 svc
= which('service')
3333 '{cluster}'.format(cluster
=cluster
),
3335 'osd.{osd_id}'.format(osd_id
=osd_id
),
3338 elif os
.path
.exists(os
.path
.join(path
, 'systemd')):
3339 systemd_stop(path
, osd_id
)
3340 elif os
.path
.exists(os
.path
.join(path
, 'openrc')):
3343 '/etc/init.d/ceph-osd.{osd_id}'.format(osd_id
=osd_id
),
3347 elif os
.path
.exists(os
.path
.join(path
, 'bsdrc')):
3350 '/usr/local/etc/rc.d/ceph stop osd.{osd_id}'
3351 .format(osd_id
=osd_id
),
3355 raise Error('{cluster} osd.{osd_id} '
3356 'is not tagged with an init system'
3357 .format(cluster
=cluster
, osd_id
=osd_id
))
3358 except subprocess
.CalledProcessError
as e
:
3359 raise Error('ceph osd stop failed', e
)
3362 def detect_fstype(dev
):
3364 fstype
= _check_output(
3372 fstype
= _check_output(
3375 # we don't want stale cached results
3383 fstype
= must_be_one_line(fstype
)
3387 def dmcrypt_is_mapped(uuid
):
3388 path
= os
.path
.join('/dev/mapper', uuid
)
3389 if os
.path
.exists(path
):
3395 def dmcrypt_map(dev
, dmcrypt_key_dir
):
3396 ptype
= get_partition_type(dev
)
3397 if ptype
in Ptype
.get_ready_by_type('plain'):
3399 cryptsetup_parameters
= ['--key-size', '256']
3400 elif ptype
in Ptype
.get_ready_by_type('luks'):
3402 cryptsetup_parameters
= []
3404 raise Error('--dmcrypt called for dev %s with invalid ptype %s'
3406 part_uuid
= get_partition_uuid(dev
)
3407 dmcrypt_key
= get_dmcrypt_key(part_uuid
, dmcrypt_key_dir
, luks
)
3408 return _dmcrypt_map(
3412 cryptsetup_parameters
=cryptsetup_parameters
,
3420 activate_key_template
,
3428 part_uuid
= get_partition_uuid(dev
)
3429 dev
= dmcrypt_map(dev
, dmcrypt_key_dir
)
3431 fstype
= detect_fstype(dev
=dev
)
3432 except (subprocess
.CalledProcessError
,
3434 TooManyLinesError
) as e
:
3435 raise FilesystemTypeError(
3436 'device {dev}'.format(dev
=dev
),
3440 # TODO always using mount options from cluster=ceph for
3441 # now; see http://tracker.newdream.net/issues/3253
3442 mount_options
= get_mount_options(cluster
='ceph', fs_type
=fstype
)
3444 path
= mount(dev
=dev
, fstype
=fstype
, options
=mount_options
)
3446 # check if the disk is deactive, change the journal owner, group
3447 # mode for correct user and group.
3448 if os
.path
.exists(os
.path
.join(path
, 'deactive')):
3449 # logging to syslog will help us easy to know udev triggered failure
3452 # we need to unmap again because dmcrypt map will create again
3453 # on bootup stage (due to deactivate)
3454 if '/dev/mapper/' in dev
:
3455 part_uuid
= dev
.replace('/dev/mapper/', '')
3456 dmcrypt_unmap(part_uuid
)
3457 LOG
.info('OSD deactivated! reactivate with: --reactivate')
3458 raise Error('OSD deactivated! reactivate with: --reactivate')
3459 # flag to activate a deactive osd.
3467 (osd_id
, cluster
) = activate(path
, activate_key_template
, init
)
3469 # Now active successfully
3470 # If we got reactivate and deactive, remove the deactive file
3471 if deactive
and reactivate
:
3472 os
.remove(os
.path
.join(path
, 'deactive'))
3473 LOG
.info('Remove `deactive` file.')
3475 # check if the disk is already active, or if something else is already
3479 src_dev
= os
.stat(path
).st_dev
3481 dst_dev
= os
.stat((STATEDIR
+ '/osd/{cluster}-{osd_id}').format(
3483 osd_id
=osd_id
)).st_dev
3484 if src_dev
== dst_dev
:
3487 parent_dev
= os
.stat(STATEDIR
+ '/osd').st_dev
3488 if dst_dev
!= parent_dev
:
3490 elif os
.listdir(get_mount_point(cluster
, osd_id
)):
3491 LOG
.info(get_mount_point(cluster
, osd_id
) +
3492 " is not empty, won't override")
3499 LOG
.info('%s osd.%s already mounted in position; unmounting ours.'
3500 % (cluster
, osd_id
))
3503 raise Error('another %s osd.%s already mounted in position '
3504 '(old/different cluster instance?); unmounting ours.'
3505 % (cluster
, osd_id
))
3513 mount_options
=mount_options
,
3515 return cluster
, osd_id
3518 LOG
.error('Failed to activate')
3522 # remove our temp dir
3523 if os
.path
.exists(path
):
3529 activate_key_template
,
3533 if not os
.path
.exists(path
):
3535 'directory %s does not exist' % path
3538 (osd_id
, cluster
) = activate(path
, activate_key_template
, init
)
3540 if init
not in (None, 'none'):
3541 canonical
= (STATEDIR
+ '/osd/{cluster}-{osd_id}').format(
3544 if path
!= canonical
:
3545 # symlink it from the proper location
3547 if os
.path
.lexists(canonical
):
3548 old
= os
.readlink(canonical
)
3550 LOG
.debug('Removing old symlink %s -> %s', canonical
, old
)
3552 os
.unlink(canonical
)
3554 raise Error('unable to remove old symlink', canonical
)
3558 LOG
.debug('Creating symlink %s -> %s', canonical
, path
)
3560 os
.symlink(path
, canonical
)
3562 raise Error('unable to create symlink %s -> %s'
3563 % (canonical
, path
))
3565 return cluster
, osd_id
3568 def find_cluster_by_uuid(_uuid
):
3570 Find a cluster name by searching /etc/ceph/*.conf for a conf file
3571 with the right uuid.
3573 _uuid
= _uuid
.lower()
3575 if not os
.path
.exists(SYSCONFDIR
):
3577 for conf_file
in os
.listdir(SYSCONFDIR
):
3578 if not conf_file
.endswith('.conf'):
3580 cluster
= conf_file
[:-5]
3582 fsid
= get_fsid(cluster
)
3584 if 'getting cluster uuid from configuration failed' not in str(e
):
3586 no_fsid
.append(cluster
)
3590 # be tolerant of /etc/ceph/ceph.conf without an fsid defined.
3591 if len(no_fsid
) == 1 and no_fsid
[0] == 'ceph':
3592 LOG
.warning('No fsid defined in ' + SYSCONFDIR
+
3593 '/ceph.conf; using anyway')
3600 activate_key_template
,
3604 check_osd_magic(path
)
3606 ceph_fsid
= read_one_line(path
, 'ceph_fsid')
3607 if ceph_fsid
is None:
3608 raise Error('No cluster uuid assigned.')
3609 LOG
.debug('Cluster uuid is %s', ceph_fsid
)
3611 cluster
= find_cluster_by_uuid(ceph_fsid
)
3613 raise Error('No cluster conf found in ' + SYSCONFDIR
+
3614 ' with fsid %s' % ceph_fsid
)
3615 LOG
.debug('Cluster name is %s', cluster
)
3617 fsid
= read_one_line(path
, 'fsid')
3619 raise Error('No OSD uuid assigned.')
3620 LOG
.debug('OSD uuid is %s', fsid
)
3622 keyring
= activate_key_template
.format(cluster
=cluster
,
3625 osd_id
= get_osd_id(path
)
3627 osd_id
= allocate_osd_id(
3632 write_one_line(path
, 'whoami', osd_id
)
3633 LOG
.debug('OSD id is %s', osd_id
)
3635 if not os
.path
.exists(os
.path
.join(path
, 'ready')):
3636 LOG
.debug('Initializing OSD...')
3637 # re-running mkfs is safe, so just run until it completes
3646 if init
not in (None, 'none'):
3648 conf_val
= get_conf(
3652 if conf_val
is not None:
3657 LOG
.debug('Marking with init system %s', init
)
3658 init_path
= os
.path
.join(path
, init
)
3659 with
open(init_path
, 'w'):
3660 path_set_context(init_path
)
3662 # remove markers for others, just in case.
3663 for other
in INIT_SYSTEMS
:
3666 os
.unlink(os
.path
.join(path
, other
))
3670 if not os
.path
.exists(os
.path
.join(path
, 'active')):
3671 LOG
.debug('Authorizing OSD key...')
3678 write_one_line(path
, 'active', 'ok')
3679 LOG
.debug('%s osd.%s data dir is ready at %s', cluster
, osd_id
, path
)
3680 return (osd_id
, cluster
)
3683 def main_activate(args
):
3687 LOG
.info('path = ' + str(args
.path
))
3688 if not os
.path
.exists(args
.path
):
3689 raise Error('%s does not exist' % args
.path
)
3691 if is_suppressed(args
.path
):
3692 LOG
.info('suppressed activate request on %s', args
.path
)
3696 mode
= os
.stat(args
.path
).st_mode
3697 if stat
.S_ISBLK(mode
):
3698 if (is_partition(args
.path
) and
3699 (get_partition_type(args
.path
) ==
3700 PTYPE
['mpath']['osd']['ready']) and
3701 not is_mpath(args
.path
)):
3702 raise Error('%s is not a multipath block device' %
3704 (cluster
, osd_id
) = mount_activate(
3706 activate_key_template
=args
.activate_key_template
,
3707 init
=args
.mark_init
,
3708 dmcrypt
=args
.dmcrypt
,
3709 dmcrypt_key_dir
=args
.dmcrypt_key_dir
,
3710 reactivate
=args
.reactivate
,
3712 osd_data
= get_mount_point(cluster
, osd_id
)
3714 elif stat
.S_ISDIR(mode
):
3715 (cluster
, osd_id
) = activate_dir(
3717 activate_key_template
=args
.activate_key_template
,
3718 init
=args
.mark_init
,
3720 osd_data
= args
.path
3723 raise Error('%s is not a directory or block device' % args
.path
)
3725 # exit with 0 if the journal device is not up, yet
3726 # journal device will do the activation
3727 osd_journal
= '{path}/journal'.format(path
=osd_data
)
3728 if os
.path
.islink(osd_journal
) and not os
.access(osd_journal
, os
.F_OK
):
3729 LOG
.info("activate: Journal not present, not starting, yet")
3732 if (not args
.no_start_daemon
and args
.mark_init
== 'none'):
3736 '--cluster={cluster}'.format(cluster
=cluster
),
3737 '--id={osd_id}'.format(osd_id
=osd_id
),
3738 '--osd-data={path}'.format(path
=osd_data
),
3739 '--osd-journal={journal}'.format(journal
=osd_journal
),
3743 if (not args
.no_start_daemon
and
3744 args
.mark_init
not in (None, 'none')):
3752 def main_activate_lockbox(args
):
3754 main_activate_lockbox_protected(args
)
3757 def main_activate_lockbox_protected(args
):
3758 partition
= DevicePartition
.factory(
3759 path
=None, dev
=args
.path
, args
=args
)
3761 lockbox
= Lockbox(args
)
3762 lockbox
.set_partition(partition
)
3766 ###########################
3768 def _mark_osd_out(cluster
, osd_id
):
3769 LOG
.info('Prepare to mark osd.%d out...', osd_id
)
3778 def _check_osd_status(cluster
, osd_id
):
3780 report the osd status:
3781 00(0) : means OSD OUT AND DOWN
3782 01(1) : means OSD OUT AND UP
3783 10(2) : means OSD IN AND DOWN
3784 11(3) : means OSD IN AND UP
3786 LOG
.info("Checking osd id: %s ..." % osd_id
)
3789 out
, err
, ret
= command([
3793 '--cluster={cluster}'.format(
3799 out_json
= json
.loads(out
)
3800 for item
in out_json
[u
'osds']:
3801 if item
.get(u
'osd') == int(osd_id
):
3803 if item
.get(u
'in') is 1:
3805 if item
.get(u
'up') is 1:
3808 raise Error('Could not osd.%s in osd tree!' % osd_id
)
3812 def _remove_osd_directory_files(mounted_path
, cluster
):
3814 To remove the 'ready', 'active', INIT-specific files.
3816 if os
.path
.exists(os
.path
.join(mounted_path
, 'ready')):
3817 os
.remove(os
.path
.join(mounted_path
, 'ready'))
3818 LOG
.info('Remove `ready` file.')
3820 LOG
.info('`ready` file is already removed.')
3822 if os
.path
.exists(os
.path
.join(mounted_path
, 'active')):
3823 os
.remove(os
.path
.join(mounted_path
, 'active'))
3824 LOG
.info('Remove `active` file.')
3826 LOG
.info('`active` file is already removed.')
3828 # Just check `upstart` and `sysvinit` directly if filename is init-spec.
3829 conf_val
= get_conf(
3833 if conf_val
is not None:
3837 os
.remove(os
.path
.join(mounted_path
, init
))
3838 LOG
.info('Remove `%s` file.', init
)
3842 def main_deactivate(args
):
3844 main_deactivate_locked(args
)
3847 def main_deactivate_locked(args
):
3848 osd_id
= args
.deactivate_by_id
3852 devices
= list_devices()
3854 # list all devices and found we need
3855 for device
in devices
:
3856 if 'partitions' in device
:
3857 for dev_part
in device
.get('partitions'):
3859 'whoami' in dev_part
and
3860 dev_part
['whoami'] == osd_id
):
3861 target_dev
= dev_part
3863 'path' in dev_part
and
3864 dev_part
['path'] == path
):
3865 target_dev
= dev_part
3867 raise Error('Cannot find any match device!!')
3869 # set up all we need variable
3870 osd_id
= target_dev
['whoami']
3871 part_type
= target_dev
['ptype']
3872 mounted_path
= target_dev
['mount']
3873 if Ptype
.is_dmcrypt(part_type
, 'osd'):
3876 # Do not do anything if osd is already down.
3877 status_code
= _check_osd_status(args
.cluster
, osd_id
)
3878 if status_code
== OSD_STATUS_IN_UP
:
3879 if args
.mark_out
is True:
3880 _mark_osd_out(args
.cluster
, int(osd_id
))
3881 stop_daemon(args
.cluster
, osd_id
)
3882 elif status_code
== OSD_STATUS_IN_DOWN
:
3883 if args
.mark_out
is True:
3884 _mark_osd_out(args
.cluster
, int(osd_id
))
3885 LOG
.info("OSD already out/down. Do not do anything now.")
3887 elif status_code
== OSD_STATUS_OUT_UP
:
3888 stop_daemon(args
.cluster
, osd_id
)
3889 elif status_code
== OSD_STATUS_OUT_DOWN
:
3890 LOG
.info("OSD already out/down. Do not do anything now.")
3894 # remove 'ready', 'active', and INIT-specific files.
3895 _remove_osd_directory_files(mounted_path
, args
.cluster
)
3897 # Write deactivate to osd directory!
3898 with
open(os
.path
.join(mounted_path
, 'deactive'), 'w'):
3899 path_set_context(os
.path
.join(mounted_path
, 'deactive'))
3901 unmount(mounted_path
)
3902 LOG
.info("Umount `%s` successfully.", mounted_path
)
3905 lockbox
= os
.path
.join(STATEDIR
, 'osd-lockbox')
3906 command(['umount', os
.path
.join(lockbox
, target_dev
['uuid'])])
3908 dmcrypt_unmap(target_dev
['uuid'])
3909 for name
in Space
.NAMES
:
3910 if name
+ '_uuid' in target_dev
:
3911 dmcrypt_unmap(target_dev
[name
+ '_uuid'])
3913 ###########################
3916 def _remove_from_crush_map(cluster
, osd_id
):
3917 LOG
.info("Prepare to remove osd.%s from crush map..." % osd_id
)
3927 def _delete_osd_auth_key(cluster
, osd_id
):
3928 LOG
.info("Prepare to delete osd.%s cephx key..." % osd_id
)
3937 def _deallocate_osd_id(cluster
, osd_id
):
3938 LOG
.info("Prepare to deallocate the osd-id: %s..." % osd_id
)
3947 def _remove_lockbox(uuid
, cluster
):
3950 '--cluster', cluster
,
3953 'client.osd-lockbox.' + uuid
,
3957 '--cluster', cluster
,
3960 'dm-crypt/osd/' + uuid
+ '/luks',
3962 lockbox
= os
.path
.join(STATEDIR
, 'osd-lockbox')
3963 if not os
.path
.exists(lockbox
):
3965 canonical
= os
.path
.join(lockbox
, uuid
)
3966 command(['umount', canonical
])
3967 for name
in os
.listdir(lockbox
):
3968 path
= os
.path
.join(lockbox
, name
)
3969 if os
.path
.islink(path
) and os
.readlink(path
) == canonical
:
3973 def destroy_lookup_device(args
, predicate
, description
):
3974 devices
= list_devices()
3975 for device
in devices
:
3976 for partition
in device
.get('partitions', []):
3977 if partition
['type'] == 'lockbox':
3978 if not is_mounted(partition
['path']):
3979 main_activate_lockbox_protected(
3980 argparse
.Namespace(verbose
=args
.verbose
,
3981 path
=partition
['path']))
3982 for device
in devices
:
3983 for partition
in device
.get('partitions', []):
3984 if partition
['dmcrypt']:
3985 dmcrypt_path
= dmcrypt_is_mapped(partition
['uuid'])
3989 dmcrypt_path
= dmcrypt_map(partition
['path'],
3990 args
.dmcrypt_key_dir
)
3992 list_dev_osd(dmcrypt_path
, {}, partition
)
3994 dmcrypt_unmap(partition
['uuid'])
3998 if predicate(partition
):
3999 return dmcrypt
, partition
4000 raise Error('found no device matching ', description
)
4003 def main_destroy(args
):
4005 main_destroy_locked(args
)
4008 def main_destroy_locked(args
):
4009 osd_id
= args
.destroy_by_id
4014 if not is_partition(path
):
4015 raise Error(path
+ " must be a partition device")
4016 path
= os
.path
.realpath(path
)
4019 (dmcrypt
, target_dev
) = destroy_lookup_device(
4020 args
, lambda x
: x
.get('path') == path
,
4023 (dmcrypt
, target_dev
) = destroy_lookup_device(
4024 args
, lambda x
: x
.get('whoami') == osd_id
,
4025 'osd id ' + str(osd_id
))
4027 osd_id
= target_dev
['whoami']
4028 dev_path
= target_dev
['path']
4029 if target_dev
['ptype'] == PTYPE
['mpath']['osd']['ready']:
4030 base_dev
= get_partition_base_mpath(dev_path
)
4032 base_dev
= get_partition_base(dev_path
)
4034 # Before osd deactivate, we cannot destroy it
4035 status_code
= _check_osd_status(args
.cluster
, osd_id
)
4036 if status_code
!= OSD_STATUS_OUT_DOWN
and \
4037 status_code
!= OSD_STATUS_IN_DOWN
:
4038 raise Error("Could not destroy the active osd. (osd-id: %s)" %
4041 # Remove OSD from crush map
4042 _remove_from_crush_map(args
.cluster
, osd_id
)
4044 # Remove OSD cephx key
4045 _delete_osd_auth_key(args
.cluster
, osd_id
)
4048 _deallocate_osd_id(args
.cluster
, osd_id
)
4050 # we remove the crypt map and device mapper (if dmcrypt is True)
4052 for name
in Space
.NAMES
:
4053 if target_dev
.get(name
+ '_uuid'):
4054 dmcrypt_unmap(target_dev
[name
+ '_uuid'])
4055 _remove_lockbox(target_dev
['uuid'], args
.cluster
)
4057 # Check zap flag. If we found zap flag, we need to find device for
4058 # destroy this osd data.
4059 if args
.zap
is True:
4060 # erase the osd data
4061 LOG
.info("Prepare to zap the device %s" % base_dev
)
4065 def get_space_osd_uuid(name
, path
):
4066 if not os
.path
.exists(path
):
4067 raise Error('%s does not exist' % path
)
4069 mode
= os
.stat(path
).st_mode
4070 if not stat
.S_ISBLK(mode
):
4071 raise Error('%s is not a block device' % path
)
4073 if (is_partition(path
) and
4074 get_partition_type(path
) in (PTYPE
['mpath']['journal']['ready'],
4075 PTYPE
['mpath']['block']['ready']) and
4076 not is_mpath(path
)):
4077 raise Error('%s is not a multipath block device' %
4081 out
= _check_output(
4084 '--get-device-fsid',
4089 except subprocess
.CalledProcessError
as e
:
4091 'failed to get osd uuid/fsid from %s' % name
,
4094 value
= str(out
).split('\n', 1)[0]
4095 LOG
.debug('%s %s has OSD UUID %s', name
.capitalize(), path
, value
)
4099 def main_activate_space(name
, args
):
4100 if not os
.path
.exists(args
.dev
):
4101 raise Error('%s does not exist' % args
.dev
)
4109 dev
= dmcrypt_map(args
.dev
, args
.dmcrypt_key_dir
)
4112 # FIXME: For an encrypted journal dev, does this return the
4113 # cyphertext or plaintext dev uuid!? Also, if the journal is
4114 # encrypted, is the data partition also always encrypted, or
4115 # are mixed pairs supported!?
4116 osd_uuid
= get_space_osd_uuid(name
, dev
)
4117 path
= os
.path
.join('/dev/disk/by-partuuid/', osd_uuid
.lower())
4119 if is_suppressed(path
):
4120 LOG
.info('suppressed activate request on %s', path
)
4123 # warn and exit with 0 if the data device is not up, yet
4124 # data device will do the activation
4125 if not os
.access(path
, os
.F_OK
):
4126 LOG
.info("activate: OSD device not present, not starting, yet")
4129 (cluster
, osd_id
) = mount_activate(
4131 activate_key_template
=args
.activate_key_template
,
4132 init
=args
.mark_init
,
4133 dmcrypt
=args
.dmcrypt
,
4134 dmcrypt_key_dir
=args
.dmcrypt_key_dir
,
4135 reactivate
=args
.reactivate
,
4144 ###########################
4147 def main_activate_all(args
):
4148 dir = '/dev/disk/by-parttypeuuid'
4149 LOG
.debug('Scanning %s', dir)
4150 if not os
.path
.exists(dir):
4153 for name
in os
.listdir(dir):
4154 if name
.find('.') < 0:
4156 (tag
, uuid
) = name
.split('.')
4158 if tag
in Ptype
.get_ready_by_name('osd'):
4160 if Ptype
.is_dmcrypt(tag
, 'osd'):
4161 path
= os
.path
.join('/dev/mapper', uuid
)
4163 path
= os
.path
.join(dir, name
)
4165 if is_suppressed(path
):
4166 LOG
.info('suppressed activate request on %s', path
)
4169 LOG
.info('Activating %s', path
)
4172 # never map dmcrypt cyphertext devices
4173 (cluster
, osd_id
) = mount_activate(
4175 activate_key_template
=args
.activate_key_template
,
4176 init
=args
.mark_init
,
4185 except Exception as e
:
4187 '{prog}: {msg}'.format(prog
=args
.prog
, msg
=e
),
4194 raise Error('One or more partitions failed to activate')
4197 ###########################
4200 dev
= os
.path
.realpath(dev
)
4201 with
open(PROCDIR
+ '/swaps', 'rb') as proc_swaps
:
4202 for line
in proc_swaps
.readlines()[1:]:
4203 fields
= line
.split()
4206 swaps_dev
= fields
[0]
4207 if os
.path
.isabs(swaps_dev
) and os
.path
.exists(swaps_dev
):
4208 swaps_dev
= os
.path
.realpath(swaps_dev
)
4209 if swaps_dev
== dev
:
4214 def get_oneliner(base
, name
):
4215 path
= os
.path
.join(base
, name
)
4216 if os
.path
.isfile(path
):
4217 with
open(path
, 'rb') as _file
:
4218 return _bytes2str(_file
.readline().rstrip())
4222 def get_dev_fs(dev
):
4224 fstype
, _
, ret
= command(
4234 fscheck
, _
, _
= command(
4242 if 'TYPE' in fscheck
:
4243 fstype
= fscheck
.split()[1].split('"')[1]
4248 def split_dev_base_partnum(dev
):
4250 partnum
= partnum_mpath(dev
)
4251 base
= get_partition_base_mpath(dev
)
4254 partnum
= open(os
.path
.join(b
, 'partition')).read().strip()
4255 base
= get_partition_base(dev
)
4256 return base
, partnum
4259 def get_partition_type(part
):
4260 return get_blkid_partition_info(part
, 'ID_PART_ENTRY_TYPE')
4263 def get_partition_uuid(part
):
4264 return get_blkid_partition_info(part
, 'ID_PART_ENTRY_UUID')
4267 def get_blkid_partition_info(dev
, what
=None):
4268 out
, _
, _
= command(
4278 for line
in out
.splitlines():
4279 (key
, value
) = line
.split('=')
4287 def more_osd_info(path
, uuid_map
, desc
):
4288 desc
['ceph_fsid'] = get_oneliner(path
, 'ceph_fsid')
4289 if desc
['ceph_fsid']:
4290 desc
['cluster'] = find_cluster_by_uuid(desc
['ceph_fsid'])
4291 desc
['whoami'] = get_oneliner(path
, 'whoami')
4292 for name
in Space
.NAMES
:
4293 uuid
= get_oneliner(path
, name
+ '_uuid')
4295 desc
[name
+ '_uuid'] = uuid
.lower()
4296 if desc
[name
+ '_uuid'] in uuid_map
:
4297 desc
[name
+ '_dev'] = uuid_map
[desc
[name
+ '_uuid']]
4300 def list_dev_osd(dev
, uuid_map
, desc
):
4301 desc
['mount'] = is_mounted(dev
)
4302 desc
['fs_type'] = get_dev_fs(dev
)
4303 desc
['state'] = 'unprepared'
4305 desc
['state'] = 'active'
4306 more_osd_info(desc
['mount'], uuid_map
, desc
)
4307 elif desc
['fs_type']:
4309 tpath
= mount(dev
=dev
, fstype
=desc
['fs_type'], options
='')
4312 magic
= get_oneliner(tpath
, 'magic')
4313 if magic
is not None:
4314 desc
['magic'] = magic
4315 desc
['state'] = 'prepared'
4316 more_osd_info(tpath
, uuid_map
, desc
)
4323 def list_dev_lockbox(dev
, uuid_map
, desc
):
4324 desc
['mount'] = is_mounted(dev
)
4325 desc
['fs_type'] = get_dev_fs(dev
)
4326 desc
['state'] = 'unprepared'
4328 desc
['state'] = 'active'
4329 desc
['osd_uuid'] = get_oneliner(desc
['mount'], 'osd-uuid')
4330 elif desc
['fs_type']:
4332 tpath
= tempfile
.mkdtemp(prefix
='mnt.', dir=STATEDIR
+ '/tmp')
4333 args
= ['mount', '-t', 'ext4', dev
, tpath
]
4334 LOG
.debug('Mounting lockbox ' + str(" ".join(args
)))
4335 command_check_call(args
)
4336 magic
= get_oneliner(tpath
, 'magic')
4337 if magic
is not None:
4338 desc
['magic'] = magic
4339 desc
['state'] = 'prepared'
4340 desc
['osd_uuid'] = get_oneliner(tpath
, 'osd-uuid')
4342 except subprocess
.CalledProcessError
:
4344 if desc
.get('osd_uuid') in uuid_map
:
4345 desc
['lockbox_for'] = uuid_map
[desc
['osd_uuid']]
4348 def list_format_lockbox_plain(dev
):
4350 if dev
.get('lockbox_for'):
4351 desc
.append('for ' + dev
['lockbox_for'])
4352 elif dev
.get('osd_uuid'):
4353 desc
.append('for osd ' + dev
['osd_uuid'])
4357 def list_format_more_osd_info_plain(dev
):
4359 if dev
.get('ceph_fsid'):
4360 if dev
.get('cluster'):
4361 desc
.append('cluster ' + dev
['cluster'])
4363 desc
.append('unknown cluster ' + dev
['ceph_fsid'])
4364 if dev
.get('whoami'):
4365 desc
.append('osd.%s' % dev
['whoami'])
4366 for name
in Space
.NAMES
:
4367 if dev
.get(name
+ '_dev'):
4368 desc
.append(name
+ ' %s' % dev
[name
+ '_dev'])
4372 def list_format_dev_plain(dev
, prefix
=''):
4374 if dev
['ptype'] == PTYPE
['regular']['osd']['ready']:
4375 desc
= (['ceph data', dev
['state']] +
4376 list_format_more_osd_info_plain(dev
))
4377 elif dev
['ptype'] in (PTYPE
['regular']['lockbox']['ready'],
4378 PTYPE
['mpath']['lockbox']['ready']):
4379 desc
= (['ceph lockbox', dev
['state']] +
4380 list_format_lockbox_plain(dev
))
4381 elif Ptype
.is_dmcrypt(dev
['ptype'], 'osd'):
4382 dmcrypt
= dev
['dmcrypt']
4383 if not dmcrypt
['holders']:
4384 desc
= ['ceph data (dmcrypt %s)' % dmcrypt
['type'],
4385 'not currently mapped']
4386 elif len(dmcrypt
['holders']) == 1:
4387 holder
= get_dev_path(dmcrypt
['holders'][0])
4388 desc
= ['ceph data (dmcrypt %s %s)' %
4389 (dmcrypt
['type'], holder
)]
4390 desc
+= list_format_more_osd_info_plain(dev
)
4392 desc
= ['ceph data (dmcrypt %s)' % dmcrypt
['type'],
4393 'holders: ' + ','.join(dmcrypt
['holders'])]
4394 elif Ptype
.is_regular_space(dev
['ptype']):
4395 name
= Ptype
.space_ptype_to_name(dev
['ptype'])
4396 desc
.append('ceph ' + name
)
4397 if dev
.get(name
+ '_for'):
4398 desc
.append('for %s' % dev
[name
+ '_for'])
4399 elif Ptype
.is_dmcrypt_space(dev
['ptype']):
4400 name
= Ptype
.space_ptype_to_name(dev
['ptype'])
4401 dmcrypt
= dev
['dmcrypt']
4402 if dmcrypt
['holders'] and len(dmcrypt
['holders']) == 1:
4403 holder
= get_dev_path(dmcrypt
['holders'][0])
4404 desc
= ['ceph ' + name
+ ' (dmcrypt %s %s)' %
4405 (dmcrypt
['type'], holder
)]
4407 desc
= ['ceph ' + name
+ ' (dmcrypt %s)' % dmcrypt
['type']]
4408 if dev
.get(name
+ '_for'):
4409 desc
.append('for %s' % dev
[name
+ '_for'])
4411 desc
.append(dev
['type'])
4412 if dev
.get('fs_type'):
4413 desc
.append(dev
['fs_type'])
4414 elif dev
.get('ptype'):
4415 desc
.append(dev
['ptype'])
4416 if dev
.get('mount'):
4417 desc
.append('mounted on %s' % dev
['mount'])
4418 return '%s%s %s' % (prefix
, dev
['path'], ', '.join(desc
))
4421 def list_format_plain(devices
):
4423 for device
in devices
:
4424 if device
.get('partitions'):
4425 lines
.append('%s :' % device
['path'])
4426 for p
in sorted(device
['partitions'], key
=lambda x
: x
['path']):
4427 lines
.append(list_format_dev_plain(dev
=p
,
4430 lines
.append(list_format_dev_plain(dev
=device
,
4432 return "\n".join(lines
)
4435 def list_dev(dev
, uuid_map
, space_map
):
4441 info
['is_partition'] = is_partition(dev
)
4442 if info
['is_partition']:
4443 ptype
= get_partition_type(dev
)
4444 info
['uuid'] = get_partition_uuid(dev
)
4447 info
['ptype'] = ptype
4448 LOG
.info("list_dev(dev = " + dev
+ ", ptype = " + str(ptype
) + ")")
4449 if ptype
in (PTYPE
['regular']['osd']['ready'],
4450 PTYPE
['mpath']['osd']['ready']):
4451 info
['type'] = 'data'
4452 if ptype
== PTYPE
['mpath']['osd']['ready']:
4453 info
['multipath'] = True
4454 list_dev_osd(dev
, uuid_map
, info
)
4455 elif ptype
in (PTYPE
['regular']['lockbox']['ready'],
4456 PTYPE
['mpath']['lockbox']['ready']):
4457 info
['type'] = 'lockbox'
4458 if ptype
== PTYPE
['mpath']['osd']['ready']:
4459 info
['multipath'] = True
4460 list_dev_lockbox(dev
, uuid_map
, info
)
4461 elif ptype
== PTYPE
['plain']['osd']['ready']:
4462 holders
= is_held(dev
)
4463 info
['type'] = 'data'
4464 info
['dmcrypt']['holders'] = holders
4465 info
['dmcrypt']['type'] = 'plain'
4466 if len(holders
) == 1:
4467 list_dev_osd(get_dev_path(holders
[0]), uuid_map
, info
)
4468 elif ptype
== PTYPE
['luks']['osd']['ready']:
4469 holders
= is_held(dev
)
4470 info
['type'] = 'data'
4471 info
['dmcrypt']['holders'] = holders
4472 info
['dmcrypt']['type'] = 'LUKS'
4473 if len(holders
) == 1:
4474 list_dev_osd(get_dev_path(holders
[0]), uuid_map
, info
)
4475 elif Ptype
.is_regular_space(ptype
) or Ptype
.is_mpath_space(ptype
):
4476 name
= Ptype
.space_ptype_to_name(ptype
)
4478 if ptype
== PTYPE
['mpath'][name
]['ready']:
4479 info
['multipath'] = True
4480 if info
.get('uuid') in space_map
:
4481 info
[name
+ '_for'] = space_map
[info
['uuid']]
4482 elif Ptype
.is_plain_space(ptype
):
4483 name
= Ptype
.space_ptype_to_name(ptype
)
4484 holders
= is_held(dev
)
4486 info
['dmcrypt']['type'] = 'plain'
4487 info
['dmcrypt']['holders'] = holders
4488 if info
.get('uuid') in space_map
:
4489 info
[name
+ '_for'] = space_map
[info
['uuid']]
4490 elif Ptype
.is_luks_space(ptype
):
4491 name
= Ptype
.space_ptype_to_name(ptype
)
4492 holders
= is_held(dev
)
4494 info
['dmcrypt']['type'] = 'LUKS'
4495 info
['dmcrypt']['holders'] = holders
4496 if info
.get('uuid') in space_map
:
4497 info
[name
+ '_for'] = space_map
[info
['uuid']]
4499 path
= is_mounted(dev
)
4500 fs_type
= get_dev_fs(dev
)
4502 info
['type'] = 'swap'
4504 info
['type'] = 'other'
4506 info
['fs_type'] = fs_type
4508 info
['mount'] = path
4514 partmap
= list_all_partitions()
4518 for base
, parts
in sorted(partmap
.items()):
4520 dev
= get_dev_path(p
)
4521 part_uuid
= get_partition_uuid(dev
)
4523 uuid_map
[part_uuid
] = dev
4524 ptype
= get_partition_type(dev
)
4525 LOG
.debug("main_list: " + dev
+
4526 " ptype = " + str(ptype
) +
4527 " uuid = " + str(part_uuid
))
4528 if ptype
in Ptype
.get_ready_by_name('osd'):
4529 if Ptype
.is_dmcrypt(ptype
, 'osd'):
4530 holders
= is_held(dev
)
4531 if len(holders
) != 1:
4533 dev_to_mount
= get_dev_path(holders
[0])
4537 fs_type
= get_dev_fs(dev_to_mount
)
4538 if fs_type
is not None:
4539 mount_options
= get_mount_options(cluster
='ceph',
4542 tpath
= mount(dev
=dev_to_mount
,
4543 fstype
=fs_type
, options
=mount_options
)
4545 for name
in Space
.NAMES
:
4546 space_uuid
= get_oneliner(tpath
,
4549 space_map
[space_uuid
.lower()] = dev
4555 LOG
.debug("main_list: " + str(partmap
) + ", uuid_map = " +
4556 str(uuid_map
) + ", space_map = " + str(space_map
))
4559 for base
, parts
in sorted(partmap
.items()):
4561 disk
= {'path': get_dev_path(base
)}
4563 for p
in sorted(parts
):
4564 partitions
.append(list_dev(get_dev_path(p
),
4567 disk
['partitions'] = partitions
4568 devices
.append(disk
)
4570 device
= list_dev(get_dev_path(base
), uuid_map
, space_map
)
4571 device
['path'] = get_dev_path(base
)
4572 devices
.append(device
)
4573 LOG
.debug("list_devices: " + str(devices
))
4579 out
, err
, ret
= command(
4583 '-o', 'name,mountpoint'
4586 except subprocess
.CalledProcessError
as e
:
4587 LOG
.info('zfs list -o name,mountpoint '
4588 'fails.\n (Error: %s)' % e
)
4590 lines
= out
.splitlines()
4591 for line
in lines
[1:]:
4592 vdevline
= line
.split()
4593 if os
.path
.exists(os
.path
.join(vdevline
[1], 'active')):
4594 elems
= os
.path
.split(vdevline
[1])
4595 print(vdevline
[0], "ceph data, active, cluster ceph,", elems
[1],
4596 "mounted on:", vdevline
[1])
4598 print(vdevline
[0] + " other, zfs, mounted on: " + vdevline
[1])
4601 def main_list(args
):
4604 main_list_freebsd(args
)
4606 main_list_protected(args
)
4609 def main_list_protected(args
):
4610 devices
= list_devices()
4613 for path
in args
.path
:
4614 if os
.path
.exists(path
):
4615 paths
.append(os
.path
.realpath(path
))
4618 selected_devices
= []
4619 for device
in devices
:
4621 if re
.search(path
+ '$', device
['path']):
4622 selected_devices
.append(device
)
4624 selected_devices
= devices
4625 if args
.format
== 'json':
4626 print(json
.dumps(selected_devices
))
4628 output
= list_format_plain(selected_devices
)
4633 def main_list_freebsd(args
):
4634 # Currently accomodate only ZFS Filestore partitions
4635 # return a list of VDEVs and mountpoints
4637 # NAME USED AVAIL REFER MOUNTPOINT
4638 # osd0 1.01G 1.32T 1.01G /var/lib/ceph/osd/osd.0
4639 # osd1 1.01G 1.32T 1.01G /var/lib/ceph/osd/osd.1
4643 ###########################
4645 # Mark devices that we want to suppress activates on with a
4648 # /var/lib/ceph/tmp/suppress-activate.sdb
4650 # where the last bit is the sanitized device name (/dev/X without the
4651 # /dev/ prefix) and the is_suppress() check matches a prefix. That
4652 # means suppressing sdb will stop activate on sdb1, sdb2, etc.
4655 def is_suppressed(path
):
4656 disk
= os
.path
.realpath(path
)
4658 if (not disk
.startswith('/dev/') or
4659 not stat
.S_ISBLK(os
.lstat(disk
).st_mode
)):
4661 base
= get_dev_name(disk
)
4663 if os
.path
.exists(SUPPRESS_PREFIX
+ base
): # noqa
4670 def set_suppress(path
):
4671 disk
= os
.path
.realpath(path
)
4672 if not os
.path
.exists(disk
):
4673 raise Error('does not exist', path
)
4674 if not stat
.S_ISBLK(os
.lstat(path
).st_mode
):
4675 raise Error('not a block device', path
)
4676 base
= get_dev_name(disk
)
4678 with
open(SUPPRESS_PREFIX
+ base
, 'w') as f
: # noqa
4680 LOG
.info('set suppress flag on %s', base
)
4683 def unset_suppress(path
):
4684 disk
= os
.path
.realpath(path
)
4685 if not os
.path
.exists(disk
):
4686 raise Error('does not exist', path
)
4687 if not stat
.S_ISBLK(os
.lstat(path
).st_mode
):
4688 raise Error('not a block device', path
)
4689 assert disk
.startswith('/dev/')
4690 base
= get_dev_name(disk
)
4692 fn
= SUPPRESS_PREFIX
+ base
# noqa
4693 if not os
.path
.exists(fn
):
4694 raise Error('not marked as suppressed', path
)
4698 LOG
.info('unset suppress flag on %s', base
)
4699 except OSError as e
:
4700 raise Error('failed to unsuppress', e
)
4703 def main_suppress(args
):
4704 set_suppress(args
.path
)
4707 def main_unsuppress(args
):
4708 unset_suppress(args
.path
)
4712 for dev
in args
.dev
:
4716 def main_trigger(args
):
4717 LOG
.debug("main_trigger: " + str(args
))
4718 if is_systemd() and not args
.sync
:
4719 # http://www.freedesktop.org/software/systemd/man/systemd-escape.html
4720 escaped_dev
= args
.dev
[1:].replace('-', '\\x2d')
4721 service
= 'ceph-disk@{dev}.service'.format(dev
=escaped_dev
)
4722 LOG
.info('systemd detected, triggering %s' % service
)
4732 if is_upstart() and not args
.sync
:
4733 LOG
.info('upstart detected, triggering ceph-disk task')
4739 'dev={dev}'.format(dev
=args
.dev
),
4740 'pid={pid}'.format(pid
=os
.getpid()),
4745 if get_ceph_user() == 'ceph':
4746 command_check_call(['chown', 'ceph:ceph', args
.dev
])
4747 parttype
= get_partition_type(args
.dev
)
4748 partid
= get_partition_uuid(args
.dev
)
4750 LOG
.info('trigger {dev} parttype {parttype} uuid {partid}'.format(
4756 ceph_disk
= ['ceph-disk']
4758 ceph_disk
.append('--verbose')
4760 if parttype
in (PTYPE
['regular']['osd']['ready'],
4761 PTYPE
['mpath']['osd']['ready']):
4762 out
, err
, ret
= command(
4770 elif parttype
in (PTYPE
['plain']['osd']['ready'],
4771 PTYPE
['luks']['osd']['ready']):
4772 out
, err
, ret
= command(
4781 elif parttype
in (PTYPE
['regular']['journal']['ready'],
4782 PTYPE
['mpath']['journal']['ready']):
4783 out
, err
, ret
= command(
4791 elif parttype
in (PTYPE
['plain']['journal']['ready'],
4792 PTYPE
['luks']['journal']['ready']):
4793 out
, err
, ret
= command(
4802 elif parttype
in (PTYPE
['regular']['block']['ready'],
4803 PTYPE
['regular']['block.db']['ready'],
4804 PTYPE
['regular']['block.wal']['ready'],
4805 PTYPE
['mpath']['block']['ready'],
4806 PTYPE
['mpath']['block.db']['ready'],
4807 PTYPE
['mpath']['block.wal']['ready']):
4808 out
, err
, ret
= command(
4816 elif parttype
in (PTYPE
['plain']['block']['ready'],
4817 PTYPE
['plain']['block.db']['ready'],
4818 PTYPE
['plain']['block.wal']['ready'],
4819 PTYPE
['luks']['block']['ready'],
4820 PTYPE
['luks']['block.db']['ready'],
4821 PTYPE
['luks']['block.wal']['ready']):
4822 out
, err
, ret
= command(
4831 elif parttype
in (PTYPE
['regular']['lockbox']['ready'],
4832 PTYPE
['mpath']['lockbox']['ready']):
4833 out
, err
, ret
= command(
4842 raise Error('unrecognized partition type %s' % parttype
)
4847 raise Error('return code ' + str(ret
))
4854 # A hash table containing 'path': ('uid', 'gid', blocking, recursive)
4856 ('/usr/bin/ceph-mon', 'root', 'root', True, False),
4857 ('/usr/bin/ceph-mds', 'root', 'root', True, False),
4858 ('/usr/bin/ceph-osd', 'root', 'root', True, False),
4859 ('/usr/bin/radosgw', 'root', 'root', True, False),
4860 ('/etc/ceph', 'root', 'root', True, True),
4861 ('/var/run/ceph', 'ceph', 'ceph', True, True),
4862 ('/var/log/ceph', 'ceph', 'ceph', True, True),
4863 ('/var/log/radosgw', 'ceph', 'ceph', True, True),
4864 ('/var/lib/ceph', 'ceph', 'ceph', True, False),
4867 # Relabel/chown all files under /var/lib/ceph/ recursively (except for osd)
4868 for directory
in glob
.glob('/var/lib/ceph/*'):
4869 if directory
== '/var/lib/ceph/osd':
4870 fix_table
.append((directory
, 'ceph', 'ceph', True, False))
4872 fix_table
.append((directory
, 'ceph', 'ceph', True, True))
4874 # Relabel/chown the osds recursively and in parallel
4875 for directory
in glob
.glob('/var/lib/ceph/osd/*'):
4876 fix_table
.append((directory
, 'ceph', 'ceph', False, True))
4878 LOG
.debug("fix_table: " + str(fix_table
))
4880 # The lists of background processes
4882 permissions_processes
= []
4883 selinux_processes
= []
4885 # Preliminary checks
4886 if args
.selinux
or args
.all
:
4887 out
, err
, ret
= command(['selinuxenabled'])
4889 LOG
.error('SELinux is not enabled, please enable it, first.')
4890 raise Error('no SELinux')
4892 for daemon
in ['ceph-mon', 'ceph-osd', 'ceph-mds', 'radosgw', 'ceph-mgr']:
4893 out
, err
, ret
= command(['pgrep', daemon
])
4895 LOG
.error(daemon
+ ' is running, please stop it, first')
4896 raise Error(daemon
+ ' running')
4898 # Relabel the basic system data without the ceph files
4899 if args
.system
or args
.all
:
4900 c
= ['restorecon', '-R', '/']
4901 for directory
, _
, _
, _
, _
in fix_table
:
4902 # Skip /var/lib/ceph subdirectories
4903 if directory
.startswith('/var/lib/ceph/'):
4908 out
, err
, ret
= command(c
)
4911 LOG
.error("Failed to restore labels of the underlying system")
4913 raise Error("basic restore failed")
4915 # Use find to relabel + chown ~simultaenously
4917 for directory
, uid
, gid
, blocking
, recursive
in fix_table
:
4918 # Skip directories/files that are not installed
4919 if not os
.access(directory
, os
.F_OK
):
4927 ':'.join((uid
, gid
)),
4936 # Just pass -maxdepth 0 for non-recursive calls
4938 c
+= ['-maxdepth', '0']
4941 out
, err
, ret
= command(c
)
4944 LOG
.error("Failed to fix " + directory
)
4946 raise Error(directory
+ " fix failed")
4948 all_processes
.append(command_init(c
))
4950 LOG
.debug("all_processes: " + str(all_processes
))
4951 for process
in all_processes
:
4952 out
, err
, ret
= command_wait(process
)
4954 LOG
.error("A background find process failed")
4956 raise Error("background failed")
4959 if args
.permissions
:
4960 for directory
, uid
, gid
, blocking
, recursive
in fix_table
:
4961 # Skip directories/files that are not installed
4962 if not os
.access(directory
, os
.F_OK
):
4969 ':'.join((uid
, gid
)),
4975 ':'.join((uid
, gid
)),
4980 out
, err
, ret
= command(c
)
4983 LOG
.error("Failed to chown " + directory
)
4985 raise Error(directory
+ " chown failed")
4987 permissions_processes
.append(command_init(c
))
4989 LOG
.debug("permissions_processes: " + str(permissions_processes
))
4990 for process
in permissions_processes
:
4991 out
, err
, ret
= command_wait(process
)
4993 LOG
.error("A background permissions process failed")
4995 raise Error("background failed")
4997 # Fix SELinux labels
4999 for directory
, uid
, gid
, blocking
, recursive
in fix_table
:
5000 # Skip directories/files that are not installed
5001 if not os
.access(directory
, os
.F_OK
):
5017 out
, err
, ret
= command(c
)
5020 LOG
.error("Failed to restore labels for " + directory
)
5022 raise Error(directory
+ " relabel failed")
5024 selinux_processes
.append(command_init(c
))
5026 LOG
.debug("selinux_processes: " + str(selinux_processes
))
5027 for process
in selinux_processes
:
5028 out
, err
, ret
= command_wait(process
)
5030 LOG
.error("A background selinux process failed")
5032 raise Error("background failed")
5035 "The ceph files has been fixed, please reboot "
5036 "the system for the changes to take effect."
5040 def setup_statedir(dir):
5041 # XXX The following use of globals makes linting
5042 # really hard. Global state in Python is iffy and
5043 # should be avoided.
5047 if not os
.path
.exists(STATEDIR
):
5049 if not os
.path
.exists(STATEDIR
+ "/tmp"):
5050 os
.mkdir(STATEDIR
+ "/tmp")
5053 prepare_lock
= FileLock(STATEDIR
+ '/tmp/ceph-disk.prepare.lock')
5055 global activate_lock
5056 activate_lock
= FileLock(STATEDIR
+ '/tmp/ceph-disk.activate.lock')
5058 global SUPPRESS_PREFIX
5059 SUPPRESS_PREFIX
= STATEDIR
+ '/tmp/suppress-activate.'
5062 def setup_sysconfdir(dir):
5067 def parse_args(argv
):
5068 parser
= argparse
.ArgumentParser(
5071 parser
.add_argument(
5073 action
='store_true', default
=None,
5074 help='be more verbose',
5076 parser
.add_argument(
5078 action
='store_true', default
=None,
5079 help='log to stdout',
5081 parser
.add_argument(
5082 '--prepend-to-path',
5085 help=('prepend PATH to $PATH for backward compatibility '
5086 '(default /usr/bin)'),
5088 parser
.add_argument(
5091 default
='/var/lib/ceph',
5092 help=('directory in which ceph state is preserved '
5093 '(default /var/lib/ceph)'),
5095 parser
.add_argument(
5098 default
='/etc/ceph',
5099 help=('directory in which ceph configuration files are found '
5100 '(default /etc/ceph)'),
5102 parser
.add_argument(
5106 help='use the given user for subprocesses, rather than ceph or root'
5108 parser
.add_argument(
5112 help='use the given group for subprocesses, rather than ceph or root'
5114 parser
.set_defaults(
5115 # we want to hold on to this, for later
5119 subparsers
= parser
.add_subparsers(
5120 title
='subcommands',
5121 description
='valid subcommands',
5122 help='sub-command help',
5125 Prepare
.set_subparser(subparsers
)
5126 make_activate_parser(subparsers
)
5127 make_activate_lockbox_parser(subparsers
)
5128 make_activate_block_parser(subparsers
)
5129 make_activate_journal_parser(subparsers
)
5130 make_activate_all_parser(subparsers
)
5131 make_list_parser(subparsers
)
5132 make_suppress_parser(subparsers
)
5133 make_deactivate_parser(subparsers
)
5134 make_destroy_parser(subparsers
)
5135 make_zap_parser(subparsers
)
5136 make_trigger_parser(subparsers
)
5137 make_fix_parser(subparsers
)
5139 args
= parser
.parse_args(argv
)
5143 def make_fix_parser(subparsers
):
5144 fix_parser
= subparsers
.add_parser(
5146 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5147 description
=textwrap
.fill(textwrap
.dedent("""\
5149 help='fix SELinux labels and/or file permissions')
5151 fix_parser
.add_argument(
5153 action
='store_true',
5155 help='fix SELinux labels for the non-ceph system data'
5157 fix_parser
.add_argument(
5159 action
='store_true',
5161 help='fix SELinux labels for ceph data'
5163 fix_parser
.add_argument(
5165 action
='store_true',
5167 help='fix file permissions for ceph data'
5169 fix_parser
.add_argument(
5171 action
='store_true',
5173 help='perform all the fix-related operations'
5175 fix_parser
.set_defaults(
5181 def make_trigger_parser(subparsers
):
5182 trigger_parser
= subparsers
.add_parser(
5184 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5185 description
=textwrap
.fill(textwrap
.dedent("""\
5186 The partition given in argument is activated. The type of the
5187 partition (data, lockbox, journal etc.) is detected by its
5188 type. If the init system is upstart or systemd, the activation is
5189 delegated to it and runs asynchronously, which
5190 helps reduce the execution time of udev actions.
5192 help='activate any device (called by udev)')
5193 trigger_parser
.add_argument(
5197 trigger_parser
.add_argument(
5201 help='cluster name to assign this disk to',
5203 trigger_parser
.add_argument(
5205 action
='store_true', default
=None,
5206 help='map devices with dm-crypt',
5208 trigger_parser
.add_argument(
5209 '--dmcrypt-key-dir',
5211 default
='/etc/ceph/dmcrypt-keys',
5212 help='directory where dm-crypt keys are stored',
5214 trigger_parser
.add_argument(
5216 action
='store_true', default
=None,
5217 help='do operation synchronously; do not trigger systemd',
5219 trigger_parser
.set_defaults(
5222 return trigger_parser
5225 def make_activate_parser(subparsers
):
5226 activate_parser
= subparsers
.add_parser(
5228 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5229 description
=textwrap
.fill(textwrap
.dedent("""\
5230 Activate the OSD found at PATH (can be a directory
5231 or a device partition, possibly encrypted). When
5232 activated for the first time, a unique OSD id is obtained
5233 from the cluster. If PATH is a directory, a symbolic
5234 link is added in {statedir}/osd/ceph-$id. If PATH is
5235 a partition, it is mounted on {statedir}/osd/ceph-$id.
5236 Finally, the OSD daemon is run.
5238 If the OSD depends on auxiliary partitions (journal, block, ...)
5239 they need to be available otherwise activation will fail. It
5240 may happen if a journal is encrypted and cryptsetup was not
5242 """.format(statedir
=STATEDIR
))),
5243 help='Activate a Ceph OSD')
5244 activate_parser
.add_argument(
5246 action
='store_true', default
=None,
5247 help='mount a block device [deprecated, ignored]',
5249 activate_parser
.add_argument(
5252 help='bootstrap-osd keyring path template (%(default)s)',
5253 dest
='activate_key_template',
5255 activate_parser
.add_argument(
5257 metavar
='INITSYSTEM',
5258 help='init system to manage this dir',
5260 choices
=INIT_SYSTEMS
,
5262 activate_parser
.add_argument(
5263 '--no-start-daemon',
5264 action
='store_true', default
=None,
5265 help='do not start the daemon',
5267 activate_parser
.add_argument(
5270 help='path to block device or directory',
5272 activate_parser
.add_argument(
5274 action
='store_true', default
=None,
5275 help='map DATA and/or JOURNAL devices with dm-crypt',
5277 activate_parser
.add_argument(
5278 '--dmcrypt-key-dir',
5280 default
='/etc/ceph/dmcrypt-keys',
5281 help='directory where dm-crypt keys are stored',
5283 activate_parser
.add_argument(
5285 action
='store_true', default
=False,
5286 help='activate the deactived OSD',
5288 activate_parser
.set_defaults(
5289 activate_key_template
='{statedir}/bootstrap-osd/{cluster}.keyring',
5292 return activate_parser
5295 def make_activate_lockbox_parser(subparsers
):
5296 parser
= subparsers
.add_parser(
5298 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5299 description
=textwrap
.fill(textwrap
.dedent("""\
5300 Mount the partition found at PATH on {statedir}/osd-lockbox/$uuid
5301 where $uuid uniquely identifies the OSD that needs this lockbox
5302 to retrieve keys from the monitor and unlock its partitions.
5304 If the OSD has one or more auxiliary devices (journal, block, ...)
5305 symbolic links are created at {statedir}/osd-lockbox/$other_uuid
5306 and point to {statedir}/osd-lockbox/$uuid. This will, for instance,
5307 allow a journal encrypted in a partition identified by $other_uuid to
5308 fetch the keys it needs from the monitor.
5310 Finally the OSD is activated, as it would be with ceph-disk activate.
5311 """.format(statedir
=STATEDIR
))),
5312 help='Activate a Ceph lockbox')
5313 parser
.add_argument(
5315 help='bootstrap-osd keyring path template (%(default)s)',
5316 dest
='activate_key_template',
5318 parser
.add_argument(
5319 '--dmcrypt-key-dir',
5321 default
='/etc/ceph/dmcrypt-keys',
5322 help='directory where dm-crypt keys are stored',
5324 parser
.add_argument(
5327 help='path to block device',
5329 parser
.set_defaults(
5330 activate_key_template
='{statedir}/bootstrap-osd/{cluster}.keyring',
5331 func
=main_activate_lockbox
,
5336 def make_activate_block_parser(subparsers
):
5337 return make_activate_space_parser('block', subparsers
)
5340 def make_activate_journal_parser(subparsers
):
5341 return make_activate_space_parser('journal', subparsers
)
5344 def make_activate_space_parser(name
, subparsers
):
5345 activate_space_parser
= subparsers
.add_parser(
5346 'activate-%s' % name
,
5347 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5348 description
=textwrap
.fill(textwrap
.dedent("""\
5349 Activating a {name} partition is only meaningfull
5350 if it is encrypted and it will map it using
5353 Finally the corresponding OSD is activated,
5354 as it would be with ceph-disk activate.
5355 """.format(name
=name
))),
5356 help='Activate an OSD via its %s device' % name
)
5357 activate_space_parser
.add_argument(
5360 help='path to %s block device' % name
,
5362 activate_space_parser
.add_argument(
5365 help='bootstrap-osd keyring path template (%(default)s)',
5366 dest
='activate_key_template',
5368 activate_space_parser
.add_argument(
5370 metavar
='INITSYSTEM',
5371 help='init system to manage this dir',
5373 choices
=INIT_SYSTEMS
,
5375 activate_space_parser
.add_argument(
5377 action
='store_true', default
=None,
5378 help=('map data and/or auxiliariy (journal, etc.) '
5379 'devices with dm-crypt'),
5381 activate_space_parser
.add_argument(
5382 '--dmcrypt-key-dir',
5384 default
='/etc/ceph/dmcrypt-keys',
5385 help='directory where dm-crypt keys are stored',
5387 activate_space_parser
.add_argument(
5389 action
='store_true', default
=False,
5390 help='activate the deactived OSD',
5392 activate_space_parser
.set_defaults(
5393 activate_key_template
='{statedir}/bootstrap-osd/{cluster}.keyring',
5394 func
=lambda args
: main_activate_space(name
, args
),
5396 return activate_space_parser
5399 def make_activate_all_parser(subparsers
):
5400 activate_all_parser
= subparsers
.add_parser(
5402 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5403 description
=textwrap
.fill(textwrap
.dedent("""\
5404 Activate all OSD partitions found in /dev/disk/by-parttypeuuid.
5405 The partitions containing auxiliary devices (journal, block, ...)
5408 help='Activate all tagged OSD partitions')
5409 activate_all_parser
.add_argument(
5412 help='bootstrap-osd keyring path template (%(default)s)',
5413 dest
='activate_key_template',
5415 activate_all_parser
.add_argument(
5417 metavar
='INITSYSTEM',
5418 help='init system to manage this dir',
5420 choices
=INIT_SYSTEMS
,
5422 activate_all_parser
.set_defaults(
5423 activate_key_template
='{statedir}/bootstrap-osd/{cluster}.keyring',
5424 func
=main_activate_all
,
5426 return activate_all_parser
5429 def make_list_parser(subparsers
):
5430 list_parser
= subparsers
.add_parser(
5432 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5433 description
=textwrap
.fill(textwrap
.dedent("""\
5434 Display all partitions on the system and their
5435 associated Ceph information, if any.
5437 help='List disks, partitions, and Ceph OSDs')
5438 list_parser
.add_argument(
5440 help='output format',
5442 choices
=['json', 'plain'],
5444 list_parser
.add_argument(
5448 help='path to block devices, relative to /sys/block',
5450 list_parser
.set_defaults(
5456 def make_suppress_parser(subparsers
):
5457 suppress_parser
= subparsers
.add_parser(
5458 'suppress-activate',
5459 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5460 description
=textwrap
.fill(textwrap
.dedent("""\
5461 Add a prefix to the list of suppressed device names
5462 so that they are ignored by all activate* subcommands.
5464 help='Suppress activate on a device (prefix)')
5465 suppress_parser
.add_argument(
5468 help='path to block device or directory',
5470 suppress_parser
.set_defaults(
5474 unsuppress_parser
= subparsers
.add_parser(
5475 'unsuppress-activate',
5476 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5477 description
=textwrap
.fill(textwrap
.dedent("""\
5478 Remove a prefix from the list of suppressed device names
5479 so that they are no longer ignored by all
5480 activate* subcommands.
5482 help='Stop suppressing activate on a device (prefix)')
5483 unsuppress_parser
.add_argument(
5486 help='path to block device or directory',
5488 unsuppress_parser
.set_defaults(
5489 func
=main_unsuppress
,
5491 return suppress_parser
5494 def make_deactivate_parser(subparsers
):
5495 deactivate_parser
= subparsers
.add_parser(
5497 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5498 description
=textwrap
.fill(textwrap
.dedent("""\
5499 Deactivate the OSD located at PATH. It stops the OSD daemon
5500 and optionally marks it out (with --mark-out). The content of
5501 the OSD is left untouched.
5503 By default, the, ready, active, INIT-specific files are
5504 removed (so that it is not automatically re-activated by the
5505 udev rules or ceph-disk trigger) and the file deactive is
5506 created to remember the OSD is deactivated.
5508 If the --once option is given, the ready, active, INIT-specific
5509 files are not removed and the OSD will reactivate whenever
5510 ceph-disk trigger is run on one of the devices (journal, data,
5511 block, lockbox, ...).
5513 If the OSD is dmcrypt, remove the data dmcrypt map. When
5514 deactivate finishes, the OSD is down.
5516 help='Deactivate a Ceph OSD')
5517 deactivate_parser
.add_argument(
5521 help='cluster name to assign this disk to',
5523 deactivate_parser
.add_argument(
5527 help='path to block device or directory',
5529 deactivate_parser
.add_argument(
5530 '--deactivate-by-id',
5532 help='ID of OSD to deactive'
5534 deactivate_parser
.add_argument(
5536 action
='store_true', default
=False,
5537 help='option to mark the osd out',
5539 deactivate_parser
.add_argument(
5541 action
='store_true', default
=False,
5542 help='does not need --reactivate to activate again',
5544 deactivate_parser
.set_defaults(
5545 func
=main_deactivate
,
5549 def make_destroy_parser(subparsers
):
5550 destroy_parser
= subparsers
.add_parser(
5552 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5553 description
=textwrap
.fill(textwrap
.dedent("""\
5554 Destroy the OSD located at PATH.
5555 It removes the OSD from the cluster, the crushmap and
5556 deallocates the OSD id. An OSD must be down before it
5559 help='Destroy a Ceph OSD')
5560 destroy_parser
.add_argument(
5564 help='cluster name to assign this disk to',
5566 destroy_parser
.add_argument(
5570 help='path to block device or directory',
5572 destroy_parser
.add_argument(
5575 help='ID of OSD to destroy'
5577 destroy_parser
.add_argument(
5578 '--dmcrypt-key-dir',
5580 default
='/etc/ceph/dmcrypt-keys',
5581 help=('directory where dm-crypt keys are stored '
5582 '(If you don\'t know how it work, '
5583 'dont use it. we have default value)'),
5585 destroy_parser
.add_argument(
5587 action
='store_true', default
=False,
5588 help='option to erase data and partition',
5590 destroy_parser
.set_defaults(
5595 def make_zap_parser(subparsers
):
5596 zap_parser
= subparsers
.add_parser(
5598 formatter_class
=argparse
.RawDescriptionHelpFormatter
,
5599 description
=textwrap
.fill(textwrap
.dedent("""\
5600 Zap/erase/destroy a device's partition table and contents. It
5601 actually uses sgdisk and it's option --zap-all to
5602 destroy both GPT and MBR data structures so that the disk
5603 becomes suitable for repartitioning.
5605 help='Zap/erase/destroy a device\'s partition table (and contents)')
5606 zap_parser
.add_argument(
5610 help='path to block device',
5612 zap_parser
.set_defaults(
5619 args
= parse_args(argv
)
5621 setup_logging(args
.verbose
, args
.log_stdout
)
5623 if args
.prepend_to_path
!= '':
5624 path
= os
.environ
.get('PATH', os
.defpath
)
5625 os
.environ
['PATH'] = args
.prepend_to_path
+ ":" + path
5627 if args
.func
.__name
__ != 'main_trigger':
5628 # trigger may run when statedir is unavailable and does not use it
5629 setup_statedir(args
.statedir
)
5630 setup_sysconfdir(args
.sysconfdir
)
5632 global CEPH_PREF_USER
5633 CEPH_PREF_USER
= args
.setuser
5634 global CEPH_PREF_GROUP
5635 CEPH_PREF_GROUP
= args
.setgroup
5640 main_catch(args
.func
, args
)
5643 def setup_logging(verbose
, log_stdout
):
5644 loglevel
= logging
.WARNING
5646 loglevel
= logging
.DEBUG
5649 ch
= logging
.StreamHandler(stream
=sys
.stdout
)
5650 ch
.setLevel(loglevel
)
5651 formatter
= logging
.Formatter('%(funcName)s: %(message)s')
5652 ch
.setFormatter(formatter
)
5654 LOG
.setLevel(loglevel
)
5656 logging
.basicConfig(
5658 format
='%(funcName)s: %(message)s',
5662 def main_catch(func
, args
):
5669 '{prog}: {msg}'.format(
5675 except CephDiskException
as error
:
5676 exc_name
= error
.__class
__.__name
__
5678 '{prog} {exc_name}: {msg}'.format(
5690 if __name__
== '__main__':