#!/usr/bin/env python
#
-# Copyright (C) 2015, 2016 Red Hat <contact@redhat.com>
+# Copyright (C) 2015, 2016, 2017 Red Hat <contact@redhat.com>
# Copyright (C) 2014 Inktank <info@inktank.com>
# Copyright (C) 2014 Cloudwatt <libre.licensing@cloudwatt.com>
# Copyright (C) 2014 Catalyst.net Ltd
import base64
import errno
import fcntl
+import functools
import json
import logging
import os
import uuid
import time
import shlex
+import shutil
import pwd
import grp
import textwrap
},
}
+try:
+ # see https://bugs.python.org/issue23098
+ os.major(0x80002b00)
+except OverflowError:
+ os.major = lambda devid: ((devid >> 8) & 0xfff) | ((devid >> 32) & ~0xfff)
+ os.minor = lambda devid: (devid & 0xff) | ((devid >> 12) & ~0xff)
+
class Ptype(object):
PROCDIR = '/compat/linux/proc'
# FreeBSD does not have blockdevices any more
BLOCKDIR = '/dev'
+ ROOTGROUP = 'wheel'
else:
FREEBSD = False
DEFAULT_FS_TYPE = 'xfs'
PROCDIR = '/proc'
BLOCKDIR = '/sys/block'
+ ROOTGROUP = 'root'
"""
OSD STATUS Definition
return _bytes2str(out), _bytes2str(err), process.returncode
+def command_with_stdin(arguments, stdin):
+ LOG.info("Running command with stdin: " + " ".join(arguments))
+ process = subprocess.Popen(
+ arguments,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ out, err = process.communicate(stdin)
+ LOG.debug(out)
+ if process.returncode != 0:
+ LOG.error(err)
+ raise SystemExit(
+ "'{cmd}' failed with status code {returncode}".format(
+ cmd=arguments,
+ returncode=process.returncode,
+ )
+ )
+ return out
+
+
def _bytes2str(string):
return string.decode('utf-8') if isinstance(string, bytes) else string
os.close(fd)
+def stmode_is_diskdevice(dmode):
+ if stat.S_ISBLK(dmode):
+ return True
+ else:
+ # FreeBSD does not have block devices
+ # All disks are character devices
+ return FREEBSD and stat.S_ISCHR(dmode)
+
+
+def dev_is_diskdevice(dev):
+ dmode = os.stat(dev).st_mode
+ return stmode_is_diskdevice(dmode)
+
+
+def ldev_is_diskdevice(dev):
+ dmode = os.lstat(dev).st_mode
+ return stmode_is_diskdevice(dmode)
+
+
+def path_is_diskdevice(path):
+ dev = os.path.realpath(path)
+ return dev_is_diskdevice(dev)
+
+
def get_partition_mpath(dev, pnum):
part_re = "part{pnum}-mpath-".format(pnum=pnum)
partitions = list_partitions_mpath(dev, part_re)
return None
+def retry(on_error=Exception, max_tries=10, wait=0.2, backoff=0):
+ def wrapper(func):
+ @functools.wraps(func)
+ def repeat(*args, **kwargs):
+ for tries in range(max_tries - 1):
+ try:
+ return func(*args, **kwargs)
+ except on_error:
+ time.sleep(wait + backoff * tries)
+ return func(*args, **kwargs)
+ return repeat
+ return wrapper
+
+
+@retry(Error)
def get_partition_dev(dev, pnum):
"""
get the device name for a partition
sda 1 -> sda1
cciss/c0d1 1 -> cciss!c0d1p1
"""
- max_retry = 10
- for retry in range(0, max_retry + 1):
- partname = None
- error_msg = ""
- if is_mpath(dev):
- partname = get_partition_mpath(dev, pnum)
- else:
- name = get_dev_name(os.path.realpath(dev))
- sys_entry = os.path.join(BLOCKDIR, name)
- error_msg = " in %s" % sys_entry
- for f in os.listdir(sys_entry):
- if f.startswith(name) and f.endswith(str(pnum)):
- # we want the shortest name that starts with the base name
- # and ends with the partition number
- if not partname or len(f) < len(partname):
- partname = f
- if partname:
- if retry:
- LOG.info('Found partition %d for %s after %d tries' %
- (pnum, dev, retry))
- return get_dev_path(partname)
- else:
- if retry < max_retry:
- LOG.info('Try %d/%d : partition %d for %s does not exist%s' %
- (retry + 1, max_retry, pnum, dev, error_msg))
- time.sleep(.2)
- continue
- else:
- raise Error('partition %d for %s does not appear to exist%s' %
- (pnum, dev, error_msg))
+ partname = None
+ error_msg = ""
+ if is_mpath(dev):
+ partname = get_partition_mpath(dev, pnum)
+ else:
+ name = get_dev_name(os.path.realpath(dev))
+ sys_entry = os.path.join(BLOCKDIR, name)
+ error_msg = " in %s" % sys_entry
+ for f in os.listdir(sys_entry):
+ if f.startswith(name) and f.endswith(str(pnum)):
+ # we want the shortest name that starts with the base name
+ # and ends with the partition number
+ if not partname or len(f) < len(partname):
+ partname = f
+ if partname:
+ return get_dev_path(partname)
+ else:
+ raise Error('partition %d for %s does not appear to exist%s' %
+ (pnum, dev, error_msg))
def list_all_partitions():
Get the base device for a partition
"""
dev = os.path.realpath(dev)
- if not stat.S_ISBLK(os.lstat(dev).st_mode):
+ if not ldev_is_diskdevice(dev):
raise Error('not a block device', dev)
name = get_dev_name(dev)
dev = os.path.realpath(dev)
st = os.lstat(dev)
- if not stat.S_ISBLK(st.st_mode):
+ if not stmode_is_diskdevice(st.st_mode):
raise Error('not a block device', dev)
name = get_dev_name(dev)
cluster,
fsid,
keyring,
+ path,
):
"""
- Accocates an OSD id on the given cluster.
+ Allocates an OSD id on the given cluster.
:raises: Error if the call to allocate the OSD id fails.
:return: The allocated OSD id.
"""
+ lockbox_path = os.path.join(STATEDIR, 'osd-lockbox', fsid)
+ lockbox_osd_id = read_one_line(lockbox_path, 'whoami')
+ osd_keyring = os.path.join(path, 'keyring')
+ if lockbox_osd_id:
+ LOG.debug('Getting OSD id from Lockbox...')
+ osd_id = lockbox_osd_id
+ shutil.move(os.path.join(lockbox_path, 'osd_keyring'),
+ osd_keyring)
+ path_set_context(osd_keyring)
+ os.unlink(os.path.join(lockbox_path, 'whoami'))
+ return osd_id
LOG.debug('Allocating OSD id...')
+ secrets = Secrets()
try:
- osd_id = _check_output(
- args=[
+ wanttobe = read_one_line(path, 'wanttobe')
+ if os.path.exists(os.path.join(path, 'wanttobe')):
+ os.unlink(os.path.join(path, 'wanttobe'))
+ id_arg = wanttobe and [wanttobe] or []
+ osd_id = command_with_stdin(
+ [
'ceph',
'--cluster', cluster,
'--name', 'client.bootstrap-osd',
'--keyring', keyring,
- 'osd', 'create', '--concise',
+ '-i', '-',
+ 'osd', 'new',
fsid,
- ],
+ ] + id_arg,
+ secrets.get_json()
)
except subprocess.CalledProcessError as e:
raise Error('ceph osd create failed', e, e.output)
osd_id = must_be_one_line(osd_id)
check_osd_id(osd_id)
+ secrets.write_osd_keyring(osd_keyring, osd_id)
return osd_id
:return: The fsid or raises Error.
"""
fsid = get_conf_with_default(cluster=cluster, variable='fsid')
- if fsid is None:
+ # uuids from boost always default to 'the empty uuid'
+ if fsid == '00000000-0000-0000-0000-000000000000':
raise Error('getting cluster uuid from configuration failed')
return fsid.lower()
osd_uuid = get_oneliner(path, 'osd-uuid')
ceph_fsid = read_one_line(path, 'ceph_fsid')
if ceph_fsid is None:
- raise Error('No cluster uuid assigned.')
- cluster = find_cluster_by_uuid(ceph_fsid)
- if cluster is None:
- raise Error('No cluster conf found in ' + SYSCONFDIR +
- ' with fsid %s' % ceph_fsid)
+ LOG.warning("no `ceph_fsid` found falling back to 'ceph' "
+ "for cluster name")
+ cluster = 'ceph'
+ else:
+ cluster = find_cluster_by_uuid(ceph_fsid)
+ if cluster is None:
+ raise Error('No cluster conf found in ' + SYSCONFDIR +
+ ' with fsid %s' % ceph_fsid)
if mode == KEY_MANAGEMENT_MODE_V1:
key, stderr, ret = command(
rawdev,
] + cryptsetup_parameters
- def run(args, stdin):
- LOG.info(" ".join(args))
- process = subprocess.Popen(
- args,
- stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- out, err = process.communicate(stdin)
- LOG.debug(out)
- LOG.error(err)
- assert process.returncode == 0
-
try:
if luks:
if format_dev:
- run(luksFormat_args, key)
- run(luksOpen_args, key)
+ command_with_stdin(luksFormat_args, key)
+ command_with_stdin(luksOpen_args, key)
else:
# Plain mode has no format function, nor any validation
# that the key is correct.
- run(create_args, key)
+ command_with_stdin(create_args, key)
# set proper ownership of mapped device
command_check_call(['chown', 'ceph:ceph', dev])
return dev
raise Error('unable to map device', rawdev, e)
-def dmcrypt_unmap(
- _uuid
-):
+@retry(Error, max_tries=10, wait=0.5, backoff=1.0)
+def dmcrypt_unmap(_uuid):
if not os.path.exists('/dev/mapper/' + _uuid):
return
- retries = 0
- while True:
- try:
- command_check_call(['cryptsetup', 'remove', _uuid])
- break
- except subprocess.CalledProcessError as e:
- if retries == 10:
- raise Error('unable to unmap device', _uuid, e)
- else:
- time.sleep(0.5 + retries * 1.0)
- retries += 1
+ try:
+ command_check_call(['cryptsetup', 'remove', _uuid])
+ except subprocess.CalledProcessError as e:
+ raise Error('unable to unmap device', _uuid, e)
def mount(
return path
+@retry(UnmountError, max_tries=3, wait=0.5, backoff=1.0)
def unmount(
path,
+ do_rm=True,
):
"""
Unmount and removes the given mount point.
"""
- retries = 0
- while True:
- try:
- LOG.debug('Unmounting %s', path)
- command_check_call(
- [
- '/bin/umount',
- '--',
- path,
- ],
- )
- break
- except subprocess.CalledProcessError as e:
- # on failure, retry 3 times with incremental backoff
- if retries == 3:
- raise UnmountError(e)
- else:
- time.sleep(0.5 + retries * 1.0)
- retries += 1
-
+ try:
+ LOG.debug('Unmounting %s', path)
+ command_check_call(
+ [
+ '/bin/umount',
+ '--',
+ path,
+ ],
+ )
+ except subprocess.CalledProcessError as e:
+ raise UnmountError(e)
+ if not do_rm:
+ return
os.rmdir(path)
command_check_call(['udevadm', 'settle', '--timeout=600'])
-def zap(dev):
- """
- Destroy the partition table and content of a given disk.
- """
- dev = os.path.realpath(dev)
- dmode = os.stat(dev).st_mode
- if not stat.S_ISBLK(dmode) or is_partition(dev):
- raise Error('not full block device; cannot zap', dev)
+def zap_linux(dev):
try:
# Thoroughly wipe all partitions of any traces of
# Filesystems or OSD Journals
#
- # In addition we need to write 10M of data to each partition
- # to make sure that after re-creating the same partition
+ # In addition we need to write 110M (read following comment for more
+ # details on the context of this magic number) of data to each
+ # partition to make sure that after re-creating the same partition
# there is no trace left of any previous Filesystem or OSD
# Journal
],
)
+ # for an typical bluestore device, it has
+ # 1. a 100M xfs data partition
+ # 2. a bluestore_block_size block partition
+ # 3. a bluestore_block_db_size block.db partition
+ # 4. a bluestore_block_wal_size block.wal partition
+ # so we need to wipe out the bits storing the bits storing
+ # bluestore's collections' meta information in that case to
+ # prevent OSD from comparing the meta data, like OSD id and fsid,
+ # stored on the device to be zapped with the oness passed in. here,
+ # we assume that the allocator of bluestore puts these meta data
+ # at the beginning of the block partition. without knowning the
+ # actual layout of the bluefs, we add extra 10M to be on the safe
+ # side. if this partition was formatted for a filesystem, 10MB
+ # would be more than enough to nuke its superblock.
+ count = min(PrepareBluestoreData.SPACE_SIZE + 10,
+ get_dev_size(partition))
command_check_call(
[
'dd',
'if=/dev/zero',
'of={path}'.format(path=partition),
'bs=1M',
- 'count=10',
+ 'count={count}'.format(count=count),
],
)
dev,
],
)
-
update_partition(dev, 'zapped')
except subprocess.CalledProcessError as e:
raise Error(e)
+def zap_freebsd(dev):
+ try:
+ # For FreeBSD we just need to zap the partition.
+ command_check_call(
+ [
+ 'gpart',
+ 'destroy',
+ '-F',
+ dev,
+ ],
+ )
+
+ except subprocess.CalledProcessError as e:
+ raise Error(e)
+
+
+def zap(dev):
+ """
+ Destroy the partition table and content of a given disk.
+ """
+ dev = os.path.realpath(dev)
+ dmode = os.stat(dev).st_mode
+ if not stat.S_ISBLK(dmode) or is_partition(dev):
+ raise Error('not full block device; cannot zap', dev)
+ if FREEBSD:
+ zap_freebsd(dev)
+ else:
+ zap_linux(dev)
+
+
def adjust_symlink(target, path):
create = True
if os.path.lexists(path):
return self.ptype_map[name]['ready']
@staticmethod
+ @retry(OSError)
def factory(path, dev, args):
dmcrypt_type = CryptHelpers.get_dmcrypt_type(args)
if ((path is not None and is_mpath(path)) or
metavar='UUID',
help='unique OSD uuid to assign this disk to',
)
+ parser.add_argument(
+ '--osd-id',
+ metavar='ID',
+ help='unique OSD id to assign this disk to',
+ )
parser.add_argument(
'--crush-device-class',
help='crush device class to assign this disk to',
def set_type(self):
name = self.name
args = self.args
- dmode = os.stat(args.data).st_mode
if (self.wants_space() and
- stat.S_ISBLK(dmode) and
+ dev_is_diskdevice(args.data) and
not is_partition(args.data) and
getattr(args, name) is None and
getattr(args, name + '_file') is None):
return
mode = os.stat(getattr(args, name)).st_mode
- if stat.S_ISBLK(mode):
+ if stmode_is_diskdevice(mode):
if getattr(args, name + '_file'):
raise Error('%s is not a regular file' % name.capitalize,
getattr(args, name))
return None
+class Secrets(object):
+
+ def __init__(self):
+ secret, stderr, ret = command(['ceph-authtool', '--gen-print-key'])
+ LOG.debug("stderr " + stderr)
+ assert ret == 0
+ self.keys = {
+ 'cephx_secret': secret.strip(),
+ }
+
+ def write_osd_keyring(self, keyring, osd_id):
+ command_check_call(
+ [
+ 'ceph-authtool', keyring,
+ '--create-keyring',
+ '--name', 'osd.' + str(osd_id),
+ '--add-key', self.keys['cephx_secret'],
+ ])
+ path_set_context(keyring)
+
+ def get_json(self):
+ return bytearray(json.dumps(self.keys), 'ascii')
+
+
+class LockboxSecrets(Secrets):
+
+ def __init__(self, args):
+ super(LockboxSecrets, self).__init__()
+
+ key_size = CryptHelpers.get_dmcrypt_keysize(args)
+ key = open('/dev/urandom', 'rb').read(key_size / 8)
+ base64_key = base64.b64encode(key).decode('ascii')
+
+ secret, stderr, ret = command(['ceph-authtool', '--gen-print-key'])
+ LOG.debug("stderr " + stderr)
+ assert ret == 0
+
+ self.keys.update({
+ 'dmcrypt_key': base64_key,
+ 'cephx_lockbox_secret': secret.strip(),
+ })
+
+ def write_lockbox_keyring(self, path, osd_uuid):
+ keyring = os.path.join(path, 'keyring')
+ command_check_call(
+ [
+ 'ceph-authtool', keyring,
+ '--create-keyring',
+ '--name', 'client.osd-lockbox.' + osd_uuid,
+ '--add-key', self.keys['cephx_lockbox_secret'],
+ ])
+ path_set_context(keyring)
+
+
class Lockbox(object):
def __init__(self, args):
def create_partition(self):
self.device = Device.factory(self.args.lockbox, argparse.Namespace())
- partition_number = 3
+ partition_number = 5
self.device.create_partition(uuid=self.args.lockbox_uuid,
name='lockbox',
num=partition_number,
self.partition = self.create_partition()
def create_key(self):
- key_size = CryptHelpers.get_dmcrypt_keysize(self.args)
- key = open('/dev/urandom', 'rb').read(key_size / 8)
- base64_key = base64.b64encode(key)
cluster = self.args.cluster
bootstrap = self.args.prepare_key_template.format(cluster=cluster,
statedir=STATEDIR)
- command_check_call(
- [
- 'ceph',
- '--cluster', cluster,
- '--name', 'client.bootstrap-osd',
- '--keyring', bootstrap,
- 'config-key',
- 'put',
- 'dm-crypt/osd/' + self.args.osd_uuid + '/luks',
- base64_key,
- ],
- )
- keyring, stderr, ret = command(
+ path = self.get_mount_point()
+ secrets = LockboxSecrets(self.args)
+ id_arg = self.args.osd_id and [self.args.osd_id] or []
+ osd_id = command_with_stdin(
[
'ceph',
'--cluster', cluster,
'--name', 'client.bootstrap-osd',
'--keyring', bootstrap,
- 'auth',
- 'get-or-create',
- 'client.osd-lockbox.' + self.args.osd_uuid,
- 'mon',
- ('allow command "config-key get" with key="dm-crypt/osd/' +
- self.args.osd_uuid + '/luks"'),
- ],
+ '-i', '-',
+ 'osd', 'new', self.args.osd_uuid,
+ ] + id_arg,
+ secrets.get_json()
)
- LOG.debug("stderr " + stderr)
- assert ret == 0
- path = self.get_mount_point()
- open(os.path.join(path, 'keyring'), 'w').write(keyring)
+ secrets.write_lockbox_keyring(path, self.args.osd_uuid)
+ osd_id = must_be_one_line(osd_id)
+ check_osd_id(osd_id)
+ write_one_line(path, 'whoami', osd_id)
+ secrets.write_osd_keyring(os.path.join(path, 'osd_keyring'), osd_id)
write_one_line(path, 'key-management-mode', KEY_MANAGEMENT_MODE_V1)
def symlink_spaces(self, path):
if stat.S_ISDIR(dmode):
self.type = self.FILE
- elif stat.S_ISBLK(dmode):
+ elif stmode_is_diskdevice(dmode):
self.type = self.DEVICE
else:
raise Error('not a dir or block device', self.args.data)
write_one_line(path, 'ceph_fsid', self.args.cluster_uuid)
write_one_line(path, 'fsid', self.args.osd_uuid)
+ if self.args.osd_id:
+ write_one_line(path, 'wanttobe', self.args.osd_id)
if self.args.crush_device_class:
write_one_line(path, 'crush_device_class',
self.args.crush_device_class)
class PrepareBluestoreData(PrepareData):
+ SPACE_SIZE = 100
def get_space_size(self):
- return 100 # MB
+ return self.SPACE_SIZE # MB
def prepare_device(self, *to_prepare_list):
super(PrepareBluestoreData, self).prepare_device(*to_prepare_list)
write_one_line(path, 'type', 'bluestore')
-#
-# Temporary workaround: if ceph-osd --mkfs does not
-# complete within 5 minutes, assume it is blocked
-# because of http://tracker.ceph.com/issues/13522
-# and retry a few times.
-#
-# Remove this function calls with command_check_call
-# when http://tracker.ceph.com/issues/13522 is fixed
-#
-def ceph_osd_mkfs(arguments):
- timeout = _get_command_executable(['timeout'])
- mkfs_ok = False
- error = 'unknown error'
- for delay in os.environ.get('CEPH_OSD_MKFS_DELAYS',
- '300 300 300 300 300').split():
- try:
- _check_output(timeout + [delay] + arguments)
- mkfs_ok = True
- break
- except subprocess.CalledProcessError as e:
- error = e.output
- if e.returncode == 124: # timeout fired, retry
- LOG.debug('%s timed out : %s (retry)'
- % (str(arguments), error))
- else:
- break
- if not mkfs_ok:
- raise Error('%s failed : %s' % (str(arguments), error))
-
-
def mkfs(
path,
cluster,
osd_type = read_one_line(path, 'type')
if osd_type == 'bluestore':
- ceph_osd_mkfs(
+ command_check_call(
[
'ceph-osd',
'--cluster', cluster,
'--mkfs',
- '--mkkey',
'-i', osd_id,
'--monmap', monmap,
'--osd-data', path,
'--osd-uuid', fsid,
- '--keyring', os.path.join(path, 'keyring'),
'--setuser', get_ceph_user(),
'--setgroup', get_ceph_group(),
],
)
elif osd_type == 'filestore':
- ceph_osd_mkfs(
+ command_check_call(
[
'ceph-osd',
'--cluster', cluster,
'--mkfs',
- '--mkkey',
'-i', osd_id,
'--monmap', monmap,
'--osd-data', path,
'--osd-journal', os.path.join(path, 'journal'),
'--osd-uuid', fsid,
- '--keyring', os.path.join(path, 'keyring'),
'--setuser', get_ceph_user(),
'--setgroup', get_ceph_group(),
],
raise Error('unrecognized objectstore type %s' % osd_type)
-def auth_key(
- path,
- cluster,
- osd_id,
- keyring,
-):
- try:
- # try dumpling+ cap scheme
- command_check_call(
- [
- 'ceph',
- '--cluster', cluster,
- '--name', 'client.bootstrap-osd',
- '--keyring', keyring,
- 'auth', 'add', 'osd.{osd_id}'.format(osd_id=osd_id),
- '-i', os.path.join(path, 'keyring'),
- 'osd', 'allow *',
- 'mon', 'allow profile osd',
- ],
- )
- except subprocess.CalledProcessError as err:
- if err.returncode == errno.EINVAL:
- # try old cap scheme
- command_check_call(
- [
- 'ceph',
- '--cluster', cluster,
- '--name', 'client.bootstrap-osd',
- '--keyring', keyring,
- 'auth', 'add', 'osd.{osd_id}'.format(osd_id=osd_id),
- '-i', os.path.join(path, 'keyring'),
- 'osd', 'allow *',
- 'mon', 'allow rwx',
- ],
- )
- else:
- raise
-
-
def get_mount_point(cluster, osd_id):
parent = STATEDIR + '/osd'
return os.path.join(
osd_id,
):
systemd_disable(path, osd_id)
- if is_mounted(path):
+ if os.path.ismount(path):
style = ['--runtime']
else:
style = []
cluster=cluster,
fsid=fsid,
keyring=keyring,
+ path=path,
)
write_one_line(path, 'whoami', osd_id)
LOG.debug('OSD id is %s', osd_id)
pass
if not os.path.exists(os.path.join(path, 'active')):
- LOG.debug('Authorizing OSD key...')
- auth_key(
- path=path,
- cluster=cluster,
- osd_id=osd_id,
- keyring=keyring,
- )
write_one_line(path, 'active', 'ok')
LOG.debug('%s osd.%s data dir is ready at %s', cluster, osd_id, path)
return (osd_id, cluster)
with activate_lock:
mode = os.stat(args.path).st_mode
- if stat.S_ISBLK(mode):
+ if stmode_is_diskdevice(mode):
if (is_partition(args.path) and
(get_partition_type(args.path) ==
PTYPE['mpath']['osd']['ready']) and
)
osd_data = get_mount_point(cluster, osd_id)
+ args.cluster = cluster
+ if args.dmcrypt:
+ for name in Space.NAMES:
+ # Check if encrypted device in journal
+ dev_path = os.path.join(osd_data, name + '_dmcrypt')
+ if not os.path.exists(dev_path):
+ continue
+ partition = DevicePartition.factory(
+ path=None,
+ dev=dev_path,
+ args=args)
+ partition.rawdev = args.path
+ partition.map()
+
elif stat.S_ISDIR(mode):
(cluster, osd_id) = activate_dir(
path=args.path,
with open(os.path.join(mounted_path, 'deactive'), 'w'):
path_set_context(os.path.join(mounted_path, 'deactive'))
- unmount(mounted_path)
+ unmount(mounted_path, do_rm=not args.once)
LOG.info("Umount `%s` successfully.", mounted_path)
if dmcrypt:
###########################
-def _remove_from_crush_map(cluster, osd_id):
- LOG.info("Prepare to remove osd.%s from crush map..." % osd_id)
- command([
- 'ceph',
- 'osd',
- 'crush',
- 'remove',
- 'osd.%s' % osd_id,
- ])
-
-
-def _delete_osd_auth_key(cluster, osd_id):
- LOG.info("Prepare to delete osd.%s cephx key..." % osd_id)
- command([
- 'ceph',
- 'auth',
- 'del',
- 'osd.%s' % osd_id,
- ])
-
-
-def _deallocate_osd_id(cluster, osd_id):
- LOG.info("Prepare to deallocate the osd-id: %s..." % osd_id)
- command([
- 'ceph',
- 'osd',
- 'rm',
- '%s' % osd_id,
- ])
-
-
def _remove_lockbox(uuid, cluster):
- command([
- 'ceph',
- '--cluster', cluster,
- 'auth',
- 'del',
- 'client.osd-lockbox.' + uuid,
- ])
- command([
- 'ceph',
- '--cluster', cluster,
- 'config-key',
- 'del',
- 'dm-crypt/osd/' + uuid + '/luks',
- ])
lockbox = os.path.join(STATEDIR, 'osd-lockbox')
if not os.path.exists(lockbox):
return
raise Error("Could not destroy the active osd. (osd-id: %s)" %
osd_id)
- # Remove OSD from crush map
- _remove_from_crush_map(args.cluster, osd_id)
-
- # Remove OSD cephx key
- _delete_osd_auth_key(args.cluster, osd_id)
-
- # Deallocate OSD ID
- _deallocate_osd_id(args.cluster, osd_id)
+ if args.purge:
+ action = 'purge'
+ else:
+ action = 'destroy'
+ LOG.info("Prepare to %s osd.%s" % (action, osd_id))
+ command([
+ 'ceph',
+ 'osd',
+ action,
+ 'osd.%s' % osd_id,
+ '--yes-i-really-mean-it',
+ ])
# we remove the crypt map and device mapper (if dmcrypt is True)
if dmcrypt:
if not os.path.exists(path):
raise Error('%s does not exist' % path)
- mode = os.stat(path).st_mode
- if not stat.S_ISBLK(mode):
+ if not path_is_diskdevice(path):
raise Error('%s is not a block device' % path)
if (is_partition(path) and
if not os.path.exists(args.dev):
raise Error('%s does not exist' % args.dev)
+ if is_suppressed(args.dev):
+ LOG.info('suppressed activate request on space %s', args.dev)
+ return
+
cluster = None
osd_id = None
osd_uuid = None
disk = os.path.realpath(path)
try:
if (not disk.startswith('/dev/') or
- not stat.S_ISBLK(os.lstat(disk).st_mode)):
+ not ldev_is_diskdevice(disk)):
return False
base = get_dev_name(disk)
while len(base):
disk = os.path.realpath(path)
if not os.path.exists(disk):
raise Error('does not exist', path)
- if not stat.S_ISBLK(os.lstat(path).st_mode):
+ if not ldev_is_diskdevice(path):
raise Error('not a block device', path)
base = get_dev_name(disk)
disk = os.path.realpath(path)
if not os.path.exists(disk):
raise Error('does not exist', path)
- if not stat.S_ISBLK(os.lstat(path).st_mode):
+ if not ldev_is_diskdevice(path):
raise Error('not a block device', path)
assert disk.startswith('/dev/')
base = get_dev_name(disk)
def main_fix(args):
# A hash table containing 'path': ('uid', 'gid', blocking, recursive)
fix_table = [
- ('/usr/bin/ceph-mon', 'root', 'root', True, False),
- ('/usr/bin/ceph-mds', 'root', 'root', True, False),
- ('/usr/bin/ceph-osd', 'root', 'root', True, False),
- ('/usr/bin/radosgw', 'root', 'root', True, False),
- ('/etc/ceph', 'root', 'root', True, True),
+ ('/usr/bin/ceph-mon', 'root', ROOTGROUP, True, False),
+ ('/usr/bin/ceph-mds', 'root', ROOTGROUP, True, False),
+ ('/usr/bin/ceph-osd', 'root', ROOTGROUP, True, False),
+ ('/usr/bin/radosgw', 'root', ROOTGROUP, True, False),
+ ('/etc/ceph', 'root', ROOTGROUP, True, True),
('/var/run/ceph', 'ceph', 'ceph', True, True),
('/var/log/ceph', 'ceph', 'ceph', True, True),
('/var/log/radosgw', 'ceph', 'ceph', True, True),
destroy_parser = subparsers.add_parser(
'destroy',
formatter_class=argparse.RawDescriptionHelpFormatter,
- description=textwrap.fill(textwrap.dedent("""\
- Destroy the OSD located at PATH.
- It removes the OSD from the cluster, the crushmap and
- deallocates the OSD id. An OSD must be down before it
- can be destroyed.
- """)),
+ description=textwrap.fill(textwrap.dedent("""\ Destroy the OSD located at PATH. It removes the OSD from the
+ cluster and marks it destroyed. An OSD must be down before it
+ can be destroyed. Once it is destroyed, a new OSD can be created
+ in its place, reusing the same OSD id and position (e.g. after
+ a failed HDD or SSD is replaced). Alternatively, if the
+ --purge option is also specified, the OSD is removed from the
+ CRUSH map and the OSD id is deallocated.""")),
help='Destroy a Ceph OSD')
destroy_parser.add_argument(
'--cluster',
action='store_true', default=False,
help='option to erase data and partition',
)
+ destroy_parser.add_argument(
+ '--purge',
+ action='store_true', default=False,
+ help='option to remove OSD from CRUSH map and deallocate the id',
+ )
destroy_parser.set_defaults(
func=main_destroy,
)