#!/usr/bin/python3
DEFAULT_IMAGE='docker.io/ceph/ceph:v15'
+DEFAULT_IMAGE_IS_MASTER=False
+LATEST_STABLE_RELEASE='octopus'
DATA_DIR='/var/lib/ceph'
LOG_DIR='/var/log/ceph'
LOCK_DIR='/run/cephadm'
# config-json options
self.pool = json_get('pool', require=True)
self.namespace = json_get('namespace')
+ self.userid = json_get('userid')
+ self.extra_args = json_get('extra_args', [])
self.files = json_get('files', {})
# validate the supplied args
cname = '%s-%s' % (cname, desc)
return cname
+ def get_daemon_args(self):
+ # type: () -> List[str]
+ return self.daemon_args + self.extra_args
+
def get_file_content(self, fname):
# type: (str) -> str
"""Normalize the json file content into a string"""
args=['--pool', self.pool]
if self.namespace:
args += ['--ns', self.namespace]
+ if self.userid:
+ args += ['--userid', self.userid]
args += [action, self.get_daemon_name()]
data_dir = get_data_dir(self.fsid, self.daemon_type, self.daemon_id)
##################################
+class CephIscsi(object):
+ """Defines a Ceph-Iscsi container"""
+
+ daemon_type = 'iscsi'
+ entrypoint = '/usr/bin/rbd-target-api'
+
+ required_files = ['iscsi-gateway.cfg']
+
+ def __init__(self,
+ fsid,
+ daemon_id,
+ config_json,
+ image=DEFAULT_IMAGE):
+ # type: (str, Union[int, str], Dict, str) -> None
+ self.fsid = fsid
+ self.daemon_id = daemon_id
+ self.image = image
+
+ def json_get(key, default=None, require=False):
+ if require and not key in config_json.keys():
+ raise Error('{} missing from config-json'.format(key))
+ return config_json.get(key, default)
+
+ # config-json options
+ self.files = json_get('files', {})
+
+ # validate the supplied args
+ self.validate()
+
+ @classmethod
+ def init(cls, fsid, daemon_id):
+ # type: (str, Union[int, str]) -> CephIscsi
+ return cls(fsid, daemon_id, get_parm(args.config_json), args.image)
+
+ @staticmethod
+ def get_container_mounts(data_dir, log_dir):
+ # type: (str, str) -> Dict[str, str]
+ mounts = dict()
+ mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z'
+ mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z'
+ mounts[os.path.join(data_dir, 'iscsi-gateway.cfg')] = '/etc/ceph/iscsi-gateway.cfg:z'
+ mounts[os.path.join(data_dir, 'configfs')] = '/sys/kernel/config:z'
+ mounts[log_dir] = '/var/log/rbd-target-api:z'
+ mounts['/dev/log'] = '/dev/log:z'
+ return mounts
+
+ @staticmethod
+ def get_version(container_id):
+ # type(str) -> Optional[str]
+ version = None
+ out, err, code = call(
+ [container_path, 'exec', container_id,
+ '/usr/bin/python3', '-c', "import pkg_resources; print(pkg_resources.require('ceph_iscsi')[0].version)"])
+ if code == 0:
+ version = out
+ return version
+
+ def validate(self):
+ # type () -> None
+ if not is_fsid(self.fsid):
+ raise Error('not an fsid: %s' % self.fsid)
+ if not self.daemon_id:
+ raise Error('invalid daemon_id: %s' % self.daemon_id)
+ if not self.image:
+ raise Error('invalid image: %s' % self.image)
+
+ # check for the required files
+ if self.required_files:
+ for fname in self.required_files:
+ if fname not in self.files:
+ raise Error('required file missing from config-json: %s' % fname)
+
+ def get_daemon_name(self):
+ # type: () -> str
+ return '%s.%s' % (self.daemon_type, self.daemon_id)
+
+ def get_container_name(self, desc=None):
+ # type: (Optional[str]) -> str
+ cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name())
+ if desc:
+ cname = '%s-%s' % (cname, desc)
+ return cname
+
+ def get_file_content(self, fname):
+ # type: (str) -> str
+ """Normalize the json file content into a string"""
+ content = self.files.get(fname)
+ if isinstance(content, list):
+ content = '\n'.join(content)
+ return content
+
+ def create_daemon_dirs(self, data_dir, uid, gid):
+ # type: (str, int, int) -> None
+ """Create files under the container data dir"""
+ if not os.path.isdir(data_dir):
+ raise OSError('data_dir is not a directory: %s' % (data_dir))
+
+ logger.info('Creating ceph-iscsi config...')
+ configfs_dir = os.path.join(data_dir, 'configfs')
+ makedirs(configfs_dir, uid, gid, 0o755)
+
+ # populate files from the config-json
+ for fname in self.files:
+ config_file = os.path.join(data_dir, fname)
+ config_content = self.get_file_content(fname)
+ logger.info('Write file: %s' % (config_file))
+ with open(config_file, 'w') as f:
+ os.fchown(f.fileno(), uid, gid)
+ os.fchmod(f.fileno(), 0o600)
+ f.write(config_content)
+
+ @staticmethod
+ def configfs_mount_umount(data_dir, mount=True):
+ mount_path = os.path.join(data_dir, 'configfs')
+ if mount:
+ cmd = "if ! grep -qs {0} /proc/mounts; then " \
+ "mount -t configfs none {0}; fi".format(mount_path)
+ else:
+ cmd = "if grep -qs {0} /proc/mounts; then " \
+ "umount {0}; fi".format(mount_path)
+ return cmd.split()
+
+##################################
+
def get_supported_daemons():
supported_daemons = list(Ceph.daemons)
supported_daemons.extend(Monitoring.components)
supported_daemons.append(NFSGanesha.daemon_type)
+ supported_daemons.append(CephIscsi.daemon_type)
assert len(supported_daemons) == len(set(supported_daemons))
return supported_daemons
logger.info('Verifying IP %s port %d ...' % (ip, port))
if ip.startswith('[') or '::' in ip:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+ if ip.startswith('[') and ip.endswith(']'):
+ ip = ip[1:-1]
else:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
return _infer_fsid
+def _get_default_image():
+ if DEFAULT_IMAGE_IS_MASTER:
+ yellow = '\033[93m'
+ end = '\033[0m'
+ warn = '''This is a development version of cephadm.
+For information regarding the latest stable release:
+ https://docs.ceph.com/docs/{}/cephadm/install
+'''.format(LATEST_STABLE_RELEASE)
+ for line in warn.splitlines():
+ logger.warning('{}{}{}'.format(yellow, line, end))
+ return DEFAULT_IMAGE
+
def infer_image(func):
"""
Use the most recent ceph image
if not args.image:
args.image = get_last_local_ceph_image()
if not args.image:
- args.image = DEFAULT_IMAGE
+ args.image = _get_default_image()
return func()
return _infer_image
if not args.image:
args.image = os.environ.get('CEPHADM_IMAGE')
if not args.image:
- args.image = DEFAULT_IMAGE
+ args.image = _get_default_image()
+
return func()
return _default_image
for peer in peers:
r += ["--cluster.peer={}".format(peer)]
elif daemon_type == NFSGanesha.daemon_type:
- r += NFSGanesha.daemon_args
+ nfs_ganesha = NFSGanesha.init(fsid, daemon_id)
+ r += nfs_ganesha.get_daemon_args()
return r
def create_daemon_dirs(fsid, daemon_type, daemon_id, uid, gid,
- config=None, keyring=None,
- reconfig=False):
+ config=None, keyring=None, reconfig=False):
# type: (str, str, Union[int, str], int, int, Optional[str], Optional[str], Optional[bool]) -> None
data_dir = make_data_dir(fsid, daemon_type, daemon_id, uid=uid, gid=gid)
make_log_dir(fsid, uid=uid, gid=gid)
nfs_ganesha = NFSGanesha.init(fsid, daemon_id)
nfs_ganesha.create_daemon_dirs(data_dir, uid, gid)
+ if daemon_type == CephIscsi.daemon_type:
+ ceph_iscsi = CephIscsi.init(fsid, daemon_id)
+ ceph_iscsi.create_daemon_dirs(data_dir, uid, gid)
+
def get_parm(option):
# type: (str) -> Dict[str, str]
data_dir = get_data_dir(fsid, daemon_type, daemon_id)
mounts.update(NFSGanesha.get_container_mounts(data_dir))
+ if daemon_type == CephIscsi.daemon_type:
+ assert daemon_id
+ data_dir = get_data_dir(fsid, daemon_type, daemon_id)
+ log_dir = get_log_dir(fsid)
+ mounts.update(CephIscsi.get_container_mounts(data_dir, log_dir))
+
return mounts
def get_container(fsid, daemon_type, daemon_id,
elif daemon_type == NFSGanesha.daemon_type:
entrypoint = NFSGanesha.entrypoint
name = '%s.%s' % (daemon_type, daemon_id)
+ elif daemon_type == CephIscsi.daemon_type:
+ entrypoint = CephIscsi.entrypoint
+ name = '%s.%s' % (daemon_type, daemon_id)
else:
entrypoint = ''
name = ''
nfs_ganesha = NFSGanesha.init(fsid, daemon_id)
prestart = nfs_ganesha.get_rados_grace_container('add')
f.write(' '.join(prestart.run_cmd()) + '\n')
+ elif daemon_type == CephIscsi.daemon_type:
+ f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=True)) + '\n')
+
+ if daemon_type in Ceph.daemons:
+ install_path = find_program('install')
+ f.write('{install_path} -d -m0770 -o {uid} -g {gid} /var/run/ceph/{fsid}\n'.format(install_path=install_path, fsid=fsid, uid=uid, gid=gid))
# container run command
f.write(' '.join(c.run_cmd()) + '\n')
nfs_ganesha = NFSGanesha.init(fsid, daemon_id)
poststop = nfs_ganesha.get_rados_grace_container('remove')
f.write(' '.join(poststop.run_cmd()) + '\n')
+ elif daemon_type == CephIscsi.daemon_type:
+ f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=False)) + '\n')
os.fchmod(f.fileno(), 0o600)
os.rename(data_dir + '/unit.poststop.new',
data_dir + '/unit.poststop')
# systemd
install_base_units(fsid)
- unit = get_unit_file(fsid, uid, gid)
+ unit = get_unit_file(fsid)
unit_file = 'ceph-%s@.service' % (fsid)
with open(args.unit_dir + '/' + unit_file + '.new', 'w') as f:
f.write(unit)
}
""" % fsid)
-def get_unit_file(fsid, uid, gid):
- # type: (str, int, int) -> str
- install_path = find_program('install')
+def get_unit_file(fsid):
+ # type: (str) -> str
u = """# generated by cephadm
[Unit]
Description=Ceph %i for {fsid}
LimitNPROC=1048576
EnvironmentFile=-/etc/environment
ExecStartPre=-{container_path} rm ceph-{fsid}-%i
-ExecStartPre=-{install_path} -d -m0770 -o {uid} -g {gid} /var/run/ceph/{fsid}
ExecStart=/bin/bash {data_dir}/{fsid}/%i/unit.run
ExecStop=-{container_path} stop ceph-{fsid}-%i
ExecStopPost=-/bin/bash {data_dir}/{fsid}/%i/unit.poststop
WantedBy=ceph-{fsid}.target
""".format(
container_path=container_path,
- install_path=install_path,
fsid=fsid,
- uid=uid,
- gid=gid,
data_dir=args.data_dir)
return u
cmd = ['dashboard', 'ac-user-create', args.initial_dashboard_user, password, 'administrator', '--force-password']
if not args.dashboard_password_noupdate:
cmd.append('--pwd-update-required')
- cli(cmd)
+ cli(cmd)
logger.info('Fetching dashboard port number...')
out = cli(['config', 'get', 'mgr', 'mgr/dashboard/ssl_server_port'])
port = int(out)
deploy_daemon(args.fsid, daemon_type, daemon_id, c, uid, gid,
config=config, keyring=keyring,
reconfig=args.reconfig)
+ elif daemon_type == CephIscsi.daemon_type:
+ (config, keyring) = get_config_and_keyring()
+ (uid, gid) = extract_uid_gid()
+ c = get_container(args.fsid, daemon_type, daemon_id)
+ deploy_daemon(args.fsid, daemon_type, daemon_id, c, uid, gid,
+ config=config, keyring=keyring,
+ reconfig=args.reconfig)
else:
raise Error("{} not implemented in command_deploy function".format(daemon_type))
'-e', 'LANG=C',
'-e', "PS1=%s" % CUSTOM_PS1,
]
- c = get_container(args.fsid, daemon_type, daemon_id,
- container_args=container_args)
+ c = CephContainer(
+ image=args.image,
+ entrypoint='doesnotmatter',
+ container_args=container_args,
+ cname='ceph-%s-%s.%s' % (args.fsid, daemon_type, daemon_id),
+ )
command = c.exec_cmd(command)
return call_timeout(command, args.timeout)
if args.fsid:
make_log_dir(args.fsid)
+ l = FileLock(args.fsid)
+ l.acquire()
+
(uid, gid) = (0, 0) # ceph-volume runs as root
mounts = get_container_mounts(args.fsid, 'osd', None)
version = seen_versions.get(image_id, None)
if daemon_type == NFSGanesha.daemon_type:
version = NFSGanesha.get_version(container_id)
+ if daemon_type == CephIscsi.daemon_type:
+ version = CephIscsi.get_version(container_id)
elif not version:
if daemon_type in Ceph.daemons:
out, err, code = call(
raise Error('daemon type %s not recognized' % daemon_type)
+class AdoptOsd(object):
+ def __init__(self, osd_data_dir, osd_id):
+ # type: (str, str) -> None
+ self.osd_data_dir = osd_data_dir
+ self.osd_id = osd_id
+
+ def check_online_osd(self):
+ # type: () -> Tuple[Optional[str], Optional[str]]
+
+ osd_fsid, osd_type = None, None
+
+ path = os.path.join(self.osd_data_dir, 'fsid')
+ try:
+ with open(path, 'r') as f:
+ osd_fsid = f.read().strip()
+ logger.info("Found online OSD at %s" % path)
+ if os.path.exists(os.path.join(self.osd_data_dir, 'type')):
+ with open(os.path.join(self.osd_data_dir, 'type')) as f:
+ osd_type = f.read().strip()
+ else:
+ logger.info('"type" file missing for OSD data dir')
+ except IOError:
+ logger.info('Unable to read OSD fsid from %s' % path)
+
+ return osd_fsid, osd_type
+
+ def check_offline_lvm_osd(self):
+ # type: () -> Tuple[Optional[str], Optional[str]]
+
+ osd_fsid, osd_type = None, None
+
+ c = CephContainer(
+ image=args.image,
+ entrypoint='/usr/sbin/ceph-volume',
+ args=['lvm', 'list', '--format=json'],
+ privileged=True
+ )
+ out, err, code = call_throws(c.run_cmd(), verbose=False)
+ if not code:
+ try:
+ js = json.loads(out)
+ if self.osd_id in js:
+ logger.info("Found offline LVM OSD {}".format(self.osd_id))
+ osd_fsid = js[self.osd_id][0]['tags']['ceph.osd_fsid']
+ for device in js[self.osd_id]:
+ if device['tags']['ceph.type'] == 'block':
+ osd_type = 'bluestore'
+ break
+ if device['tags']['ceph.type'] == 'data':
+ osd_type = 'filestore'
+ break
+ except ValueError as e:
+ logger.info("Invalid JSON in ceph-volume lvm list: {}".format(e))
+
+ return osd_fsid, osd_type
+
+ def check_offline_simple_osd(self):
+ # type: () -> Tuple[Optional[str], Optional[str]]
+
+ osd_fsid, osd_type = None, None
+
+ osd_file = glob("/etc/ceph/osd/{}-[a-f0-9-]*.json".format(self.osd_id))
+ if len(osd_file) == 1:
+ with open(osd_file[0], 'r') as f:
+ try:
+ js = json.loads(f.read())
+ logger.info("Found offline simple OSD {}".format(self.osd_id))
+ osd_fsid = js["fsid"]
+ osd_type = js["type"]
+ if osd_type != "filestore":
+ # need this to be mounted for the adopt to work, as it
+ # needs to move files from this directory
+ call_throws(['mount', js["data"]["path"], self.osd_data_dir])
+ except ValueError as e:
+ logger.info("Invalid JSON in {}: {}".format(osd_file, e))
+
+ return osd_fsid, osd_type
+
def command_adopt_ceph(daemon_type, daemon_id, fsid):
# type: (str, str, str) -> None
(daemon_type, args.cluster, daemon_id))
data_dir_src = os.path.abspath(args.legacy_dir + data_dir_src)
+ if not os.path.exists(data_dir_src):
+ raise Error("{}.{} data directory '{}' does not exist. "
+ "Incorrect ID specified, or daemon alrady adopted?".format(
+ daemon_type, daemon_id, data_dir_src))
+
osd_fsid = None
if daemon_type == 'osd':
- path = os.path.join(data_dir_src, 'fsid')
- try:
- with open(path, 'r') as f:
- osd_fsid = f.read().strip()
- except IOError:
- raise Error('unable to read OSD fsid from %s' % path)
- os_type = None
- if os.path.exists(os.path.join(data_dir_src, 'type')):
- with open(os.path.join(data_dir_src, 'type')) as f:
- os_type = f.read().strip()
- else:
- raise Error('"type" file missing for OSD data dir')
- logger.info('objectstore_type is %s' % os_type)
- if os_type == 'filestore':
+ adopt_osd = AdoptOsd(data_dir_src, daemon_id)
+ osd_fsid, osd_type = adopt_osd.check_online_osd()
+ if not osd_fsid:
+ osd_fsid, osd_type = adopt_osd.check_offline_lvm_osd()
+ if not osd_fsid:
+ osd_fsid, osd_type = adopt_osd.check_offline_simple_osd()
+ if not osd_fsid:
+ raise Error('Unable to find OSD {}'.format(daemon_id))
+ logger.info('objectstore_type is %s' % osd_type)
+ if osd_type == 'filestore':
raise Error('FileStore is not supported by cephadm')
# NOTE: implicit assumption here that the units correspond to the
logger.info('Renaming %s -> %s', simple_fn, new_fn)
os.rename(simple_fn, new_fn)
logger.info('Disabling host unit ceph-volume@ simple unit...')
- call_throws(['systemctl', 'disable',
- 'ceph-volume@simple-%s-%s.service' % (
- daemon_id, osd_fsid)])
+ call(['systemctl', 'disable',
+ 'ceph-volume@simple-%s-%s.service' % (daemon_id, osd_fsid)])
else:
# assume this is an 'lvm' c-v for now, but don't error
# out if it's not.
c = get_container(fsid, daemon_type, daemon_id)
deploy_daemon_units(fsid, uid, gid, daemon_type, daemon_id, c,
enable=True, # unconditionally enable the new unit
- start=(state == 'running'),
+ start=(state == 'running' or args.force_start),
osd_fsid=osd_fsid)
update_firewalld(daemon_type)
config_src = '/etc/prometheus/prometheus.yml'
config_src = os.path.abspath(args.legacy_dir + config_src)
config_dst = os.path.join(data_dir_dst, 'etc/prometheus')
+ makedirs(config_dst, uid, gid, 0o755)
copy_files([config_src], config_dst, uid=uid, gid=gid)
# data
# rm logrotate config
call_throws(['rm', '-f', args.logrotate_dir + '/ceph-%s' % args.fsid])
+ # clean up config, keyring, and pub key files
+ files = ['/etc/ceph/ceph.conf', '/etc/ceph/ceph.pub', '/etc/ceph/ceph.client.admin.keyring']
+
+ if os.path.exists(files[0]):
+ valid_fsid = False
+ with open(files[0]) as f:
+ if args.fsid in f.read():
+ valid_fsid = True
+ if valid_fsid:
+ for n in range(0, len(files)):
+ if os.path.exists(files[n]):
+ os.remove(files[n])
+
##################################
def command_check_host():
# type: () -> None
- # caller already checked for docker/podman
- logger.info('podman|docker (%s) is present' % container_path)
-
+ errors = []
commands = ['systemctl', 'lvcreate']
+ if args.docker:
+ container_path = find_program('docker')
+ else:
+ for i in CONTAINER_PREFERENCE:
+ try:
+ container_path = find_program(i)
+ break
+ except Exception as e:
+ logger.debug('Could not locate %s: %s' % (i, e))
+ if not container_path:
+ errors.append('Unable to locate any of %s' % CONTAINER_PREFERENCE)
+ else:
+ logger.info('podman|docker (%s) is present' % container_path)
+
for command in commands:
try:
find_program(command)
logger.info('%s is present' % command)
except ValueError:
- raise Error('%s binary does not appear to be installed' % command)
+ errors.append('%s binary does not appear to be installed' % command)
# check for configured+running chronyd or ntp
if not check_time_sync():
- raise Error('No time synchronization is active')
+ errors.append('No time synchronization is active')
if 'expect_hostname' in args and args.expect_hostname:
- if get_hostname() != args.expect_hostname:
- raise Error('hostname "%s" does not match expected hostname "%s"' % (
+ if get_hostname().lower() != args.expect_hostname.lower():
+ errors.append('hostname "%s" does not match expected hostname "%s"' % (
get_hostname(), args.expect_hostname))
logger.info('Hostname "%s" matches what is expected.',
args.expect_hostname)
+ if errors:
+ raise Error('\n'.join(errors))
+
logger.info('Host looks OK')
##################################
def command_add_repo():
if args.version and args.release:
raise Error('you can specify either --release or --version but not both')
+ if not args.version and not args.release and not args.dev and not args.dev_commit:
+ raise Error('please supply a --release, --version, --dev or --dev-commit argument')
if args.version:
try:
(x, y, z) = args.version.split('.')
'--skip-pull',
action='store_true',
help='do not pull the latest image before adopting')
+ parser_adopt.add_argument(
+ '--force-start',
+ action='store_true',
+ help="start newly adoped daemon, even if it wasn't running previously")
parser_rm_daemon = subparsers.add_parser(
'rm-daemon', help='remove daemon instance')
parser_add_repo.set_defaults(func=command_add_repo)
parser_add_repo.add_argument(
'--release',
- help='use latest version of a named release (e.g., octopus)')
+ help='use latest version of a named release (e.g., {})'.format(LATEST_STABLE_RELEASE))
parser_add_repo.add_argument(
'--version',
help='use specific upstream version (x.y.z)')
sys.stderr.write('ERROR: cephadm should be run as root\n')
sys.exit(1)
- # podman or docker?
- if args.docker:
- container_path = find_program('docker')
- else:
- for i in CONTAINER_PREFERENCE:
- try:
- container_path = find_program(i)
- break
- except Exception as e:
- logger.debug('Could not locate %s: %s' % (i, e))
- if not container_path and args.func != command_prepare_host:
- sys.stderr.write('Unable to locate any of %s\n' % CONTAINER_PREFERENCE)
- sys.exit(1)
-
if 'func' not in args:
sys.stderr.write('No command specified; pass -h or --help for usage\n')
sys.exit(1)
+ # podman or docker?
+ if args.func != command_check_host:
+ if args.docker:
+ container_path = find_program('docker')
+ else:
+ for i in CONTAINER_PREFERENCE:
+ try:
+ container_path = find_program(i)
+ break
+ except Exception as e:
+ logger.debug('Could not locate %s: %s' % (i, e))
+ if not container_path and args.func != command_prepare_host\
+ and args.func != command_add_repo:
+ sys.stderr.write('Unable to locate any of %s\n' % CONTAINER_PREFERENCE)
+ sys.exit(1)
+
try:
r = args.func()
except Error as e:
projects / ceph.git / blobdiff