[pve-manager.git] / PVE / Ceph / Services.pm

package PVE::Ceph::Services;

use strict;
use warnings;

use PVE::Ceph::Tools;
use PVE::Cluster qw(cfs_read_file);
use PVE::Tools qw(run_command);
use PVE::RADOS;

use JSON;
use File::Path;

use constant SERVICE_REGEX => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?';

# checks /etc/systemd/system/ceph-* to list all services, even if not running
# also checks /var/lib/ceph/$type
sub get_local_services {
    my $res = {};

    for my $type (qw(mds mgr mon)) {
	$res->{$type} = {};

	my $path = "/etc/systemd/system/ceph-$type.target.wants";
	my $regex = "ceph-$type\@(.*)\.service";
	PVE::Tools::dir_glob_foreach($path, $regex, sub {
	    my (undef, $id) = @_;
	    $res->{$type}->{$id}->{service} = 1;
	});

	$path = "/var/lib/ceph/$type";
	$regex = "([^-]+)-(.*)";
	PVE::Tools::dir_glob_foreach($path, $regex, sub {
	    my (undef, $clustername, $id) = @_;
	    $res->{$type}->{$id}->{direxists} = 1;
	});
    }
    return $res;
}

sub broadcast_ceph_services {
    my $services = get_local_services();

    for my $type (keys %$services) {
	my $data = encode_json($services->{$type});
	PVE::Cluster::broadcast_node_kv("ceph-$type", $data);
    }
}

sub broadcast_ceph_versions {
    my ($version, $buildcommit, $vers_parts) = PVE::Ceph::Tools::get_local_version(1);

    if ($version) {
	my $nodename = PVE::INotify::nodename();
	my $old = PVE::Cluster::get_node_kv("ceph-versions", $nodename);
	if (defined($old->{$nodename})) {
	    $old = eval { decode_json($old->{$nodename}) };
	    warn $@ if $@; # should not happen
	    if (defined($old) && $old->{buildcommit} eq $buildcommit && $old->{version}->{str} eq $version) {
		return; # up to date, nothing to do so avoid (not exactly cheap) broadcast
	    }
	}
	# FIXME: remove with 8.0 (or 7.2, its not _that_ bad) - for backward compat only
	PVE::Cluster::broadcast_node_kv("ceph-version", $version);

	my $node_versions = {
	    version => {
		str => $version,
		parts => $vers_parts,
	    },
	    buildcommit => $buildcommit,
	};
	PVE::Cluster::broadcast_node_kv("ceph-versions", encode_json($node_versions));
    }
}

sub get_ceph_versions {
    my $res;

    if (defined(my $versions = PVE::Cluster::get_node_kv("ceph-versions"))) {
	$res = {
	    map { eval { $_ => decode_json($versions->{$_}) } } keys %$versions
	};
    }

    return $res;
}

sub get_cluster_service {
    my ($type) = @_;

    my $raw = PVE::Cluster::get_node_kv("ceph-$type");
    my $res = {
	map { $_ => eval { decode_json($raw->{$_}) } } keys $raw->%*
    };

    return $res;
}

sub ceph_service_cmd {
    my ($action, $service) = @_;

    if ($service && $service =~ m/^(mon|osd|mds|mgr|radosgw)(\.(${\SERVICE_REGEX}))?$/) {
	$service = defined($3) ? "ceph-$1\@$3" : "ceph-$1.target";
    } else {
	$service = "ceph.target";
    }

    run_command(['/bin/systemctl', $action, $service]);
}

sub get_services_info {
    my ($type, $cfg, $rados) = @_;

    my $result = {};
    my $services = get_cluster_service($type);

    foreach my $host (sort keys %$services) {
	foreach my $id (sort keys %{$services->{$host}}) {
	    my $service = $result->{$id} = $services->{$host}->{$id};
	    $service->{host} = $host;
	    $service->{name} = $id;
	    $service->{state} = 'unknown';
	    if ($service->{service}) {
		$service->{state} = 'stopped';
	    }
	}
    }

    if (!$cfg) {
	$cfg = cfs_read_file('ceph.conf');
    }

    foreach my $section (keys %$cfg) {
	my $d = $cfg->{$section};
	if ($section =~ m/^$type\.(\S+)$/) {
	    my $id = $1;
	    my $service = $result->{$id};
	    my $addr = $d->{"${type}_addr"} // $d->{public_addr} // $d->{host};
	    $service->{name} //= $id;
	    $service->{addr} //= $addr;
	    $service->{state} //= 'unknown';
	    $service->{host} //= $d->{host};
	}
    }

    if (!$rados) {
	return $result;
    }

    my $metadata = $rados->mon_command({ prefix => "$type metadata" });
    foreach my $info (@$metadata) {
	my $id = $info->{name} // $info->{id};
	my $service = $result->{$id};
	$service->{ceph_version_short} = $info->{ceph_version_short};
	$service->{ceph_version} = $info->{ceph_version};
	$service->{host} //= $info->{hostname};
	$service->{addr} //= $info->{addr};
    }

    return $result;
}

# MDS

sub list_local_mds_ids {
    my $mds_list = [];
    my $ceph_mds_data_dir = PVE::Ceph::Tools::get_config('ceph_mds_data_dir');
    my $ccname = PVE::Ceph::Tools::get_config('ccname');

    PVE::Tools::dir_glob_foreach($ceph_mds_data_dir, qr/$ccname-(\S+)/, sub {
	my (undef, $mds_id) = @_;
	push @$mds_list, $mds_id;
    });

    return $mds_list;
}

sub get_cluster_mds_state {
    my ($rados) = @_;

    my $mds_state = {};

    if (!defined($rados)) {
	$rados = PVE::RADOS->new();
    }

    my $add_state = sub {
	my ($mds, $fsname) = @_;

	my $state = {};
	$state->{addr} = $mds->{addr};
	$state->{rank} = $mds->{rank};
	$state->{standby_replay} = $mds->{standby_replay} ? 1 : 0;
	$state->{state} = $mds->{state};
	$state->{fs_name} = $fsname;

	$mds_state->{$mds->{name}} = $state;
    };

    my $mds_dump = $rados->mon_command({ prefix => 'mds stat' });
    my $fsmap = $mds_dump->{fsmap};


    foreach my $mds (@{$fsmap->{standbys}}) {
	$add_state->($mds);
    }

    for my $fs_info (@{$fsmap->{filesystems}}) {
	my $active_mds = $fs_info->{mdsmap}->{info};

	# normally there's only one active MDS, but we can have multiple active for
	# different ranks (e.g., different cephs path hierarchy). So just add all.
	foreach my $mds (values %$active_mds) {
	    $add_state->($mds, $fs_info->{mdsmap}->{fs_name});
	}
    }

    return $mds_state;
}

sub is_mds_active {
    my ($rados, $fs_name) = @_;

    if (!defined($rados)) {
	$rados = PVE::RADOS->new();
    }

    my $mds_dump = $rados->mon_command({ prefix => 'mds stat' });
    my $fsmap = $mds_dump->{fsmap}->{filesystems};

    if (!($fsmap && scalar(@$fsmap) > 0)) {
	return undef;
    }
    for my $fs (@$fsmap) {
	next if defined($fs_name) && $fs->{mdsmap}->{fs_name} ne $fs_name;

	my $active_mds = $fs->{mdsmap}->{info};
	for my $mds (values %$active_mds) {
	    return 1 if $mds->{state} eq 'up:active';
	}
    }

    return 0;
}

sub create_mds {
    my ($id, $rados) = @_;

    # `ceph fs status` fails with numeric only ID.
    die "ID: $id, numeric only IDs are not supported\n"
	if $id =~ /^\d+$/;

    if (!defined($rados)) {
	$rados = PVE::RADOS->new();
    }

    my $ccname = PVE::Ceph::Tools::get_config('ccname');
    my $service_dir = "/var/lib/ceph/mds/$ccname-$id";
    my $service_keyring = "$service_dir/keyring";
    my $service_name = "mds.$id";

    die "ceph MDS directory '$service_dir' already exists\n"
	if -d $service_dir;

    print "creating MDS directory '$service_dir'\n";
    eval { File::Path::mkpath($service_dir) };
    my $err = $@;
    die "creation MDS directory '$service_dir' failed\n" if $err;

    # http://docs.ceph.com/docs/luminous/install/manual-deployment/#adding-mds
    my $priv = [
	mon => 'allow profile mds',
	osd => 'allow rwx',
	mds => 'allow *',
    ];

    print "creating keys for '$service_name'\n";
    my $output = $rados->mon_command({
	prefix => 'auth get-or-create',
	entity => $service_name,
	caps => $priv,
	format => 'plain',
    });

    PVE::Tools::file_set_contents($service_keyring, $output);

    print "setting ceph as owner for service directory\n";
    run_command(["chown", 'ceph:ceph', '-R', $service_dir]);

    print "enabling service 'ceph-mds\@$id.service'\n";
    ceph_service_cmd('enable', $service_name);
    print "starting service 'ceph-mds\@$id.service'\n";
    ceph_service_cmd('start', $service_name);

    broadcast_ceph_services();

    return undef;
};

sub destroy_mds {
    my ($id, $rados) = @_;

    if (!defined($rados)) {
	$rados = PVE::RADOS->new();
    }

    my $ccname = PVE::Ceph::Tools::get_config('ccname');

    my $service_name = "mds.$id";
    my $service_dir = "/var/lib/ceph/mds/$ccname-$id";

    print "disabling service 'ceph-mds\@$id.service'\n";
    ceph_service_cmd('disable', $service_name);
    print "stopping service 'ceph-mds\@$id.service'\n";
    ceph_service_cmd('stop', $service_name);

    if (-d $service_dir) {
	print "removing ceph-mds directory '$service_dir'\n";
	File::Path::remove_tree($service_dir);
    } else {
	warn "cannot cleanup MDS $id directory, '$service_dir' not found\n"
    }

    print "removing ceph auth for '$service_name'\n";
    $rados->mon_command({
	    prefix => 'auth del',
	    entity => $service_name,
	    format => 'plain'
	});

    broadcast_ceph_services();

    return undef;
};

# MGR

sub create_mgr {
    my ($id, $rados) = @_;

    my $clustername = PVE::Ceph::Tools::get_config('ccname');
    my $mgrdir = "/var/lib/ceph/mgr/$clustername-$id";
    my $mgrkeyring = "$mgrdir/keyring";
    my $mgrname = "mgr.$id";

    die "ceph manager directory '$mgrdir' already exists\n" if -d $mgrdir;

    print "creating manager directory '$mgrdir'\n";
    mkdir $mgrdir;
    print "creating keys for '$mgrname'\n";
    my $output = $rados->mon_command({
	prefix => 'auth get-or-create',
	entity => $mgrname,
	caps => [
	    mon => 'allow profile mgr',
	    osd => 'allow *',
	    mds => 'allow *',
	],
	format => 'plain'
    });
    PVE::Tools::file_set_contents($mgrkeyring, $output);

    print "setting owner for directory\n";
    run_command(["chown", 'ceph:ceph', '-R', $mgrdir]);

    print "enabling service 'ceph-mgr\@$id.service'\n";
    ceph_service_cmd('enable', $mgrname);
    print "starting service 'ceph-mgr\@$id.service'\n";
    ceph_service_cmd('start', $mgrname);

    broadcast_ceph_services();

    return undef;
}

sub destroy_mgr {
    my ($mgrid, $rados) = @_;

    my $clustername = PVE::Ceph::Tools::get_config('ccname');
    my $mgrname = "mgr.$mgrid";
    my $mgrdir = "/var/lib/ceph/mgr/$clustername-$mgrid";

    die "ceph manager directory '$mgrdir' not found\n"
	if ! -d $mgrdir;

    print "disabling service 'ceph-mgr\@$mgrid.service'\n";
    ceph_service_cmd('disable', $mgrname);
    print "stopping service 'ceph-mgr\@$mgrid.service'\n";
    ceph_service_cmd('stop', $mgrname);

    print "removing manager directory '$mgrdir'\n";
    File::Path::remove_tree($mgrdir);

    print "removing authkeys for $mgrname\n";
    if (!$rados) {
	$rados = PVE::RADOS->new();
    }

    $rados->mon_command({ prefix => 'auth del', entity => "$mgrname" });

    broadcast_ceph_services();

    return undef;
}

1;
Commit	Line	Data
27439be6 DC	1	package PVE::Ceph::Services;
	2
	3	use strict;
	4	use warnings;
	5
	6	use PVE::Ceph::Tools;
d5373b7d	7	use PVE::Cluster qw(cfs_read_file);
27439be6 DC	8	use PVE::Tools qw(run_command);
	9	use PVE::RADOS;
	10
4e76dbd7	11	use JSON;
27439be6 DC	12	use File::Path;
27439be6 DC	13
7e98f79e DC	14	use constant SERVICE_REGEX => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?';
7e98f79e DC	15
ac4f971d	16	# checks /etc/systemd/system/ceph-* to list all services, even if not running
74628668	17	# also checks /var/lib/ceph/$type
23b20ae4 DC	18	sub get_local_services {
23b20ae4 DC	19	my $res = {};
ac4f971d	20
23b20ae4 DC	21	for my $type (qw(mds mgr mon)) {
23b20ae4 DC	22	$res->{$type} = {};
ac4f971d	23
23b20ae4 DC	24	my $path = "/etc/systemd/system/ceph-$type.target.wants";
	25	my $regex = "ceph-$type\@(.*)\.service";
	26	PVE::Tools::dir_glob_foreach($path, $regex, sub {
	27	my (undef, $id) = @_;
74628668 DC	28	$res->{$type}->{$id}->{service} = 1;
	29	});
	30
	31	$path = "/var/lib/ceph/$type";
	32	$regex = "([^-]+)-(.*)";
	33	PVE::Tools::dir_glob_foreach($path, $regex, sub {
	34	my (undef, $clustername, $id) = @_;
	35	$res->{$type}->{$id}->{direxists} = 1;
23b20ae4 DC	36	});
	37	}
	38	return $res;
	39	}
	40
4e76dbd7 DC	41	sub broadcast_ceph_services {
	42	my $services = get_local_services();
	43
	44	for my $type (keys %$services) {
	45	my $data = encode_json($services->{$type});
	46	PVE::Cluster::broadcast_node_kv("ceph-$type", $data);
	47	}
	48	}
	49
0496138e DC	50	sub broadcast_ceph_versions {
	51	my ($version, $buildcommit, $vers_parts) = PVE::Ceph::Tools::get_local_version(1);
	52
	53	if ($version) {
25bea057 FE	54	my $nodename = PVE::INotify::nodename();
	55	my $old = PVE::Cluster::get_node_kv("ceph-versions", $nodename);
	56	if (defined($old->{$nodename})) {
d66b726c	57	$old = eval { decode_json($old->{$nodename}) };
656667db	58	warn $@ if $@; # should not happen
d66b726c	59	if (defined($old) && $old->{buildcommit} eq $buildcommit && $old->{version}->{str} eq $version) {
656667db TL	60	return; # up to date, nothing to do so avoid (not exactly cheap) broadcast
	61	}
	62	}
04aae00d TL	63	# FIXME: remove with 8.0 (or 7.2, its not _that_ bad) - for backward compat only
	64	PVE::Cluster::broadcast_node_kv("ceph-version", $version);
	65
0496138e DC	66	my $node_versions = {
	67	version => {
	68	str => $version,
	69	parts => $vers_parts,
	70	},
	71	buildcommit => $buildcommit,
	72	};
	73	PVE::Cluster::broadcast_node_kv("ceph-versions", encode_json($node_versions));
	74	}
	75	}
	76
	77	sub get_ceph_versions {
	78	my $res;
	79
a1132828	80	if (defined(my $versions = PVE::Cluster::get_node_kv("ceph-versions"))) {
0496138e	81	$res = {
a1132828	82	map { eval { $_ => decode_json($versions->{$_}) } } keys %$versions
0496138e DC	83	};
	84	}
	85
	86	return $res;
	87	}
	88
4e76dbd7 DC	89	sub get_cluster_service {
4e76dbd7 DC	90	my ($type) = @_;
930c1d6a	91
4e76dbd7	92	my $raw = PVE::Cluster::get_node_kv("ceph-$type");
b1a3af5c TL	93	my $res = {
	94	map { $_ => eval { decode_json($raw->{$_}) } } keys $raw->%*
	95	};
4e76dbd7 DC	96
	97	return $res;
	98	}
	99
27439be6 DC	100	sub ceph_service_cmd {
	101	my ($action, $service) = @_;
	102
7e98f79e	103	if ($service && $service =~ m/^(mon\|osd\|mds\|mgr\|radosgw)(\.(${\SERVICE_REGEX}))?$/) {
bba5c712	104	$service = defined($3) ? "ceph-$1\@$3" : "ceph-$1.target";
27439be6	105	} else {
bba5c712	106	$service = "ceph.target";
27439be6	107	}
bba5c712 TL	108
bba5c712 TL	109	run_command(['/bin/systemctl', $action, $service]);
27439be6 DC	110	}
27439be6 DC	111
d5373b7d DC	112	sub get_services_info {
	113	my ($type, $cfg, $rados) = @_;
	114
	115	my $result = {};
	116	my $services = get_cluster_service($type);
	117
	118	foreach my $host (sort keys %$services) {
9ecb3d10 TL	119	foreach my $id (sort keys %{$services->{$host}}) {
	120	my $service = $result->{$id} = $services->{$host}->{$id};
	121	$service->{host} = $host;
	122	$service->{name} = $id;
	123	$service->{state} = 'unknown';
	124	if ($service->{service}) {
	125	$service->{state} = 'stopped';
d5373b7d DC	126	}
	127	}
	128	}
	129
	130	if (!$cfg) {
	131	$cfg = cfs_read_file('ceph.conf');
	132	}
	133
	134	foreach my $section (keys %$cfg) {
	135	my $d = $cfg->{$section};
	136	if ($section =~ m/^$type\.(\S+)$/) {
	137	my $id = $1;
9ecb3d10	138	my $service = $result->{$id};
342de4e7	139	my $addr = $d->{"${type}_addr"} // $d->{public_addr} // $d->{host};
9ecb3d10 TL	140	$service->{name} //= $id;
	141	$service->{addr} //= $addr;
	142	$service->{state} //= 'unknown';
	143	$service->{host} //= $d->{host};
d5373b7d DC	144	}
	145	}
	146
	147	if (!$rados) {
e7e61576	148	return $result;
d5373b7d	149	}
e7e61576	150
d5373b7d	151	my $metadata = $rados->mon_command({ prefix => "$type metadata" });
9ecb3d10	152	foreach my $info (@$metadata) {
46fb9c50 DC	153	my $id = $info->{name} // $info->{id};
46fb9c50 DC	154	my $service = $result->{$id};
9ecb3d10 TL	155	$service->{ceph_version_short} = $info->{ceph_version_short};
	156	$service->{ceph_version} = $info->{ceph_version};
	157	$service->{host} //= $info->{hostname};
	158	$service->{addr} //= $info->{addr};
d5373b7d DC	159	}
	160
	161	return $result;
	162	}
	163
27439be6 DC	164	# MDS
	165
	166	sub list_local_mds_ids {
	167	my $mds_list = [];
	168	my $ceph_mds_data_dir = PVE::Ceph::Tools::get_config('ceph_mds_data_dir');
	169	my $ccname = PVE::Ceph::Tools::get_config('ccname');
	170
	171	PVE::Tools::dir_glob_foreach($ceph_mds_data_dir, qr/$ccname-(\S+)/, sub {
	172	my (undef, $mds_id) = @_;
	173	push @$mds_list, $mds_id;
	174	});
	175
	176	return $mds_list;
	177	}
	178
	179	sub get_cluster_mds_state {
	180	my ($rados) = @_;
	181
	182	my $mds_state = {};
	183
	184	if (!defined($rados)) {
	185	$rados = PVE::RADOS->new();
	186	}
	187
	188	my $add_state = sub {
c51db18c	189	my ($mds, $fsname) = @_;
27439be6 DC	190
	191	my $state = {};
	192	$state->{addr} = $mds->{addr};
	193	$state->{rank} = $mds->{rank};
	194	$state->{standby_replay} = $mds->{standby_replay} ? 1 : 0;
	195	$state->{state} = $mds->{state};
c51db18c	196	$state->{fs_name} = $fsname;
27439be6 DC	197
	198	$mds_state->{$mds->{name}} = $state;
	199	};
	200
	201	my $mds_dump = $rados->mon_command({ prefix => 'mds stat' });
	202	my $fsmap = $mds_dump->{fsmap};
	203
	204
	205	foreach my $mds (@{$fsmap->{standbys}}) {
	206	$add_state->($mds);
	207	}
	208
c51db18c DC	209	for my $fs_info (@{$fsmap->{filesystems}}) {
c51db18c DC	210	my $active_mds = $fs_info->{mdsmap}->{info};
27439be6	211
c51db18c DC	212	# normally there's only one active MDS, but we can have multiple active for
	213	# different ranks (e.g., different cephs path hierarchy). So just add all.
	214	foreach my $mds (values %$active_mds) {
	215	$add_state->($mds, $fs_info->{mdsmap}->{fs_name});
	216	}
27439be6 DC	217	}
	218
	219	return $mds_state;
	220	}
	221
352a0e5c DC	222	sub is_mds_active {
352a0e5c DC	223	my ($rados, $fs_name) = @_;
27439be6 DC	224
	225	if (!defined($rados)) {
	226	$rados = PVE::RADOS->new();
	227	}
	228
	229	my $mds_dump = $rados->mon_command({ prefix => 'mds stat' });
352a0e5c	230	my $fsmap = $mds_dump->{fsmap}->{filesystems};
27439be6	231
352a0e5c	232	if (!($fsmap && scalar(@$fsmap) > 0)) {
27439be6 DC	233	return undef;
27439be6 DC	234	}
352a0e5c DC	235	for my $fs (@$fsmap) {
352a0e5c DC	236	next if defined($fs_name) && $fs->{mdsmap}->{fs_name} ne $fs_name;
27439be6	237
352a0e5c DC	238	my $active_mds = $fs->{mdsmap}->{info};
	239	for my $mds (values %$active_mds) {
	240	return 1 if $mds->{state} eq 'up:active';
	241	}
27439be6 DC	242	}
	243
	244	return 0;
	245	}
	246
	247	sub create_mds {
	248	my ($id, $rados) = @_;
	249
	250	# `ceph fs status` fails with numeric only ID.
	251	die "ID: $id, numeric only IDs are not supported\n"
	252	if $id =~ /^\d+$/;
	253
	254	if (!defined($rados)) {
	255	$rados = PVE::RADOS->new();
	256	}
	257
	258	my $ccname = PVE::Ceph::Tools::get_config('ccname');
	259	my $service_dir = "/var/lib/ceph/mds/$ccname-$id";
	260	my $service_keyring = "$service_dir/keyring";
	261	my $service_name = "mds.$id";
	262
	263	die "ceph MDS directory '$service_dir' already exists\n"
	264	if -d $service_dir;
	265
	266	print "creating MDS directory '$service_dir'\n";
	267	eval { File::Path::mkpath($service_dir) };
	268	my $err = $@;
	269	die "creation MDS directory '$service_dir' failed\n" if $err;
	270
	271	# http://docs.ceph.com/docs/luminous/install/manual-deployment/#adding-mds
	272	my $priv = [
	273	mon => 'allow profile mds',
	274	osd => 'allow rwx',
	275	mds => 'allow *',
	276	];
	277
	278	print "creating keys for '$service_name'\n";
	279	my $output = $rados->mon_command({
	280	prefix => 'auth get-or-create',
	281	entity => $service_name,
	282	caps => $priv,
	283	format => 'plain',
	284	});
	285
	286	PVE::Tools::file_set_contents($service_keyring, $output);
	287
	288	print "setting ceph as owner for service directory\n";
	289	run_command(["chown", 'ceph:ceph', '-R', $service_dir]);
	290
	291	print "enabling service 'ceph-mds\@$id.service'\n";
	292	ceph_service_cmd('enable', $service_name);
	293	print "starting service 'ceph-mds\@$id.service'\n";
	294	ceph_service_cmd('start', $service_name);
	295
4e76dbd7 DC	296	broadcast_ceph_services();
4e76dbd7 DC	297
27439be6 DC	298	return undef;
	299	};
	300
	301	sub destroy_mds {
	302	my ($id, $rados) = @_;
	303
	304	if (!defined($rados)) {
	305	$rados = PVE::RADOS->new();
	306	}
	307
	308	my $ccname = PVE::Ceph::Tools::get_config('ccname');
	309
	310	my $service_name = "mds.$id";
	311	my $service_dir = "/var/lib/ceph/mds/$ccname-$id";
	312
	313	print "disabling service 'ceph-mds\@$id.service'\n";
	314	ceph_service_cmd('disable', $service_name);
	315	print "stopping service 'ceph-mds\@$id.service'\n";
	316	ceph_service_cmd('stop', $service_name);
	317
	318	if (-d $service_dir) {
	319	print "removing ceph-mds directory '$service_dir'\n";
	320	File::Path::remove_tree($service_dir);
	321	} else {
	322	warn "cannot cleanup MDS $id directory, '$service_dir' not found\n"
	323	}
	324
	325	print "removing ceph auth for '$service_name'\n";
	326	$rados->mon_command({
	327	prefix => 'auth del',
	328	entity => $service_name,
	329	format => 'plain'
	330	});
	331
4e76dbd7 DC	332	broadcast_ceph_services();
4e76dbd7 DC	333
27439be6 DC	334	return undef;
	335	};
	336
be7edba1 DC	337	# MGR
	338
	339	sub create_mgr {
	340	my ($id, $rados) = @_;
	341
	342	my $clustername = PVE::Ceph::Tools::get_config('ccname');
	343	my $mgrdir = "/var/lib/ceph/mgr/$clustername-$id";
	344	my $mgrkeyring = "$mgrdir/keyring";
	345	my $mgrname = "mgr.$id";
	346
51498a26	347	die "ceph manager directory '$mgrdir' already exists\n" if -d $mgrdir;
be7edba1 DC	348
	349	print "creating manager directory '$mgrdir'\n";
	350	mkdir $mgrdir;
	351	print "creating keys for '$mgrname'\n";
51498a26 TL	352	my $output = $rados->mon_command({
	353	prefix => 'auth get-or-create',
	354	entity => $mgrname,
	355	caps => [
	356	mon => 'allow profile mgr',
	357	osd => 'allow *',
	358	mds => 'allow *',
	359	],
	360	format => 'plain'
	361	});
be7edba1 DC	362	PVE::Tools::file_set_contents($mgrkeyring, $output);
	363
	364	print "setting owner for directory\n";
	365	run_command(["chown", 'ceph:ceph', '-R', $mgrdir]);
	366
	367	print "enabling service 'ceph-mgr\@$id.service'\n";
	368	ceph_service_cmd('enable', $mgrname);
	369	print "starting service 'ceph-mgr\@$id.service'\n";
	370	ceph_service_cmd('start', $mgrname);
4e76dbd7 DC	371
	372	broadcast_ceph_services();
	373
	374	return undef;
be7edba1 DC	375	}
	376
	377	sub destroy_mgr {
b5215730	378	my ($mgrid, $rados) = @_;
be7edba1 DC	379
	380	my $clustername = PVE::Ceph::Tools::get_config('ccname');
	381	my $mgrname = "mgr.$mgrid";
	382	my $mgrdir = "/var/lib/ceph/mgr/$clustername-$mgrid";
	383
	384	die "ceph manager directory '$mgrdir' not found\n"
	385	if ! -d $mgrdir;
	386
	387	print "disabling service 'ceph-mgr\@$mgrid.service'\n";
	388	ceph_service_cmd('disable', $mgrname);
	389	print "stopping service 'ceph-mgr\@$mgrid.service'\n";
	390	ceph_service_cmd('stop', $mgrname);
	391
	392	print "removing manager directory '$mgrdir'\n";
	393	File::Path::remove_tree($mgrdir);
4e76dbd7	394
b5215730 DC	395	print "removing authkeys for $mgrname\n";
	396	if (!$rados) {
	397	$rados = PVE::RADOS->new();
	398	}
	399
	400	$rados->mon_command({ prefix => 'auth del', entity => "$mgrname" });
	401
4e76dbd7 DC	402	broadcast_ceph_services();
	403
	404	return undef;
be7edba1 DC	405	}
be7edba1 DC	406
27439be6	407	1;