[pve-manager.git] / PVE / Ceph / Services.pm

package PVE::Ceph::Services;

use strict;
use warnings;

use PVE::Ceph::Tools;
use PVE::Cluster qw(cfs_read_file);
use PVE::Tools qw(run_command);
use PVE::RADOS;

use JSON;
use File::Path;

# checks /etc/systemd/system/ceph-* to list all services, even if not running
# also checks /var/lib/ceph/$type
sub get_local_services {
    my $res = {};

    for my $type (qw(mds mgr mon)) {
	$res->{$type} = {};

	my $path = "/etc/systemd/system/ceph-$type.target.wants";
	my $regex = "ceph-$type\@(.*)\.service";
	PVE::Tools::dir_glob_foreach($path, $regex, sub {
	    my (undef, $id) = @_;
	    $res->{$type}->{$id}->{service} = 1;
	});

	$path = "/var/lib/ceph/$type";
	$regex = "([^-]+)-(.*)";
	PVE::Tools::dir_glob_foreach($path, $regex, sub {
	    my (undef, $clustername, $id) = @_;
	    $res->{$type}->{$id}->{direxists} = 1;
	});
    }
    return $res;
}

sub broadcast_ceph_services {
    my $services = get_local_services();

    for my $type (keys %$services) {
	my $data = encode_json($services->{$type});
	PVE::Cluster::broadcast_node_kv("ceph-$type", $data);
    }
}

sub get_cluster_service {
    my ($type) = @_;
    PVE::Cluster::cfs_update();
    my $raw = PVE::Cluster::get_node_kv("ceph-$type");
    my $res = {};

    for my $host (keys %$raw) {
	$res->{$host} = eval { decode_json($raw->{$host}) };
    }

    return $res;
}

sub ceph_service_cmd {
    my ($action, $service) = @_;

    my $pve_ceph_cfgpath = PVE::Ceph::Tools::get_config('pve_ceph_cfgpath');
    if ($service && $service =~ m/^(mon|osd|mds|mgr|radosgw)(\.([A-Za-z0-9\-]{1,32}))?$/) {
	$service = defined($3) ? "ceph-$1\@$3" : "ceph-$1.target";
    } else {
	$service = "ceph.target";
    }

    run_command(['/bin/systemctl', $action, $service]);
}

sub get_services_info {
    my ($type, $cfg, $rados) = @_;

    my $result = {};
    my $services = get_cluster_service($type);

    foreach my $host (sort keys %$services) {
	foreach my $id (sort keys %{$services->{$host}}) {
	    my $service = $result->{$id} = $services->{$host}->{$id};
	    $service->{host} = $host;
	    $service->{name} = $id;
	    $service->{state} = 'unknown';
	    if ($service->{service}) {
		$service->{state} = 'stopped';
	    }
	}
    }

    if (!$cfg) {
	$cfg = cfs_read_file('ceph.conf');
    }

    foreach my $section (keys %$cfg) {
	my $d = $cfg->{$section};
	if ($section =~ m/^$type\.(\S+)$/) {
	    my $id = $1;
	    my $service = $result->{$id};
	    my $addr = $d->{"$type addr"} // $d->{"${type}_addr"} // $d->{host};
	    $service->{name} //= $id;
	    $service->{addr} //= $addr;
	    $service->{state} //= 'unknown';
	    $service->{host} //= $d->{host};
	}
    }

    if (!$rados) {
	$rados = PVE::RADOS->new();
    }
    my $metadata = $rados->mon_command({ prefix => "$type metadata" });
    foreach my $info (@$metadata) {
	my $service = $result->{$info->{name}};
	$service->{ceph_version_short} = $info->{ceph_version_short};
	$service->{ceph_version} = $info->{ceph_version};
	$service->{host} //= $info->{hostname};
	$service->{addr} //= $info->{addr};
    }

    return $result;
}

# MDS

sub list_local_mds_ids {
    my $mds_list = [];
    my $ceph_mds_data_dir = PVE::Ceph::Tools::get_config('ceph_mds_data_dir');
    my $ccname = PVE::Ceph::Tools::get_config('ccname');

    PVE::Tools::dir_glob_foreach($ceph_mds_data_dir, qr/$ccname-(\S+)/, sub {
	my (undef, $mds_id) = @_;
	push @$mds_list, $mds_id;
    });

    return $mds_list;
}

sub get_cluster_mds_state {
    my ($rados) = @_;

    my $mds_state = {};

    if (!defined($rados)) {
	$rados = PVE::RADOS->new();
    }

    my $add_state = sub {
	my ($mds) = @_;

	my $state = {};
	$state->{addr} = $mds->{addr};
	$state->{rank} = $mds->{rank};
	$state->{standby_replay} = $mds->{standby_replay} ? 1 : 0;
	$state->{state} = $mds->{state};

	$mds_state->{$mds->{name}} = $state;
    };

    my $mds_dump = $rados->mon_command({ prefix => 'mds stat' });
    my $fsmap = $mds_dump->{fsmap};


    foreach my $mds (@{$fsmap->{standbys}}) {
	$add_state->($mds);
    }

    my $fs_info = $fsmap->{filesystems}->[0];
    my $active_mds = $fs_info->{mdsmap}->{info};

    # normally there's only one active MDS, but we can have multiple active for
    # different ranks (e.g., different cephs path hierarchy). So just add all.
    foreach my $mds (values %$active_mds) {
	$add_state->($mds);
    }

    return $mds_state;
}

sub is_any_mds_active {
    my ($rados) = @_;

    if (!defined($rados)) {
	$rados = PVE::RADOS->new();
    }

    my $mds_dump = $rados->mon_command({ prefix => 'mds stat' });
    my $fs = $mds_dump->{fsmap}->{filesystems};

    if (!($fs && scalar(@$fs) > 0)) {
	return undef;
    }
    my $active_mds = $fs->[0]->{mdsmap}->{info};

    for my $mds (values %$active_mds) {
	return 1 if $mds->{state} eq 'up:active';
    }

    return 0;
}

sub create_mds {
    my ($id, $rados) = @_;

    # `ceph fs status` fails with numeric only ID.
    die "ID: $id, numeric only IDs are not supported\n"
	if $id =~ /^\d+$/;

    if (!defined($rados)) {
	$rados = PVE::RADOS->new();
    }

    my $ccname = PVE::Ceph::Tools::get_config('ccname');
    my $service_dir = "/var/lib/ceph/mds/$ccname-$id";
    my $service_keyring = "$service_dir/keyring";
    my $service_name = "mds.$id";

    die "ceph MDS directory '$service_dir' already exists\n"
	if -d $service_dir;

    print "creating MDS directory '$service_dir'\n";
    eval { File::Path::mkpath($service_dir) };
    my $err = $@;
    die "creation MDS directory '$service_dir' failed\n" if $err;

    # http://docs.ceph.com/docs/luminous/install/manual-deployment/#adding-mds
    my $priv = [
	mon => 'allow profile mds',
	osd => 'allow rwx',
	mds => 'allow *',
    ];

    print "creating keys for '$service_name'\n";
    my $output = $rados->mon_command({
	prefix => 'auth get-or-create',
	entity => $service_name,
	caps => $priv,
	format => 'plain',
    });

    PVE::Tools::file_set_contents($service_keyring, $output);

    print "setting ceph as owner for service directory\n";
    run_command(["chown", 'ceph:ceph', '-R', $service_dir]);

    print "enabling service 'ceph-mds\@$id.service'\n";
    ceph_service_cmd('enable', $service_name);
    print "starting service 'ceph-mds\@$id.service'\n";
    ceph_service_cmd('start', $service_name);

    broadcast_ceph_services();

    return undef;
};

sub destroy_mds {
    my ($id, $rados) = @_;

    if (!defined($rados)) {
	$rados = PVE::RADOS->new();
    }

    my $ccname = PVE::Ceph::Tools::get_config('ccname');

    my $service_name = "mds.$id";
    my $service_dir = "/var/lib/ceph/mds/$ccname-$id";

    print "disabling service 'ceph-mds\@$id.service'\n";
    ceph_service_cmd('disable', $service_name);
    print "stopping service 'ceph-mds\@$id.service'\n";
    ceph_service_cmd('stop', $service_name);

    if (-d $service_dir) {
	print "removing ceph-mds directory '$service_dir'\n";
	File::Path::remove_tree($service_dir);
    } else {
	warn "cannot cleanup MDS $id directory, '$service_dir' not found\n"
    }

    print "removing ceph auth for '$service_name'\n";
    $rados->mon_command({
	    prefix => 'auth del',
	    entity => $service_name,
	    format => 'plain'
	});

    broadcast_ceph_services();

    return undef;
};

# MGR

sub create_mgr {
    my ($id, $rados) = @_;

    my $clustername = PVE::Ceph::Tools::get_config('ccname');
    my $mgrdir = "/var/lib/ceph/mgr/$clustername-$id";
    my $mgrkeyring = "$mgrdir/keyring";
    my $mgrname = "mgr.$id";

    die "ceph manager directory '$mgrdir' already exists\n"
	if -d $mgrdir;

    print "creating manager directory '$mgrdir'\n";
    mkdir $mgrdir;
    print "creating keys for '$mgrname'\n";
    my $output = $rados->mon_command({ prefix => 'auth get-or-create',
				       entity => $mgrname,
				       caps => [
					   mon => 'allow profile mgr',
					   osd => 'allow *',
					   mds => 'allow *',
				       ],
				       format => 'plain'});
    PVE::Tools::file_set_contents($mgrkeyring, $output);

    print "setting owner for directory\n";
    run_command(["chown", 'ceph:ceph', '-R', $mgrdir]);

    print "enabling service 'ceph-mgr\@$id.service'\n";
    ceph_service_cmd('enable', $mgrname);
    print "starting service 'ceph-mgr\@$id.service'\n";
    ceph_service_cmd('start', $mgrname);

    broadcast_ceph_services();

    return undef;
}

sub destroy_mgr {
    my ($mgrid) = @_;

    my $clustername = PVE::Ceph::Tools::get_config('ccname');
    my $mgrname = "mgr.$mgrid";
    my $mgrdir = "/var/lib/ceph/mgr/$clustername-$mgrid";

    die "ceph manager directory '$mgrdir' not found\n"
	if ! -d $mgrdir;

    print "disabling service 'ceph-mgr\@$mgrid.service'\n";
    ceph_service_cmd('disable', $mgrname);
    print "stopping service 'ceph-mgr\@$mgrid.service'\n";
    ceph_service_cmd('stop', $mgrname);

    print "removing manager directory '$mgrdir'\n";
    File::Path::remove_tree($mgrdir);

    broadcast_ceph_services();

    return undef;
}

1;
Commit	Line	Data
27439be6 DC	1	package PVE::Ceph::Services;
	2
	3	use strict;
	4	use warnings;
	5
	6	use PVE::Ceph::Tools;
d5373b7d	7	use PVE::Cluster qw(cfs_read_file);
27439be6 DC	8	use PVE::Tools qw(run_command);
	9	use PVE::RADOS;
	10
4e76dbd7	11	use JSON;
27439be6 DC	12	use File::Path;
27439be6 DC	13
ac4f971d	14	# checks /etc/systemd/system/ceph-* to list all services, even if not running
74628668	15	# also checks /var/lib/ceph/$type
23b20ae4 DC	16	sub get_local_services {
23b20ae4 DC	17	my $res = {};
ac4f971d	18
23b20ae4 DC	19	for my $type (qw(mds mgr mon)) {
23b20ae4 DC	20	$res->{$type} = {};
ac4f971d	21
23b20ae4 DC	22	my $path = "/etc/systemd/system/ceph-$type.target.wants";
	23	my $regex = "ceph-$type\@(.*)\.service";
	24	PVE::Tools::dir_glob_foreach($path, $regex, sub {
	25	my (undef, $id) = @_;
74628668 DC	26	$res->{$type}->{$id}->{service} = 1;
	27	});
	28
	29	$path = "/var/lib/ceph/$type";
	30	$regex = "([^-]+)-(.*)";
	31	PVE::Tools::dir_glob_foreach($path, $regex, sub {
	32	my (undef, $clustername, $id) = @_;
	33	$res->{$type}->{$id}->{direxists} = 1;
23b20ae4 DC	34	});
	35	}
	36	return $res;
	37	}
	38
4e76dbd7 DC	39	sub broadcast_ceph_services {
	40	my $services = get_local_services();
	41
	42	for my $type (keys %$services) {
	43	my $data = encode_json($services->{$type});
	44	PVE::Cluster::broadcast_node_kv("ceph-$type", $data);
	45	}
	46	}
	47
	48	sub get_cluster_service {
	49	my ($type) = @_;
	50	PVE::Cluster::cfs_update();
	51	my $raw = PVE::Cluster::get_node_kv("ceph-$type");
	52	my $res = {};
	53
	54	for my $host (keys %$raw) {
	55	$res->{$host} = eval { decode_json($raw->{$host}) };
	56	}
	57
	58	return $res;
	59	}
	60
27439be6 DC	61	sub ceph_service_cmd {
	62	my ($action, $service) = @_;
	63
	64	my $pve_ceph_cfgpath = PVE::Ceph::Tools::get_config('pve_ceph_cfgpath');
bba5c712 TL	65	if ($service && $service =~ m/^(mon\|osd\|mds\|mgr\|radosgw)(\.([A-Za-z0-9\-]{1,32}))?$/) {
bba5c712 TL	66	$service = defined($3) ? "ceph-$1\@$3" : "ceph-$1.target";
27439be6	67	} else {
bba5c712	68	$service = "ceph.target";
27439be6	69	}
bba5c712 TL	70
bba5c712 TL	71	run_command(['/bin/systemctl', $action, $service]);
27439be6 DC	72	}
27439be6 DC	73
d5373b7d DC	74	sub get_services_info {
	75	my ($type, $cfg, $rados) = @_;
	76
	77	my $result = {};
	78	my $services = get_cluster_service($type);
	79
	80	foreach my $host (sort keys %$services) {
9ecb3d10 TL	81	foreach my $id (sort keys %{$services->{$host}}) {
	82	my $service = $result->{$id} = $services->{$host}->{$id};
	83	$service->{host} = $host;
	84	$service->{name} = $id;
	85	$service->{state} = 'unknown';
	86	if ($service->{service}) {
	87	$service->{state} = 'stopped';
d5373b7d DC	88	}
	89	}
	90	}
	91
	92	if (!$cfg) {
	93	$cfg = cfs_read_file('ceph.conf');
	94	}
	95
	96	foreach my $section (keys %$cfg) {
	97	my $d = $cfg->{$section};
	98	if ($section =~ m/^$type\.(\S+)$/) {
	99	my $id = $1;
9ecb3d10	100	my $service = $result->{$id};
d5373b7d	101	my $addr = $d->{"$type addr"} // $d->{"${type}_addr"} // $d->{host};
9ecb3d10 TL	102	$service->{name} //= $id;
	103	$service->{addr} //= $addr;
	104	$service->{state} //= 'unknown';
	105	$service->{host} //= $d->{host};
d5373b7d DC	106	}
	107	}
	108
	109	if (!$rados) {
	110	$rados = PVE::RADOS->new();
	111	}
	112	my $metadata = $rados->mon_command({ prefix => "$type metadata" });
9ecb3d10 TL	113	foreach my $info (@$metadata) {
	114	my $service = $result->{$info->{name}};
	115	$service->{ceph_version_short} = $info->{ceph_version_short};
	116	$service->{ceph_version} = $info->{ceph_version};
	117	$service->{host} //= $info->{hostname};
	118	$service->{addr} //= $info->{addr};
d5373b7d DC	119	}
	120
	121	return $result;
	122	}
	123
27439be6 DC	124	# MDS
	125
	126	sub list_local_mds_ids {
	127	my $mds_list = [];
	128	my $ceph_mds_data_dir = PVE::Ceph::Tools::get_config('ceph_mds_data_dir');
	129	my $ccname = PVE::Ceph::Tools::get_config('ccname');
	130
	131	PVE::Tools::dir_glob_foreach($ceph_mds_data_dir, qr/$ccname-(\S+)/, sub {
	132	my (undef, $mds_id) = @_;
	133	push @$mds_list, $mds_id;
	134	});
	135
	136	return $mds_list;
	137	}
	138
	139	sub get_cluster_mds_state {
	140	my ($rados) = @_;
	141
	142	my $mds_state = {};
	143
	144	if (!defined($rados)) {
	145	$rados = PVE::RADOS->new();
	146	}
	147
	148	my $add_state = sub {
	149	my ($mds) = @_;
	150
	151	my $state = {};
	152	$state->{addr} = $mds->{addr};
	153	$state->{rank} = $mds->{rank};
	154	$state->{standby_replay} = $mds->{standby_replay} ? 1 : 0;
	155	$state->{state} = $mds->{state};
	156
	157	$mds_state->{$mds->{name}} = $state;
	158	};
	159
	160	my $mds_dump = $rados->mon_command({ prefix => 'mds stat' });
	161	my $fsmap = $mds_dump->{fsmap};
	162
	163
	164	foreach my $mds (@{$fsmap->{standbys}}) {
	165	$add_state->($mds);
	166	}
	167
	168	my $fs_info = $fsmap->{filesystems}->[0];
	169	my $active_mds = $fs_info->{mdsmap}->{info};
	170
	171	# normally there's only one active MDS, but we can have multiple active for
	172	# different ranks (e.g., different cephs path hierarchy). So just add all.
	173	foreach my $mds (values %$active_mds) {
	174	$add_state->($mds);
	175	}
	176
	177	return $mds_state;
	178	}
	179
	180	sub is_any_mds_active {
	181	my ($rados) = @_;
	182
	183	if (!defined($rados)) {
	184	$rados = PVE::RADOS->new();
	185	}
	186
	187	my $mds_dump = $rados->mon_command({ prefix => 'mds stat' });
188	my $fs = $mds_dump->{fsmap}->{filesystems};
189
190	if (!($fs && scalar(@$fs) > 0)) {
191	return undef;
192	}
193	my $active_mds = $fs->[0]->{mdsmap}->{info};
194
195	for my $mds (values %$active_mds) {
196	return 1 if $mds->{state} eq 'up:active';
197	}
198
199	return 0;
200	}
201
202	sub create_mds {
203	my ($id, $rados) = @_;
204
205	# `ceph fs status` fails with numeric only ID.
206	die "ID: $id, numeric only IDs are not supported\n"
207	if $id =~ /^\d+$/;
208
209	if (!defined($rados)) {
210	$rados = PVE::RADOS->new();
211	}
212
213	my $ccname = PVE::Ceph::Tools::get_config('ccname');
214	my $service_dir = "/var/lib/ceph/mds/$ccname-$id";
215	my $service_keyring = "$service_dir/keyring";
216	my $service_name = "mds.$id";
217
218	die "ceph MDS directory '$service_dir' already exists\n"
219	if -d $service_dir;
220
221	print "creating MDS directory '$service_dir'\n";
222	eval { File::Path::mkpath($service_dir) };
223	my $err = $@;
224	die "creation MDS directory '$service_dir' failed\n" if $err;
225
226	# http://docs.ceph.com/docs/luminous/install/manual-deployment/#adding-mds
227	my $priv = [
228	mon => 'allow profile mds',
229	osd => 'allow rwx',
230	mds => 'allow *',
231	];
232
233	print "creating keys for '$service_name'\n";
234	my $output = $rados->mon_command({
235	prefix => 'auth get-or-create',
236	entity => $service_name,
237	caps => $priv,
238	format => 'plain',
239	});
240
241	PVE::Tools::file_set_contents($service_keyring, $output);
242
243	print "setting ceph as owner for service directory\n";
244	run_command(["chown", 'ceph:ceph', '-R', $service_dir]);
245
246	print "enabling service 'ceph-mds\@$id.service'\n";
247	ceph_service_cmd('enable', $service_name);
248	print "starting service 'ceph-mds\@$id.service'\n";
249	ceph_service_cmd('start', $service_name);
250
4e76dbd7 DC	251	broadcast_ceph_services();
4e76dbd7 DC	252
27439be6 DC	253	return undef;
	254	};
	255
	256	sub destroy_mds {
	257	my ($id, $rados) = @_;
	258
	259	if (!defined($rados)) {
	260	$rados = PVE::RADOS->new();
	261	}
	262
	263	my $ccname = PVE::Ceph::Tools::get_config('ccname');
	264
	265	my $service_name = "mds.$id";
	266	my $service_dir = "/var/lib/ceph/mds/$ccname-$id";
	267
	268	print "disabling service 'ceph-mds\@$id.service'\n";
	269	ceph_service_cmd('disable', $service_name);
	270	print "stopping service 'ceph-mds\@$id.service'\n";
	271	ceph_service_cmd('stop', $service_name);
	272
	273	if (-d $service_dir) {
	274	print "removing ceph-mds directory '$service_dir'\n";
	275	File::Path::remove_tree($service_dir);
	276	} else {
	277	warn "cannot cleanup MDS $id directory, '$service_dir' not found\n"
	278	}
	279
	280	print "removing ceph auth for '$service_name'\n";
	281	$rados->mon_command({
	282	prefix => 'auth del',
	283	entity => $service_name,
	284	format => 'plain'
	285	});
	286
4e76dbd7 DC	287	broadcast_ceph_services();
4e76dbd7 DC	288
27439be6 DC	289	return undef;
	290	};
	291
be7edba1 DC	292	# MGR
	293
	294	sub create_mgr {
	295	my ($id, $rados) = @_;
	296
	297	my $clustername = PVE::Ceph::Tools::get_config('ccname');
	298	my $mgrdir = "/var/lib/ceph/mgr/$clustername-$id";
	299	my $mgrkeyring = "$mgrdir/keyring";
	300	my $mgrname = "mgr.$id";
	301
	302	die "ceph manager directory '$mgrdir' already exists\n"
	303	if -d $mgrdir;
	304
	305	print "creating manager directory '$mgrdir'\n";
	306	mkdir $mgrdir;
	307	print "creating keys for '$mgrname'\n";
	308	my $output = $rados->mon_command({ prefix => 'auth get-or-create',
	309	entity => $mgrname,
	310	caps => [
	311	mon => 'allow profile mgr',
	312	osd => 'allow *',
	313	mds => 'allow *',
	314	],
	315	format => 'plain'});
	316	PVE::Tools::file_set_contents($mgrkeyring, $output);
	317
	318	print "setting owner for directory\n";
	319	run_command(["chown", 'ceph:ceph', '-R', $mgrdir]);
	320
	321	print "enabling service 'ceph-mgr\@$id.service'\n";
	322	ceph_service_cmd('enable', $mgrname);
	323	print "starting service 'ceph-mgr\@$id.service'\n";
	324	ceph_service_cmd('start', $mgrname);
4e76dbd7 DC	325
	326	broadcast_ceph_services();
	327
	328	return undef;
be7edba1 DC	329	}
	330
	331	sub destroy_mgr {
	332	my ($mgrid) = @_;
	333
	334	my $clustername = PVE::Ceph::Tools::get_config('ccname');
	335	my $mgrname = "mgr.$mgrid";
	336	my $mgrdir = "/var/lib/ceph/mgr/$clustername-$mgrid";
	337
	338	die "ceph manager directory '$mgrdir' not found\n"
	339	if ! -d $mgrdir;
	340
	341	print "disabling service 'ceph-mgr\@$mgrid.service'\n";
	342	ceph_service_cmd('disable', $mgrname);
	343	print "stopping service 'ceph-mgr\@$mgrid.service'\n";
	344	ceph_service_cmd('stop', $mgrname);
	345
	346	print "removing manager directory '$mgrdir'\n";
	347	File::Path::remove_tree($mgrdir);
4e76dbd7 DC	348
	349	broadcast_ceph_services();
	350
	351	return undef;
be7edba1 DC	352	}
be7edba1 DC	353
27439be6	354	1;