]>
git.proxmox.com Git - pve-manager.git/blob - PVE/Ceph/Services.pm
1 package PVE
::Ceph
::Services
;
7 use PVE
::Cluster
qw(cfs_read_file);
8 use PVE
::Tools
qw(run_command);
14 use constant SERVICE_REGEX
=> '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?';
16 # checks /etc/systemd/system/ceph-* to list all services, even if not running
17 # also checks /var/lib/ceph/$type
18 sub get_local_services
{
21 for my $type (qw(mds mgr mon)) {
24 my $path = "/etc/systemd/system/ceph-$type.target.wants";
25 my $regex = "ceph-$type\@(.*)\.service";
26 PVE
::Tools
::dir_glob_foreach
($path, $regex, sub {
28 $res->{$type}->{$id}->{service
} = 1;
31 $path = "/var/lib/ceph/$type";
32 $regex = "([^-]+)-(.*)";
33 PVE
::Tools
::dir_glob_foreach
($path, $regex, sub {
34 my (undef, $clustername, $id) = @_;
35 $res->{$type}->{$id}->{direxists
} = 1;
41 sub broadcast_ceph_services
{
42 my $services = get_local_services
();
44 for my $type (keys %$services) {
45 my $data = encode_json
($services->{$type});
46 PVE
::Cluster
::broadcast_node_kv
("ceph-$type", $data);
50 sub broadcast_ceph_versions
{
51 my ($version, $buildcommit, $vers_parts) = PVE
::Ceph
::Tools
::get_local_version
(1);
54 my $nodename = PVE
::INotify
::nodename
();
55 my $old = PVE
::Cluster
::get_node_kv
("ceph-versions", $nodename);
56 if (defined($old->{$nodename})) {
57 $old = eval { decode_json
($old->{$nodename}) };
58 warn $@ if $@; # should not happen
59 if (defined($old) && $old->{buildcommit
} eq $buildcommit && $old->{version
}->{str
} eq $version) {
60 return; # up to date, nothing to do so avoid (not exactly cheap) broadcast
63 # FIXME: remove with 8.0 (or 7.2, its not _that_ bad) - for backward compat only
64 PVE
::Cluster
::broadcast_node_kv
("ceph-version", $version);
71 buildcommit
=> $buildcommit,
73 PVE
::Cluster
::broadcast_node_kv
("ceph-versions", encode_json
($node_versions));
77 sub get_ceph_versions
{
80 if (defined(my $versions = PVE
::Cluster
::get_node_kv
("ceph-versions"))) {
82 map { eval { $_ => decode_json
($versions->{$_}) } } keys %$versions
89 sub get_cluster_service
{
92 my $raw = PVE
::Cluster
::get_node_kv
("ceph-$type");
94 map { $_ => eval { decode_json
($raw->{$_}) } } keys $raw->%*
100 sub ceph_service_cmd
{
101 my ($action, $service) = @_;
103 if ($service && $service =~ m/^(mon|osd|mds|mgr|radosgw)(\.(${\SERVICE_REGEX}))?$/) {
104 $service = defined($3) ?
"ceph-$1\@$3" : "ceph-$1.target";
106 $service = "ceph.target";
109 run_command
(['/bin/systemctl', $action, $service]);
112 sub get_services_info
{
113 my ($type, $cfg, $rados) = @_;
116 my $services = get_cluster_service
($type);
118 foreach my $host (sort keys %$services) {
119 foreach my $id (sort keys %{$services->{$host}}) {
120 my $service = $result->{$id} = $services->{$host}->{$id};
121 $service->{host
} = $host;
122 $service->{name
} = $id;
123 $service->{state} = 'unknown';
124 if ($service->{service
}) {
125 $service->{state} = 'stopped';
131 $cfg = cfs_read_file
('ceph.conf');
134 foreach my $section (keys %$cfg) {
135 my $d = $cfg->{$section};
136 if ($section =~ m/^$type\.(\S+)$/) {
138 my $service = $result->{$id};
139 my $addr = $d->{"${type}_addr"} // $d->{public_addr
} // $d->{host
};
140 $service->{name
} //= $id;
141 $service->{addr
} //= $addr;
142 $service->{state} //= 'unknown';
143 $service->{host
} //= $d->{host
};
151 my $metadata = $rados->mon_command({ prefix
=> "$type metadata" });
152 foreach my $info (@$metadata) {
153 my $id = $info->{name
} // $info->{id
};
154 my $service = $result->{$id};
155 $service->{ceph_version_short
} = $info->{ceph_version_short
};
156 $service->{ceph_version
} = $info->{ceph_version
};
157 $service->{host
} //= $info->{hostname
};
158 $service->{addr
} //= $info->{addr
};
166 sub list_local_mds_ids
{
168 my $ceph_mds_data_dir = PVE
::Ceph
::Tools
::get_config
('ceph_mds_data_dir');
169 my $ccname = PVE
::Ceph
::Tools
::get_config
('ccname');
171 PVE
::Tools
::dir_glob_foreach
($ceph_mds_data_dir, qr/$ccname-(\S+)/, sub {
172 my (undef, $mds_id) = @_;
173 push @$mds_list, $mds_id;
179 sub get_cluster_mds_state
{
184 if (!defined($rados)) {
185 $rados = PVE
::RADOS-
>new();
188 my $add_state = sub {
189 my ($mds, $fsname) = @_;
192 $state->{addr
} = $mds->{addr
};
193 $state->{rank
} = $mds->{rank
};
194 $state->{standby_replay
} = $mds->{standby_replay
} ?
1 : 0;
195 $state->{state} = $mds->{state};
196 $state->{fs_name
} = $fsname;
198 $mds_state->{$mds->{name
}} = $state;
201 my $mds_dump = $rados->mon_command({ prefix
=> 'mds stat' });
202 my $fsmap = $mds_dump->{fsmap
};
205 foreach my $mds (@{$fsmap->{standbys
}}) {
209 for my $fs_info (@{$fsmap->{filesystems
}}) {
210 my $active_mds = $fs_info->{mdsmap
}->{info
};
212 # normally there's only one active MDS, but we can have multiple active for
213 # different ranks (e.g., different cephs path hierarchy). So just add all.
214 foreach my $mds (values %$active_mds) {
215 $add_state->($mds, $fs_info->{mdsmap
}->{fs_name
});
223 my ($rados, $fs_name) = @_;
225 if (!defined($rados)) {
226 $rados = PVE
::RADOS-
>new();
229 my $mds_dump = $rados->mon_command({ prefix
=> 'mds stat' });
230 my $fsmap = $mds_dump->{fsmap
}->{filesystems
};
232 if (!($fsmap && scalar(@$fsmap) > 0)) {
235 for my $fs (@$fsmap) {
236 next if defined($fs_name) && $fs->{mdsmap
}->{fs_name
} ne $fs_name;
238 my $active_mds = $fs->{mdsmap
}->{info
};
239 for my $mds (values %$active_mds) {
240 return 1 if $mds->{state} eq 'up:active';
248 my ($id, $rados) = @_;
250 # `ceph fs status` fails with numeric only ID.
251 die "ID: $id, numeric only IDs are not supported\n"
254 if (!defined($rados)) {
255 $rados = PVE
::RADOS-
>new();
258 my $ccname = PVE
::Ceph
::Tools
::get_config
('ccname');
259 my $service_dir = "/var/lib/ceph/mds/$ccname-$id";
260 my $service_keyring = "$service_dir/keyring";
261 my $service_name = "mds.$id";
263 die "ceph MDS directory '$service_dir' already exists\n"
266 print "creating MDS directory '$service_dir'\n";
267 eval { File
::Path
::mkpath
($service_dir) };
269 die "creation MDS directory '$service_dir' failed\n" if $err;
271 # http://docs.ceph.com/docs/luminous/install/manual-deployment/#adding-mds
273 mon
=> 'allow profile mds',
278 print "creating keys for '$service_name'\n";
279 my $output = $rados->mon_command({
280 prefix
=> 'auth get-or-create',
281 entity
=> $service_name,
286 PVE
::Tools
::file_set_contents
($service_keyring, $output);
288 print "setting ceph as owner for service directory\n";
289 run_command
(["chown", 'ceph:ceph', '-R', $service_dir]);
291 print "enabling service 'ceph-mds\@$id.service'\n";
292 ceph_service_cmd
('enable', $service_name);
293 print "starting service 'ceph-mds\@$id.service'\n";
294 ceph_service_cmd
('start', $service_name);
296 broadcast_ceph_services
();
302 my ($id, $rados) = @_;
304 if (!defined($rados)) {
305 $rados = PVE
::RADOS-
>new();
308 my $ccname = PVE
::Ceph
::Tools
::get_config
('ccname');
310 my $service_name = "mds.$id";
311 my $service_dir = "/var/lib/ceph/mds/$ccname-$id";
313 print "disabling service 'ceph-mds\@$id.service'\n";
314 ceph_service_cmd
('disable', $service_name);
315 print "stopping service 'ceph-mds\@$id.service'\n";
316 ceph_service_cmd
('stop', $service_name);
318 if (-d
$service_dir) {
319 print "removing ceph-mds directory '$service_dir'\n";
320 File
::Path
::remove_tree
($service_dir);
322 warn "cannot cleanup MDS $id directory, '$service_dir' not found\n"
325 print "removing ceph auth for '$service_name'\n";
326 $rados->mon_command({
327 prefix
=> 'auth del',
328 entity
=> $service_name,
332 broadcast_ceph_services
();
340 my ($id, $rados) = @_;
342 my $clustername = PVE
::Ceph
::Tools
::get_config
('ccname');
343 my $mgrdir = "/var/lib/ceph/mgr/$clustername-$id";
344 my $mgrkeyring = "$mgrdir/keyring";
345 my $mgrname = "mgr.$id";
347 die "ceph manager directory '$mgrdir' already exists\n" if -d
$mgrdir;
349 print "creating manager directory '$mgrdir'\n";
351 print "creating keys for '$mgrname'\n";
352 my $output = $rados->mon_command({
353 prefix
=> 'auth get-or-create',
356 mon
=> 'allow profile mgr',
362 PVE
::Tools
::file_set_contents
($mgrkeyring, $output);
364 print "setting owner for directory\n";
365 run_command
(["chown", 'ceph:ceph', '-R', $mgrdir]);
367 print "enabling service 'ceph-mgr\@$id.service'\n";
368 ceph_service_cmd
('enable', $mgrname);
369 print "starting service 'ceph-mgr\@$id.service'\n";
370 ceph_service_cmd
('start', $mgrname);
372 broadcast_ceph_services
();
378 my ($mgrid, $rados) = @_;
380 my $clustername = PVE
::Ceph
::Tools
::get_config
('ccname');
381 my $mgrname = "mgr.$mgrid";
382 my $mgrdir = "/var/lib/ceph/mgr/$clustername-$mgrid";
384 die "ceph manager directory '$mgrdir' not found\n"
387 print "disabling service 'ceph-mgr\@$mgrid.service'\n";
388 ceph_service_cmd
('disable', $mgrname);
389 print "stopping service 'ceph-mgr\@$mgrid.service'\n";
390 ceph_service_cmd
('stop', $mgrname);
392 print "removing manager directory '$mgrdir'\n";
393 File
::Path
::remove_tree
($mgrdir);
395 print "removing authkeys for $mgrname\n";
397 $rados = PVE
::RADOS-
>new();
400 $rados->mon_command({ prefix
=> 'auth del', entity
=> "$mgrname" });
402 broadcast_ceph_services
();