PVE/Ceph/Services.pm

   1 package PVE::Ceph::Services;
   2
   3 use strict;
   4 use warnings;
   5
   6 use PVE::Ceph::Tools;
   7 use PVE::Cluster qw(cfs_read_file);
   8 use PVE::Tools qw(run_command);
   9 use PVE::RADOS;
  10
  11 use JSON;
  12 use File::Path;
  13
  14 use constant SERVICE_REGEX => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?';
  15
  16 # checks /etc/systemd/system/ceph-* to list all services, even if not running
  17 # also checks /var/lib/ceph/$type
  18 sub get_local_services {
  19     my $res = {};
  20
  21     for my $type (qw(mds mgr mon)) {
  22         $res->{$type} = {};
  23
  24         my $path = "/etc/systemd/system/ceph-$type.target.wants";
  25         my $regex = "ceph-$type\@(.*)\.service";
  26         PVE::Tools::dir_glob_foreach($path, $regex, sub {
  27             my (undef, $id) = @_;
  28             $res->{$type}->{$id}->{service} = 1;
  29         });
  30
  31         $path = "/var/lib/ceph/$type";
  32         $regex = "([^-]+)-(.*)";
  33         PVE::Tools::dir_glob_foreach($path, $regex, sub {
  34             my (undef, $clustername, $id) = @_;
  35             $res->{$type}->{$id}->{direxists} = 1;
  36         });
  37     }
  38     return $res;
  39 }
  40
  41 sub broadcast_ceph_services {
  42     my $services = get_local_services();
  43
  44     for my $type (keys %$services) {
  45         my $data = encode_json($services->{$type});
  46         PVE::Cluster::broadcast_node_kv("ceph-$type", $data);
  47     }
  48 }
  49
  50 sub broadcast_ceph_versions {
  51     my ($version, $buildcommit, $vers_parts) = PVE::Ceph::Tools::get_local_version(1);
  52
  53     if ($version) {
  54         if (my $old = PVE::Cluster::get_node_kv("ceph-versions")) {
  55             my $nodename = PVE::INotify::nodename();
  56             $old = eval { decode_json($old->{$nodename}) };
  57             warn $@ if $@; # should not happen
  58             if (defined($old) && $old->{buildcommit} eq $buildcommit && $old->{version}->{str} eq $version) {
  59                 return; # up to date, nothing to do so avoid (not exactly cheap) broadcast
  60             }
  61         }
  62         my $node_versions = {
  63             version => {
  64                 str => $version,
  65                 parts => $vers_parts,
  66             },
  67             buildcommit => $buildcommit,
  68         };
  69         PVE::Cluster::broadcast_node_kv("ceph-versions", encode_json($node_versions));
  70     }
  71 }
  72
  73 sub get_ceph_versions {
  74     my $res;
  75
  76     if (defined(my $versions = PVE::Cluster::get_node_kv("ceph-versions"))) {
  77         $res = {
  78             map { eval { $_ => decode_json($versions->{$_}) } } keys %$versions
  79         };
  80     }
  81
  82     return $res;
  83 }
  84
  85 sub get_cluster_service {
  86     my ($type) = @_;
  87
  88     my $raw = PVE::Cluster::get_node_kv("ceph-$type");
  89     my $res = {
  90         map { $_ => eval { decode_json($raw->{$_}) } } keys $raw->%*
  91     };
  92
  93     return $res;
  94 }
  95
  96 sub ceph_service_cmd {
  97     my ($action, $service) = @_;
  98
  99     if ($service && $service =~ m/^(mon|osd|mds|mgr|radosgw)(\.(${\SERVICE_REGEX}))?$/) {
 100         $service = defined($3) ? "ceph-$1\@$3" : "ceph-$1.target";
 101     } else {
 102         $service = "ceph.target";
 103     }
 104
 105     run_command(['/bin/systemctl', $action, $service]);
 106 }
 107
 108 sub get_services_info {
 109     my ($type, $cfg, $rados) = @_;
 110
 111     my $result = {};
 112     my $services = get_cluster_service($type);
 113
 114     foreach my $host (sort keys %$services) {
 115         foreach my $id (sort keys %{$services->{$host}}) {
 116             my $service = $result->{$id} = $services->{$host}->{$id};
 117             $service->{host} = $host;
 118             $service->{name} = $id;
 119             $service->{state} = 'unknown';
 120             if ($service->{service}) {
 121                 $service->{state} = 'stopped';
 122             }
 123         }
 124     }
 125
 126     if (!$cfg) {
 127         $cfg = cfs_read_file('ceph.conf');
 128     }
 129
 130     foreach my $section (keys %$cfg) {
 131         my $d = $cfg->{$section};
 132         if ($section =~ m/^$type\.(\S+)$/) {
 133             my $id = $1;
 134             my $service = $result->{$id};
 135             my $addr = $d->{"${type}_addr"} // $d->{public_addr} // $d->{host};
 136             $service->{name} //= $id;
 137             $service->{addr} //= $addr;
 138             $service->{state} //= 'unknown';
 139             $service->{host} //= $d->{host};
 140         }
 141     }
 142
 143     if (!$rados) {
 144         return $result;
 145     }
 146
 147     my $metadata = $rados->mon_command({ prefix => "$type metadata" });
 148     foreach my $info (@$metadata) {
 149         my $id = $info->{name} // $info->{id};
 150         my $service = $result->{$id};
 151         $service->{ceph_version_short} = $info->{ceph_version_short};
 152         $service->{ceph_version} = $info->{ceph_version};
 153         $service->{host} //= $info->{hostname};
 154         $service->{addr} //= $info->{addr};
 155     }
 156
 157     return $result;
 158 }
 159
 160 # MDS
 161
 162 sub list_local_mds_ids {
 163     my $mds_list = [];
 164     my $ceph_mds_data_dir = PVE::Ceph::Tools::get_config('ceph_mds_data_dir');
 165     my $ccname = PVE::Ceph::Tools::get_config('ccname');
 166
 167     PVE::Tools::dir_glob_foreach($ceph_mds_data_dir, qr/$ccname-(\S+)/, sub {
 168         my (undef, $mds_id) = @_;
 169         push @$mds_list, $mds_id;
 170     });
 171
 172     return $mds_list;
 173 }
 174
 175 sub get_cluster_mds_state {
 176     my ($rados) = @_;
 177
 178     my $mds_state = {};
 179
 180     if (!defined($rados)) {
 181         $rados = PVE::RADOS->new();
 182     }
 183
 184     my $add_state = sub {
 185         my ($mds) = @_;
 186
 187         my $state = {};
 188         $state->{addr} = $mds->{addr};
 189         $state->{rank} = $mds->{rank};
 190         $state->{standby_replay} = $mds->{standby_replay} ? 1 : 0;
 191         $state->{state} = $mds->{state};
 192
 193         $mds_state->{$mds->{name}} = $state;
 194     };
 195
 196     my $mds_dump = $rados->mon_command({ prefix => 'mds stat' });
 197     my $fsmap = $mds_dump->{fsmap};
 198
 199
 200     foreach my $mds (@{$fsmap->{standbys}}) {
 201         $add_state->($mds);
 202     }
 203
 204     my $fs_info = $fsmap->{filesystems}->[0];
 205     my $active_mds = $fs_info->{mdsmap}->{info};
 206
 207     # normally there's only one active MDS, but we can have multiple active for
 208     # different ranks (e.g., different cephs path hierarchy). So just add all.
 209     foreach my $mds (values %$active_mds) {
 210         $add_state->($mds);
 211     }
 212
 213     return $mds_state;
 214 }
 215
 216 sub is_any_mds_active {
 217     my ($rados) = @_;
 218
 219     if (!defined($rados)) {
 220         $rados = PVE::RADOS->new();
 221     }
 222
 223     my $mds_dump = $rados->mon_command({ prefix => 'mds stat' });
 224     my $fs = $mds_dump->{fsmap}->{filesystems};
 225
 226     if (!($fs && scalar(@$fs) > 0)) {
 227         return undef;
 228     }
 229     my $active_mds = $fs->[0]->{mdsmap}->{info};
 230
 231     for my $mds (values %$active_mds) {
 232         return 1 if $mds->{state} eq 'up:active';
 233     }
 234
 235     return 0;
 236 }
 237
 238 sub create_mds {
 239     my ($id, $rados) = @_;
 240
 241     # `ceph fs status` fails with numeric only ID.
 242     die "ID: $id, numeric only IDs are not supported\n"
 243         if $id =~ /^\d+$/;
 244
 245     if (!defined($rados)) {
 246         $rados = PVE::RADOS->new();
 247     }
 248
 249     my $ccname = PVE::Ceph::Tools::get_config('ccname');
 250     my $service_dir = "/var/lib/ceph/mds/$ccname-$id";
 251     my $service_keyring = "$service_dir/keyring";
 252     my $service_name = "mds.$id";
 253
 254     die "ceph MDS directory '$service_dir' already exists\n"
 255         if -d $service_dir;
 256
 257     print "creating MDS directory '$service_dir'\n";
 258     eval { File::Path::mkpath($service_dir) };
 259     my $err = $@;
 260     die "creation MDS directory '$service_dir' failed\n" if $err;
 261
 262     # http://docs.ceph.com/docs/luminous/install/manual-deployment/#adding-mds
 263     my $priv = [
 264         mon => 'allow profile mds',
 265         osd => 'allow rwx',
 266         mds => 'allow *',
 267     ];
 268
 269     print "creating keys for '$service_name'\n";
 270     my $output = $rados->mon_command({
 271         prefix => 'auth get-or-create',
 272         entity => $service_name,
 273         caps => $priv,
 274         format => 'plain',
 275     });
 276
 277     PVE::Tools::file_set_contents($service_keyring, $output);
 278
 279     print "setting ceph as owner for service directory\n";
 280     run_command(["chown", 'ceph:ceph', '-R', $service_dir]);
 281
 282     print "enabling service 'ceph-mds\@$id.service'\n";
 283     ceph_service_cmd('enable', $service_name);
 284     print "starting service 'ceph-mds\@$id.service'\n";
 285     ceph_service_cmd('start', $service_name);
 286
 287     broadcast_ceph_services();
 288
 289     return undef;
 290 };
 291
 292 sub destroy_mds {
 293     my ($id, $rados) = @_;
 294
 295     if (!defined($rados)) {
 296         $rados = PVE::RADOS->new();
 297     }
 298
 299     my $ccname = PVE::Ceph::Tools::get_config('ccname');
 300
 301     my $service_name = "mds.$id";
 302     my $service_dir = "/var/lib/ceph/mds/$ccname-$id";
 303
 304     print "disabling service 'ceph-mds\@$id.service'\n";
 305     ceph_service_cmd('disable', $service_name);
 306     print "stopping service 'ceph-mds\@$id.service'\n";
 307     ceph_service_cmd('stop', $service_name);
 308
 309     if (-d $service_dir) {
 310         print "removing ceph-mds directory '$service_dir'\n";
 311         File::Path::remove_tree($service_dir);
 312     } else {
 313         warn "cannot cleanup MDS $id directory, '$service_dir' not found\n"
 314     }
 315
 316     print "removing ceph auth for '$service_name'\n";
 317     $rados->mon_command({
 318             prefix => 'auth del',
 319             entity => $service_name,
 320             format => 'plain'
 321         });
 322
 323     broadcast_ceph_services();
 324
 325     return undef;
 326 };
 327
 328 # MGR
 329
 330 sub create_mgr {
 331     my ($id, $rados) = @_;
 332
 333     my $clustername = PVE::Ceph::Tools::get_config('ccname');
 334     my $mgrdir = "/var/lib/ceph/mgr/$clustername-$id";
 335     my $mgrkeyring = "$mgrdir/keyring";
 336     my $mgrname = "mgr.$id";
 337
 338     die "ceph manager directory '$mgrdir' already exists\n" if -d $mgrdir;
 339
 340     print "creating manager directory '$mgrdir'\n";
 341     mkdir $mgrdir;
 342     print "creating keys for '$mgrname'\n";
 343     my $output = $rados->mon_command({
 344         prefix => 'auth get-or-create',
 345         entity => $mgrname,
 346         caps => [
 347             mon => 'allow profile mgr',
 348             osd => 'allow *',
 349             mds => 'allow *',
 350         ],
 351         format => 'plain'
 352     });
 353     PVE::Tools::file_set_contents($mgrkeyring, $output);
 354
 355     print "setting owner for directory\n";
 356     run_command(["chown", 'ceph:ceph', '-R', $mgrdir]);
 357
 358     print "enabling service 'ceph-mgr\@$id.service'\n";
 359     ceph_service_cmd('enable', $mgrname);
 360     print "starting service 'ceph-mgr\@$id.service'\n";
 361     ceph_service_cmd('start', $mgrname);
 362
 363     broadcast_ceph_services();
 364
 365     return undef;
 366 }
 367
 368 sub destroy_mgr {
 369     my ($mgrid, $rados) = @_;
 370
 371     my $clustername = PVE::Ceph::Tools::get_config('ccname');
 372     my $mgrname = "mgr.$mgrid";
 373     my $mgrdir = "/var/lib/ceph/mgr/$clustername-$mgrid";
 374
 375     die "ceph manager directory '$mgrdir' not found\n"
 376         if ! -d $mgrdir;
 377
 378     print "disabling service 'ceph-mgr\@$mgrid.service'\n";
 379     ceph_service_cmd('disable', $mgrname);
 380     print "stopping service 'ceph-mgr\@$mgrid.service'\n";
 381     ceph_service_cmd('stop', $mgrname);
 382
 383     print "removing manager directory '$mgrdir'\n";
 384     File::Path::remove_tree($mgrdir);
 385
 386     print "removing authkeys for $mgrname\n";
 387     if (!$rados) {
 388         $rados = PVE::RADOS->new();
 389     }
 390
 391     $rados->mon_command({ prefix => 'auth del', entity => "$mgrname" });
 392
 393     broadcast_ceph_services();
 394
 395     return undef;
 396 }
 397
 398 1;