PVE/Ceph/Tools.pm

   1 package PVE::Ceph::Tools;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use File::Basename;
   8 use IO::File;
   9 use JSON;
  10
  11 use PVE::Tools qw(run_command dir_glob_foreach);
  12 use PVE::Cluster qw(cfs_read_file);
  13 use PVE::RADOS;
  14 use PVE::Ceph::Services;
  15 use PVE::CephConfig;
  16
  17 my $ccname = 'ceph'; # ceph cluster name
  18 my $ceph_cfgdir = "/etc/ceph";
  19 my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf";
  20 my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf";
  21
  22 my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
  23 my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring";
  24 my $ckeyring_path = "/etc/ceph/ceph.client.admin.keyring";
  25 my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
  26 my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";
  27 my $ceph_mds_data_dir = '/var/lib/ceph/mds';
  28
  29 my $ceph_service = {
  30     ceph_bin => "/usr/bin/ceph",
  31     ceph_mon => "/usr/bin/ceph-mon",
  32     ceph_mgr => "/usr/bin/ceph-mgr",
  33     ceph_osd => "/usr/bin/ceph-osd",
  34     ceph_mds => "/usr/bin/ceph-mds",
  35     ceph_volume => '/usr/sbin/ceph-volume',
  36 };
  37
  38 my $config_hash = {
  39     ccname => $ccname,
  40     pve_ceph_cfgpath => $pve_ceph_cfgpath,
  41     pve_mon_key_path => $pve_mon_key_path,
  42     pve_ckeyring_path => $pve_ckeyring_path,
  43     ceph_bootstrap_osd_keyring => $ceph_bootstrap_osd_keyring,
  44     ceph_bootstrap_mds_keyring => $ceph_bootstrap_mds_keyring,
  45     ceph_mds_data_dir => $ceph_mds_data_dir,
  46     long_rados_timeout => 60,
  47     ceph_cfgpath => $ceph_cfgpath,
  48 };
  49
  50 sub get_local_version {
  51     my ($noerr) = @_;
  52
  53     if (check_ceph_installed('ceph_bin', $noerr)) {
  54         my $ceph_version;
  55         run_command(
  56             [ $ceph_service->{ceph_bin}, '--version' ],
  57             noerr => $noerr,
  58             outfunc => sub { $ceph_version = shift if !defined $ceph_version },
  59         );
  60         return undef if !defined $ceph_version;
  61
  62         if ($ceph_version =~ /^ceph.*\sv?(\d+(?:\.\d+)+(?:-pve\d+)?)\s+(?:\(([a-zA-Z0-9]+)\))?/) {
  63             my ($version, $buildcommit) = ($1, $2);
  64             my $subversions = [ split(/\.|-/, $version) ];
  65
  66             # return (version, buildid, major, minor, ...) : major;
  67             return wantarray
  68                 ? ($version, $buildcommit, $subversions)
  69                 : $subversions->[0];
  70         }
  71     }
  72
  73     return undef;
  74 }
  75
  76 sub get_cluster_versions {
  77     my ($service, $noerr) = @_;
  78
  79     my $rados = PVE::RADOS->new();
  80     my $cmd = $service ? "$service versions" : 'versions';
  81     return $rados->mon_command({ prefix => $cmd });
  82 }
  83
  84 sub get_config {
  85     my $key = shift;
  86
  87     my $value = $config_hash->{$key};
  88
  89     die "no such ceph config '$key'" if !$value;
  90
  91     return $value;
  92 }
  93
  94 sub purge_all_ceph_files {
  95     my ($services) = @_;
  96     my $is_local_mon;
  97     my $monlist = [ split(',', PVE::CephConfig::get_monaddr_list($pve_ceph_cfgpath)) ];
  98
  99     foreach my $service (keys %$services) {
 100         my $type = $services->{$service};
 101         next if (!%$type);
 102
 103         foreach my $name (keys %$type) {
 104             my $dir_exists = $type->{$name}->{direxists};
 105
 106             $is_local_mon = grep($type->{$name}->{addr}, @$monlist)
 107                 if $service eq 'mon';
 108
 109             my $path = "/var/lib/ceph/$service";
 110             $path = '/var/log/ceph' if $service eq 'logs';
 111             if ($dir_exists) {
 112                 my $err;
 113                 File::Path::remove_tree($path, {
 114                         keep_root => 1,
 115                         error => \$err,
 116                     });
 117                 warn "Error removing path, '$path'\n" if @$err;
 118             }
 119         }
 120     }
 121
 122     if (scalar @$monlist > 0 && !$is_local_mon) {
 123         warn "Foreign MON address in ceph.conf. Keeping config & keyrings\n"
 124     } else {
 125         print "Removing config & keyring files\n";
 126         foreach my $file (%$config_hash) {
 127             unlink $file if (-e $file);
 128         }
 129     }
 130 }
 131
 132 sub purge_all_ceph_services {
 133     my ($services) = @_;
 134
 135     foreach my $service (keys %$services) {
 136         my $type = $services->{$service};
 137         next if (!%$type);
 138
 139         foreach my $name (keys %$type) {
 140             my $service_exists = $type->{$name}->{service};
 141
 142             if ($service_exists) {
 143                 eval { PVE::Ceph::Services::ceph_service_cmd('disable', "$service.$name") };
 144                 warn "Could not disable ceph-$service\@$name, error: $@\n" if $@;
 145
 146                 eval { PVE::Ceph::Services::ceph_service_cmd('stop', "$service.$name") };
 147                 warn "Could not stop ceph-$service\@$name, error: $@\n" if $@;
 148             }
 149         }
 150     }
 151 }
 152
 153 sub check_ceph_installed {
 154     my ($service, $noerr) = @_;
 155
 156     $service = 'ceph_bin' if !defined($service);
 157
 158     # the flag file is checked as on a new installation, the binary gets
 159     # extracted by dpkg before the installation is finished
 160     if (! -x $ceph_service->{$service} || -f '/run/ceph-install-flag') {
 161         die "binary not installed: $ceph_service->{$service}\n" if !$noerr;
 162         return undef;
 163     }
 164
 165     return 1;
 166 }
 167
 168
 169 sub check_ceph_configured {
 170
 171     check_ceph_inited();
 172
 173     die "ceph not fully configured - missing '$pve_ckeyring_path'\n"
 174         if ! -f $pve_ckeyring_path;
 175
 176     return 1;
 177 }
 178
 179 sub check_ceph_inited {
 180     my ($noerr) = @_;
 181
 182     return undef if !check_ceph_installed('ceph_mon', $noerr);
 183
 184     if (! -f $pve_ceph_cfgpath) {
 185         die "pveceph configuration not initialized\n" if !$noerr;
 186         return undef;
 187     }
 188
 189     return 1;
 190 }
 191
 192 sub check_ceph_enabled {
 193     my ($noerr) = @_;
 194
 195     return undef if !check_ceph_inited($noerr);
 196
 197     if (! -f $ceph_cfgpath) {
 198         die "pveceph configuration not enabled\n" if !$noerr;
 199         return undef;
 200     }
 201
 202     return 1;
 203 }
 204
 205 my $set_pool_setting = sub {
 206     my ($pool, $setting, $value) = @_;
 207
 208     my $command;
 209     if ($setting eq 'application') {
 210         $command = {
 211             prefix => "osd pool application enable",
 212             pool   => "$pool",
 213             app    => "$value",
 214         };
 215     } else {
 216         $command = {
 217             prefix => "osd pool set",
 218             pool   => "$pool",
 219             var    => "$setting",
 220             val    => "$value",
 221             format => 'plain',
 222         };
 223     }
 224
 225     my $rados = PVE::RADOS->new();
 226     eval { $rados->mon_command($command); };
 227     return $@ ? $@ : undef;
 228 };
 229
 230 sub set_pool {
 231     my ($pool, $param) = @_;
 232
 233     # by default, pool size always resets min_size, so set it as first item
 234     # https://tracker.ceph.com/issues/44862
 235     my $keys = [ grep { $_ ne 'size' } sort keys %$param ];
 236     unshift @$keys, 'size' if exists $param->{size};
 237
 238     for my $setting (@$keys) {
 239         my $value = $param->{$setting};
 240
 241         print "pool $pool: applying $setting = $value\n";
 242         if (my $err = $set_pool_setting->($pool, $setting, $value)) {
 243             print "$err";
 244         } else {
 245             delete $param->{$setting};
 246         }
 247     }
 248
 249     if (scalar(keys %$param) > 0) {
 250         my $missing = join(', ', sort keys %$param );
 251         die "Could not set: $missing\n";
 252     }
 253
 254 }
 255
 256 sub create_pool {
 257     my ($pool, $param, $rados) = @_;
 258
 259     if (!defined($rados)) {
 260         $rados = PVE::RADOS->new();
 261     }
 262
 263     my $pg_num = $param->{pg_num} || 128;
 264
 265     $rados->mon_command({
 266         prefix => "osd pool create",
 267         pool => $pool,
 268         pg_num => int($pg_num),
 269         format => 'plain',
 270     });
 271
 272     set_pool($pool, $param);
 273
 274 }
 275
 276 sub ls_pools {
 277     my ($pool, $rados) = @_;
 278
 279     if (!defined($rados)) {
 280         $rados = PVE::RADOS->new();
 281     }
 282
 283     my $res = $rados->mon_command({ prefix => "osd lspools" });
 284
 285     return $res;
 286 }
 287
 288 sub destroy_pool {
 289     my ($pool, $rados) = @_;
 290
 291     if (!defined($rados)) {
 292         $rados = PVE::RADOS->new();
 293     }
 294
 295     # fixme: '--yes-i-really-really-mean-it'
 296     $rados->mon_command({
 297         prefix => "osd pool delete",
 298         pool => $pool,
 299         pool2 => $pool,
 300         'yes_i_really_really_mean_it' => JSON::true,
 301         format => 'plain',
 302     });
 303 }
 304
 305 sub setup_pve_symlinks {
 306     # fail if we find a real file instead of a link
 307     if (-f $ceph_cfgpath) {
 308         my $lnk = readlink($ceph_cfgpath);
 309         die "file '$ceph_cfgpath' already exists and is not a symlink to $pve_ceph_cfgpath\n"
 310             if !$lnk || $lnk ne $pve_ceph_cfgpath;
 311     } else {
 312         mkdir $ceph_cfgdir;
 313         symlink($pve_ceph_cfgpath, $ceph_cfgpath) ||
 314             die "unable to create symlink '$ceph_cfgpath' - $!\n";
 315     }
 316     my $ceph_uid = getpwnam('ceph');
 317     my $ceph_gid = getgrnam('ceph');
 318     chown $ceph_uid, $ceph_gid, $ceph_cfgdir;
 319 }
 320
 321 sub get_or_create_admin_keyring {
 322     if (! -f $pve_ckeyring_path) {
 323         run_command("ceph-authtool --create-keyring $pve_ckeyring_path " .
 324             "--gen-key -n client.admin " .
 325             "--cap mon 'allow *' " .
 326             "--cap osd 'allow *' " .
 327             "--cap mds 'allow *' " .
 328             "--cap mgr 'allow *' ");
 329         # we do not want to overwrite it
 330         if (! -f $ckeyring_path) {
 331             run_command("cp $pve_ckeyring_path $ckeyring_path");
 332             run_command("chown ceph:ceph $ckeyring_path");
 333         }
 334     }
 335     return $pve_ckeyring_path;
 336 }
 337
 338 # get ceph-volume managed osds
 339 sub ceph_volume_list {
 340     my $result = {};
 341
 342     if (!check_ceph_installed('ceph_volume', 1)) {
 343         return $result;
 344     }
 345
 346     my $output = '';
 347     my $cmd = [ $ceph_service->{ceph_volume}, 'lvm', 'list', '--format', 'json' ];
 348     run_command($cmd, outfunc => sub { $output .= shift });
 349
 350     $result = eval { decode_json($output) };
 351     warn $@ if $@;
 352     return $result;
 353 }
 354
 355 sub ceph_volume_zap {
 356     my ($osdid, $destroy) = @_;
 357
 358     die "no osdid given\n" if !defined($osdid);
 359
 360     my $cmd = [ $ceph_service->{ceph_volume}, 'lvm', 'zap', '--osd-id', $osdid ];
 361     push @$cmd, '--destroy' if $destroy;
 362
 363     run_command($cmd);
 364 }
 365
 366 sub get_db_wal_sizes {
 367     my $res = {};
 368
 369     my $rados = PVE::RADOS->new();
 370     my $db_config = $rados->mon_command({ prefix => 'config-key dump', key => 'config/' });
 371
 372     $res->{db} = $db_config->{"config/osd/bluestore_block_db_size"} //
 373                  $db_config->{"config/global/bluestore_block_db_size"};
 374
 375     $res->{wal} = $db_config->{"config/osd/bluestore_block_wal_size"} //
 376                   $db_config->{"config/global/bluestore_block_wal_size"};
 377
 378     if (!$res->{db} || !$res->{wal}) {
 379         my $cfg = cfs_read_file('ceph.conf');
 380         if (!$res->{db}) {
 381             $res->{db} = $cfg->{osd}->{bluestore_block_db_size} //
 382                          $cfg->{global}->{bluestore_block_db_size};
 383         }
 384
 385         if (!$res->{wal}) {
 386             $res->{wal} = $cfg->{osd}->{bluestore_block_wal_size} //
 387                          $cfg->{global}->{bluestore_block_wal_size};
 388         }
 389     }
 390
 391     return $res;
 392 }
 393 sub get_possible_osd_flags {
 394     my $possible_flags = {
 395         pause => {
 396             description => 'Pauses read and writes.',
 397             type => 'boolean',
 398             optional=> 1,
 399         },
 400         noup => {
 401             description => 'OSDs are not allowed to start.',
 402             type => 'boolean',
 403             optional=> 1,
 404         },
 405         nodown => {
 406             description => 'OSD failure reports are being ignored, such that the monitors will not mark OSDs down.',
 407             type => 'boolean',
 408             optional=> 1,
 409         },
 410         noout => {
 411             description => 'OSDs will not automatically be marked out after the configured interval.',
 412             type => 'boolean',
 413             optional=> 1,
 414         },
 415         noin => {
 416             description => 'OSDs that were previously marked out will not be marked back in when they start.',
 417             type => 'boolean',
 418             optional=> 1,
 419         },
 420         nobackfill => {
 421             description => 'Backfilling of PGs is suspended.',
 422             type => 'boolean',
 423             optional=> 1,
 424         },
 425         norebalance => {
 426             description => 'Rebalancing of PGs is suspended.',
 427             type => 'boolean',
 428             optional=> 1,
 429         },
 430         norecover => {
 431             description => 'Recovery of PGs is suspended.',
 432             type => 'boolean',
 433             optional=> 1,
 434         },
 435         noscrub => {
 436             description => 'Scrubbing is disabled.',
 437             type => 'boolean',
 438             optional=> 1,
 439         },
 440         'nodeep-scrub' => {
 441             description => 'Deep Scrubbing is disabled.',
 442             type => 'boolean',
 443             optional=> 1,
 444         },
 445         notieragent => {
 446             description => 'Cache tiering activity is suspended.',
 447             type => 'boolean',
 448             optional=> 1,
 449         },
 450     };
 451     return $possible_flags;
 452 }
 453
 454 sub get_real_flag_name {
 455     my ($flag) = @_;
 456
 457     # the 'pause' flag gets always set to both 'pauserd' and 'pausewr'
 458     # so decide that the 'pause' flag is set if we detect 'pauserd'
 459     my $flagmap = {
 460         'pause' => 'pauserd',
 461     };
 462
 463     return $flagmap->{$flag} // $flag;
 464 }
 465
 466 sub ceph_cluster_status {
 467     my ($rados) = @_;
 468     $rados = PVE::RADOS->new() if !$rados;
 469
 470     my $status = $rados->mon_command({ prefix => 'status' });
 471     $status->{health} = $rados->mon_command({ prefix => 'health', detail => 'detail' });
 472
 473     if (!exists $status->{monmap}->{mons}) { # octopus moved most info out of status, re-add
 474         $status->{monmap} = $rados->mon_command({ prefix => 'mon dump' });
 475         $status->{mgrmap} = $rados->mon_command({ prefix => 'mgr dump' });
 476     }
 477
 478     return $status;
 479 }
 480
 481 1;