PVE/Ceph/Tools.pm

   1 package PVE::Ceph::Tools;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use File::Basename;
   8 use IO::File;
   9 use JSON;
  10
  11 use PVE::Tools qw(run_command dir_glob_foreach extract_param);
  12 use PVE::Cluster qw(cfs_read_file);
  13 use PVE::RADOS;
  14 use PVE::Ceph::Services;
  15 use PVE::CephConfig;
  16
  17 my $ccname = 'ceph'; # ceph cluster name
  18 my $ceph_cfgdir = "/etc/ceph";
  19 my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf";
  20 my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf";
  21
  22 my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
  23 my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring";
  24 my $ckeyring_path = "/etc/ceph/ceph.client.admin.keyring";
  25 my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
  26 my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";
  27 my $ceph_mds_data_dir = '/var/lib/ceph/mds';
  28
  29 my $ceph_service = {
  30     ceph_bin => "/usr/bin/ceph",
  31     ceph_mon => "/usr/bin/ceph-mon",
  32     ceph_mgr => "/usr/bin/ceph-mgr",
  33     ceph_osd => "/usr/bin/ceph-osd",
  34     ceph_mds => "/usr/bin/ceph-mds",
  35     ceph_volume => '/usr/sbin/ceph-volume',
  36 };
  37
  38 my $config_hash = {
  39     ccname => $ccname,
  40     pve_ceph_cfgpath => $pve_ceph_cfgpath,
  41     pve_mon_key_path => $pve_mon_key_path,
  42     pve_ckeyring_path => $pve_ckeyring_path,
  43     ceph_bootstrap_osd_keyring => $ceph_bootstrap_osd_keyring,
  44     ceph_bootstrap_mds_keyring => $ceph_bootstrap_mds_keyring,
  45     ceph_mds_data_dir => $ceph_mds_data_dir,
  46     long_rados_timeout => 60,
  47     ceph_cfgpath => $ceph_cfgpath,
  48 };
  49
  50 sub get_local_version {
  51     my ($noerr) = @_;
  52
  53     if (check_ceph_installed('ceph_bin', $noerr)) {
  54         my $ceph_version;
  55         run_command(
  56             [ $ceph_service->{ceph_bin}, '--version' ],
  57             noerr => $noerr,
  58             outfunc => sub { $ceph_version = shift if !defined $ceph_version },
  59         );
  60         return undef if !defined $ceph_version;
  61
  62         if ($ceph_version =~ /^ceph.*\sv?(\d+(?:\.\d+)+(?:-pve\d+)?)\s+(?:\(([a-zA-Z0-9]+)\))?/) {
  63             my ($version, $buildcommit) = ($1, $2);
  64             my $subversions = [ split(/\.|-/, $version) ];
  65
  66             # return (version, buildid, major, minor, ...) : major;
  67             return wantarray
  68                 ? ($version, $buildcommit, $subversions)
  69                 : $subversions->[0];
  70         }
  71     }
  72
  73     return undef;
  74 }
  75
  76 sub get_cluster_versions {
  77     my ($service, $noerr) = @_;
  78
  79     my $rados = PVE::RADOS->new();
  80     my $cmd = $service ? "$service versions" : 'versions';
  81     return $rados->mon_command({ prefix => $cmd });
  82 }
  83
  84 sub get_config {
  85     my $key = shift;
  86
  87     my $value = $config_hash->{$key};
  88
  89     die "no such ceph config '$key'" if !$value;
  90
  91     return $value;
  92 }
  93
  94 sub purge_all_ceph_files {
  95     my ($services) = @_;
  96     my $is_local_mon;
  97     my $monlist = [ split(',', PVE::CephConfig::get_monaddr_list($pve_ceph_cfgpath)) ];
  98
  99     foreach my $service (keys %$services) {
 100         my $type = $services->{$service};
 101         next if (!%$type);
 102
 103         foreach my $name (keys %$type) {
 104             my $dir_exists = $type->{$name}->{direxists};
 105
 106             $is_local_mon = grep($type->{$name}->{addr}, @$monlist)
 107                 if $service eq 'mon';
 108
 109             my $path = "/var/lib/ceph/$service";
 110             $path = '/var/log/ceph' if $service eq 'logs';
 111             if ($dir_exists) {
 112                 my $err;
 113                 File::Path::remove_tree($path, {
 114                         keep_root => 1,
 115                         error => \$err,
 116                     });
 117                 warn "Error removing path, '$path'\n" if @$err;
 118             }
 119         }
 120     }
 121
 122     if (scalar @$monlist > 0 && !$is_local_mon) {
 123         warn "Foreign MON address in ceph.conf. Keeping config & keyrings\n"
 124     } else {
 125         print "Removing config & keyring files\n";
 126         foreach my $file (%$config_hash) {
 127             unlink $file if (-e $file);
 128         }
 129     }
 130 }
 131
 132 sub purge_all_ceph_services {
 133     my ($services) = @_;
 134
 135     foreach my $service (keys %$services) {
 136         my $type = $services->{$service};
 137         next if (!%$type);
 138
 139         foreach my $name (keys %$type) {
 140             my $service_exists = $type->{$name}->{service};
 141
 142             if ($service_exists) {
 143                 eval { PVE::Ceph::Services::ceph_service_cmd('disable', "$service.$name") };
 144                 warn "Could not disable ceph-$service\@$name, error: $@\n" if $@;
 145
 146                 eval { PVE::Ceph::Services::ceph_service_cmd('stop', "$service.$name") };
 147                 warn "Could not stop ceph-$service\@$name, error: $@\n" if $@;
 148             }
 149         }
 150     }
 151 }
 152
 153 sub ceph_install_flag_file { return '/run/pve-ceph-install-flag' };
 154
 155 sub check_ceph_installed {
 156     my ($service, $noerr) = @_;
 157
 158     $service = 'ceph_bin' if !defined($service);
 159
 160     # NOTE: the flag file is checked as on a new installation, the binary gets
 161     # extracted by dpkg before the installation is finished
 162     if (! -x $ceph_service->{$service} || -f ceph_install_flag_file()) {
 163         die "binary not installed: $ceph_service->{$service}\n" if !$noerr;
 164         return undef;
 165     }
 166
 167     return 1;
 168 }
 169
 170
 171 sub check_ceph_configured {
 172
 173     check_ceph_inited();
 174
 175     die "ceph not fully configured - missing '$pve_ckeyring_path'\n"
 176         if ! -f $pve_ckeyring_path;
 177
 178     return 1;
 179 }
 180
 181 sub check_ceph_inited {
 182     my ($noerr) = @_;
 183
 184     return undef if !check_ceph_installed('ceph_mon', $noerr);
 185
 186     if (! -f $pve_ceph_cfgpath) {
 187         die "pveceph configuration not initialized\n" if !$noerr;
 188         return undef;
 189     }
 190
 191     return 1;
 192 }
 193
 194 sub check_ceph_enabled {
 195     my ($noerr) = @_;
 196
 197     return undef if !check_ceph_inited($noerr);
 198
 199     if (! -f $ceph_cfgpath) {
 200         die "pveceph configuration not enabled\n" if !$noerr;
 201         return undef;
 202     }
 203
 204     return 1;
 205 }
 206
 207 my $set_pool_setting = sub {
 208     my ($pool, $setting, $value, $rados) = @_;
 209
 210     my $command;
 211     if ($setting eq 'application') {
 212         $command = {
 213             prefix => "osd pool application enable",
 214             pool   => "$pool",
 215             app    => "$value",
 216         };
 217     } else {
 218         $command = {
 219             prefix => "osd pool set",
 220             pool   => "$pool",
 221             var    => "$setting",
 222             val    => "$value",
 223             format => 'plain',
 224         };
 225     }
 226
 227     $rados = PVE::RADOS->new() if !$rados;
 228     eval { $rados->mon_command($command); };
 229     return $@ ? $@ : undef;
 230 };
 231
 232 sub set_pool {
 233     my ($pool, $param) = @_;
 234
 235     my $rados = PVE::RADOS->new();
 236
 237     if (get_pool_type($pool, $rados) eq 'erasure') {
 238         #remove parameters that cannot be changed for erasure coded pools
 239         my $ignore_params = ['size', 'crush_rule'];
 240         for my $setting (@$ignore_params) {
 241             if ($param->{$setting}) {
 242                 print "cannot set '${setting}' for erasure coded pool\n";
 243                 delete $param->{$setting};
 244             }
 245         }
 246     }
 247     # by default, pool size always resets min_size, so set it as first item
 248     # https://tracker.ceph.com/issues/44862
 249     my $keys = [ grep { $_ ne 'size' } sort keys %$param ];
 250     unshift @$keys, 'size' if exists $param->{size};
 251
 252     for my $setting (@$keys) {
 253         my $value = $param->{$setting};
 254
 255         print "pool $pool: applying $setting = $value\n";
 256         if (my $err = $set_pool_setting->($pool, $setting, $value, $rados)) {
 257             print "$err";
 258         } else {
 259             delete $param->{$setting};
 260         }
 261     }
 262
 263     if (scalar(keys %$param) > 0) {
 264         my $missing = join(', ', sort keys %$param );
 265         die "Could not set: $missing\n";
 266     }
 267
 268 }
 269
 270 sub get_pool_properties {
 271     my ($pool, $rados) = @_;
 272     $rados = PVE::RADOS->new() if !defined($rados);
 273     my $command = {
 274         prefix => "osd pool get",
 275         pool   => "$pool",
 276         var    => "all",
 277         format => 'json',
 278     };
 279     return $rados->mon_command($command);
 280 }
 281
 282 sub get_pool_type {
 283     my ($pool, $rados) = @_;
 284     $rados = PVE::RADOS->new() if !defined($rados);
 285     return 'erasure' if get_pool_properties($pool, $rados)->{erasure_code_profile};
 286     return 'replicated';
 287 }
 288
 289 sub create_pool {
 290     my ($pool, $param, $rados) = @_;
 291     $rados = PVE::RADOS->new() if !defined($rados);
 292
 293     my $pg_num = $param->{pg_num} || 128;
 294
 295     my $mon_params = {
 296         prefix => "osd pool create",
 297         pool => $pool,
 298         pg_num => int($pg_num),
 299         format => 'plain',
 300     };
 301     $mon_params->{pool_type} = extract_param($param, 'pool_type') if $param->{pool_type};
 302     $mon_params->{erasure_code_profile} = extract_param($param, 'erasure_code_profile')
 303         if $param->{erasure_code_profile};
 304
 305     $rados->mon_command($mon_params);
 306
 307     set_pool($pool, $param);
 308
 309 }
 310
 311 sub ls_pools {
 312     my ($pool, $rados) = @_;
 313     $rados = PVE::RADOS->new() if !defined($rados);
 314
 315     my $res = $rados->mon_command({ prefix => "osd lspools" });
 316
 317     return $res;
 318 }
 319
 320 sub destroy_pool {
 321     my ($pool, $rados) = @_;
 322     $rados = PVE::RADOS->new() if !defined($rados);
 323
 324     # fixme: '--yes-i-really-really-mean-it'
 325     $rados->mon_command({
 326         prefix => "osd pool delete",
 327         pool => $pool,
 328         pool2 => $pool,
 329         'yes_i_really_really_mean_it' => JSON::true,
 330         format => 'plain',
 331     });
 332 }
 333
 334 # we get something like:
 335 #[{
 336 #   'metadata_pool_id' => 2,
 337 #   'data_pool_ids' => [ 1 ],
 338 #   'metadata_pool' => 'cephfs_metadata',
 339 #   'data_pools' => [ 'cephfs_data' ],
 340 #   'name' => 'cephfs',
 341 #}]
 342 sub ls_fs {
 343     my ($rados) = @_;
 344     $rados = PVE::RADOS->new() if !defined($rados);
 345
 346     my $res = $rados->mon_command({ prefix => "fs ls" });
 347
 348     return $res;
 349 }
 350
 351 sub create_fs {
 352     my ($fs, $param, $rados) = @_;
 353
 354     if (!defined($rados)) {
 355         $rados = PVE::RADOS->new();
 356     }
 357
 358     $rados->mon_command({
 359         prefix => "fs new",
 360         fs_name => $fs,
 361         metadata => $param->{pool_metadata},
 362         data => $param->{pool_data},
 363         format => 'plain',
 364     });
 365 }
 366
 367 sub destroy_fs {
 368     my ($fs, $rados) = @_;
 369     $rados = PVE::RADOS->new() if !defined($rados);
 370
 371     $rados->mon_command({
 372         prefix => "fs rm",
 373         fs_name => $fs,
 374         'yes_i_really_mean_it' => JSON::true,
 375         format => 'plain',
 376     });
 377 }
 378
 379 sub setup_pve_symlinks {
 380     # fail if we find a real file instead of a link
 381     if (-f $ceph_cfgpath) {
 382         my $lnk = readlink($ceph_cfgpath);
 383         die "file '$ceph_cfgpath' already exists and is not a symlink to $pve_ceph_cfgpath\n"
 384             if !$lnk || $lnk ne $pve_ceph_cfgpath;
 385     } else {
 386         mkdir $ceph_cfgdir;
 387         symlink($pve_ceph_cfgpath, $ceph_cfgpath) ||
 388             die "unable to create symlink '$ceph_cfgpath' - $!\n";
 389     }
 390     my $ceph_uid = getpwnam('ceph');
 391     my $ceph_gid = getgrnam('ceph');
 392     chown $ceph_uid, $ceph_gid, $ceph_cfgdir;
 393 }
 394
 395 sub get_or_create_admin_keyring {
 396     if (! -f $pve_ckeyring_path) {
 397         run_command("ceph-authtool --create-keyring $pve_ckeyring_path " .
 398             "--gen-key -n client.admin " .
 399             "--cap mon 'allow *' " .
 400             "--cap osd 'allow *' " .
 401             "--cap mds 'allow *' " .
 402             "--cap mgr 'allow *' ");
 403         # we do not want to overwrite it
 404         if (! -f $ckeyring_path) {
 405             run_command("cp $pve_ckeyring_path $ckeyring_path");
 406             run_command("chown ceph:ceph $ckeyring_path");
 407         }
 408     }
 409     return $pve_ckeyring_path;
 410 }
 411
 412 # get ceph-volume managed osds
 413 sub ceph_volume_list {
 414     my $result = {};
 415
 416     if (!check_ceph_installed('ceph_volume', 1)) {
 417         return $result;
 418     }
 419
 420     my $output = '';
 421     my $cmd = [ $ceph_service->{ceph_volume}, 'lvm', 'list', '--format', 'json' ];
 422     run_command($cmd, outfunc => sub { $output .= shift });
 423
 424     $result = eval { decode_json($output) };
 425     warn $@ if $@;
 426     return $result;
 427 }
 428
 429 sub ceph_volume_zap {
 430     my ($osdid, $destroy) = @_;
 431
 432     die "no osdid given\n" if !defined($osdid);
 433
 434     my $cmd = [ $ceph_service->{ceph_volume}, 'lvm', 'zap', '--osd-id', $osdid ];
 435     push @$cmd, '--destroy' if $destroy;
 436
 437     run_command($cmd);
 438 }
 439
 440 sub get_db_wal_sizes {
 441     my $res = {};
 442
 443     my $rados = PVE::RADOS->new();
 444     my $db_config = $rados->mon_command({ prefix => 'config-key dump', key => 'config/' });
 445
 446     $res->{db} = $db_config->{"config/osd/bluestore_block_db_size"} //
 447                  $db_config->{"config/global/bluestore_block_db_size"};
 448
 449     $res->{wal} = $db_config->{"config/osd/bluestore_block_wal_size"} //
 450                   $db_config->{"config/global/bluestore_block_wal_size"};
 451
 452     if (!$res->{db} || !$res->{wal}) {
 453         my $cfg = cfs_read_file('ceph.conf');
 454         if (!$res->{db}) {
 455             $res->{db} = $cfg->{osd}->{bluestore_block_db_size} //
 456                          $cfg->{global}->{bluestore_block_db_size};
 457         }
 458
 459         if (!$res->{wal}) {
 460             $res->{wal} = $cfg->{osd}->{bluestore_block_wal_size} //
 461                          $cfg->{global}->{bluestore_block_wal_size};
 462         }
 463     }
 464
 465     return $res;
 466 }
 467 sub get_possible_osd_flags {
 468     my $possible_flags = {
 469         pause => {
 470             description => 'Pauses read and writes.',
 471             type => 'boolean',
 472             optional=> 1,
 473         },
 474         noup => {
 475             description => 'OSDs are not allowed to start.',
 476             type => 'boolean',
 477             optional=> 1,
 478         },
 479         nodown => {
 480             description => 'OSD failure reports are being ignored, such that the monitors will not mark OSDs down.',
 481             type => 'boolean',
 482             optional=> 1,
 483         },
 484         noout => {
 485             description => 'OSDs will not automatically be marked out after the configured interval.',
 486             type => 'boolean',
 487             optional=> 1,
 488         },
 489         noin => {
 490             description => 'OSDs that were previously marked out will not be marked back in when they start.',
 491             type => 'boolean',
 492             optional=> 1,
 493         },
 494         nobackfill => {
 495             description => 'Backfilling of PGs is suspended.',
 496             type => 'boolean',
 497             optional=> 1,
 498         },
 499         norebalance => {
 500             description => 'Rebalancing of PGs is suspended.',
 501             type => 'boolean',
 502             optional=> 1,
 503         },
 504         norecover => {
 505             description => 'Recovery of PGs is suspended.',
 506             type => 'boolean',
 507             optional=> 1,
 508         },
 509         noscrub => {
 510             description => 'Scrubbing is disabled.',
 511             type => 'boolean',
 512             optional=> 1,
 513         },
 514         'nodeep-scrub' => {
 515             description => 'Deep Scrubbing is disabled.',
 516             type => 'boolean',
 517             optional=> 1,
 518         },
 519         notieragent => {
 520             description => 'Cache tiering activity is suspended.',
 521             type => 'boolean',
 522             optional=> 1,
 523         },
 524     };
 525     return $possible_flags;
 526 }
 527
 528 sub get_real_flag_name {
 529     my ($flag) = @_;
 530
 531     # the 'pause' flag gets always set to both 'pauserd' and 'pausewr'
 532     # so decide that the 'pause' flag is set if we detect 'pauserd'
 533     my $flagmap = {
 534         'pause' => 'pauserd',
 535     };
 536
 537     return $flagmap->{$flag} // $flag;
 538 }
 539
 540 sub ceph_cluster_status {
 541     my ($rados) = @_;
 542     $rados = PVE::RADOS->new() if !$rados;
 543
 544     my $status = $rados->mon_command({ prefix => 'status' });
 545     $status->{health} = $rados->mon_command({ prefix => 'health', detail => 'detail' });
 546
 547     if (!exists $status->{monmap}->{mons}) { # octopus moved most info out of status, re-add
 548         $status->{monmap} = $rados->mon_command({ prefix => 'mon dump' });
 549         $status->{mgrmap} = $rados->mon_command({ prefix => 'mgr dump' });
 550     }
 551
 552     return $status;
 553 }
 554
 555 sub ecprofile_exists {
 556     my ($name, $rados) = @_;
 557     $rados = PVE::RADOS->new() if !$rados;
 558
 559     my $res = $rados->mon_command({ prefix => 'osd erasure-code-profile ls' });
 560
 561     my $profiles = { map { $_ => 1 } @$res };
 562     return $profiles->{$name};
 563 }
 564
 565 sub create_ecprofile {
 566     my ($name, $k, $m, $failure_domain, $device_class, $rados) = @_;
 567     $rados = PVE::RADOS->new() if !$rados;
 568
 569     $failure_domain = 'host' if !$failure_domain;
 570
 571     my $profile = [
 572         "crush-failure-domain=${failure_domain}",
 573         "k=${k}",
 574         "m=${m}",
 575     ];
 576
 577     push(@$profile, "crush-device-class=${device_class}") if $device_class;
 578
 579     $rados->mon_command({
 580         prefix => 'osd erasure-code-profile set',
 581         name => $name,
 582         profile => $profile,
 583     });
 584 }
 585
 586 sub destroy_ecprofile {
 587     my ($profile, $rados) = @_;
 588     $rados = PVE::RADOS->new() if !$rados;
 589
 590     my $command = {
 591         prefix => 'osd erasure-code-profile rm',
 592         name => $profile,
 593         format => 'plain',
 594     };
 595     return $rados->mon_command($command);
 596 }
 597
 598 sub get_ecprofile_name {
 599     my ($name) = @_;
 600     return "pve_ec_${name}";
 601 }
 602
 603 sub destroy_crush_rule {
 604     my ($rule, $rados) = @_;
 605     $rados = PVE::RADOS->new() if !$rados;
 606
 607     my $command = {
 608         prefix => 'osd crush rule rm',
 609         name => $rule,
 610         format => 'plain',
 611     };
 612     return $rados->mon_command($command);
 613 }
 614
 615 1;