PVE/CLI/pveceph.pm

   1 package PVE::CLI::pveceph;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Path;
   8 use IO::File;
   9 use JSON;
  10 use Data::Dumper;
  11 use LWP::UserAgent;
  12
  13 use Proxmox::RS::Subscription;
  14
  15 use PVE::SafeSyslog;
  16 use PVE::Cluster;
  17 use PVE::INotify;
  18 use PVE::RPCEnvironment;
  19 use PVE::Storage;
  20 use PVE::Tools qw(run_command);
  21 use PVE::JSONSchema qw(get_standard_option);
  22 use PVE::Ceph::Tools;
  23 use PVE::Ceph::Services;
  24 use PVE::API2::Ceph;
  25 use PVE::API2::Ceph::FS;
  26 use PVE::API2::Ceph::MDS;
  27 use PVE::API2::Ceph::MGR;
  28 use PVE::API2::Ceph::MON;
  29 use PVE::API2::Ceph::OSD;
  30
  31 use PVE::CLIHandler;
  32
  33 use base qw(PVE::CLIHandler);
  34
  35 my $nodename = PVE::INotify::nodename();
  36
  37 my $upid_exit = sub {
  38     my $upid = shift;
  39     my $status = PVE::Tools::upid_read_status($upid);
  40     exit(PVE::Tools::upid_status_is_error($status) ? -1 : 0);
  41 };
  42
  43 sub setup_environment {
  44     PVE::RPCEnvironment->setup_default_cli_env();
  45 }
  46
  47 __PACKAGE__->register_method ({
  48     name => 'purge',
  49     path => 'purge',
  50     method => 'POST',
  51     description => "Destroy ceph related data and configuration files.",
  52     parameters => {
  53         additionalProperties => 0,
  54         properties => {
  55             logs => {
  56                 description => 'Additionally purge Ceph logs, /var/log/ceph.',
  57                 type => 'boolean',
  58                 optional => 1,
  59             },
  60             crash => {
  61                 description => 'Additionally purge Ceph crash logs, /var/lib/ceph/crash.',
  62                 type => 'boolean',
  63                 optional => 1,
  64             },
  65         },
  66     },
  67     returns => { type => 'null' },
  68     code => sub {
  69         my ($param) = @_;
  70
  71         my $message;
  72         my $pools = [];
  73         my $monstat = {};
  74         my $mdsstat = {};
  75         my $osdstat = [];
  76
  77         eval {
  78             my $rados = PVE::RADOS->new();
  79             $pools = PVE::Ceph::Tools::ls_pools(undef, $rados);
  80             $monstat = PVE::Ceph::Services::get_services_info('mon', undef, $rados);
  81             $mdsstat = PVE::Ceph::Services::get_services_info('mds', undef, $rados);
  82             $osdstat = $rados->mon_command({ prefix => 'osd metadata' });
  83         };
  84         warn "Error gathering ceph info, already purged? Message: $@" if $@;
  85
  86         my $osd = grep { $_->{hostname} eq $nodename } @$osdstat;
  87         my $mds = grep { $mdsstat->{$_}->{host} eq $nodename } keys %$mdsstat;
  88         my $mon = grep { $monstat->{$_}->{host} eq $nodename } keys %$monstat;
  89
  90         # no pools = no data
  91         $message .= "- remove pools, this will !!DESTROY DATA!!\n" if @$pools;
  92         $message .= "- remove active OSD on $nodename\n" if $osd;
  93         $message .= "- remove active MDS on $nodename\n" if $mds;
  94         $message .= "- remove other MONs, $nodename is not the last MON\n"
  95             if scalar(keys %$monstat) > 1 && $mon;
  96
  97         # display all steps at once
  98         die "Unable to purge Ceph!\n\nTo continue:\n$message" if $message;
  99
 100         my $services = PVE::Ceph::Services::get_local_services();
 101         $services->{mon} = $monstat if $mon;
 102         $services->{crash}->{$nodename} = { direxists => 1 } if $param->{crash};
 103         $services->{logs}->{$nodename} = { direxists => 1 } if $param->{logs};
 104
 105         PVE::Ceph::Tools::purge_all_ceph_services($services);
 106         PVE::Ceph::Tools::purge_all_ceph_files($services);
 107
 108         return undef;
 109     }});
 110
 111 my sub has_valid_subscription {
 112     my $info = eval { Proxmox::RS::Subscription::read_subscription('/etc/subscription') } // {};
 113     warn "couldn't check subscription info - $@" if $@;
 114     return $info->{status} && $info->{status} eq 'active'; # age check?
 115 }
 116
 117 my $supported_ceph_versions = ['quincy', 'reef'];
 118 my $default_ceph_version = 'quincy';
 119
 120 __PACKAGE__->register_method ({
 121     name => 'install',
 122     path => 'install',
 123     method => 'POST',
 124     description => "Install ceph related packages.",
 125     parameters => {
 126         additionalProperties => 0,
 127         properties => {
 128             version => {
 129                 type => 'string',
 130                 enum => $supported_ceph_versions,
 131                 default => $default_ceph_version,
 132                 description => "Ceph version to install.",
 133                 optional => 1,
 134             },
 135             repository => {
 136                 type => 'string',
 137                 enum => ['enterprise', 'no-subscription', 'test'],
 138                 default => 'enterprise',
 139                 description => "Ceph repository to use.",
 140                 optional => 1,
 141             },
 142             'allow-experimental' => {
 143                 type => 'boolean',
 144                 default => 0,
 145                 optional => 1,
 146                 description => "Allow experimental versions. Use with care!",
 147             },
 148         },
 149     },
 150     returns => { type => 'null' },
 151     code => sub {
 152         my ($param) = @_;
 153
 154         my $cephver = $param->{version} || $default_ceph_version;
 155
 156         my $repo = $param->{'repository'} // 'enterprise';
 157         my $enterprise_repo = $repo eq 'enterprise';
 158         my $cdn = $enterprise_repo ? 'https://enterprise.proxmox.com' : 'http://download.proxmox.com';
 159
 160         if (has_valid_subscription()) {
 161             warn "\nNOTE: The node has an active subscription but a non-production Ceph repository selected.\n\n"
 162                 if !$enterprise_repo;
 163         } elsif ($enterprise_repo) {
 164             warn "\nWARN: Enterprise repository selected, but no active subscription!\n\n";
 165         } elsif ($repo eq 'no-subscription') {
 166             warn "\nHINT: The no-subscription repository is not the best choice for production setups.\n"
 167                 ."Proxmox recommends using the enterprise repository with a valid subscription.\n";
 168         } else {
 169             warn "\nWARN: The test repository should only be used for test setups or after consulting"
 170                 ." the official Proxmox support!\n\n"
 171         }
 172
 173         my $repolist;
 174         if ($cephver eq 'reef') {
 175             $repolist = "deb ${cdn}/debian/ceph-reef bookworm $repo\n";
 176         } elsif ($cephver eq 'quincy') {
 177             $repolist = "deb ${cdn}/debian/ceph-quincy bookworm $repo\n";
 178         } else {
 179             die "unsupported ceph version: $cephver";
 180         }
 181
 182         if (-t STDOUT && !$param->{version}) {
 183             print "This will install Ceph " . ucfirst($cephver) . " - continue (y/N)? ";
 184
 185             my $answer = <STDIN>;
 186             my $continue = defined($answer) && $answer =~ m/^\s*y(?:es)?\s*$/i;
 187
 188             die "Aborting installation as requested\n" if !$continue;
 189         }
 190
 191         PVE::Tools::file_set_contents("/etc/apt/sources.list.d/ceph.list", $repolist);
 192
 193         my $supported_re = join('|', $supported_ceph_versions->@*);
 194         warn "WARNING: installing non-default ceph release '$cephver'!\n" if $cephver !~ qr/^(?:$supported_re)$/;
 195
 196         local $ENV{DEBIAN_FRONTEND} = 'noninteractive';
 197         print "update available package list\n";
 198         eval {
 199             run_command(
 200                 ['apt-get', '-q', 'update'],
 201                 outfunc => sub {},
 202                 errfunc => sub { print STDERR "$_[0]\n" },
 203             )
 204         };
 205
 206         my @apt_install = qw(apt-get --no-install-recommends -o Dpkg::Options::=--force-confnew install --);
 207         my @ceph_packages = qw(
 208             ceph
 209             ceph-common
 210             ceph-fuse
 211             ceph-mds
 212             ceph-volume
 213             gdisk
 214             nvme-cli
 215         );
 216
 217         print "start installation\n";
 218
 219         # this flag helps to determine when apt is actually done installing (vs. partial extracing)
 220         my $install_flag_fn = PVE::Ceph::Tools::ceph_install_flag_file();
 221         open(my $install_flag, '>', $install_flag_fn) or die "could not create install flag - $!\n";
 222         close $install_flag;
 223
 224         if (system(@apt_install, @ceph_packages) != 0) {
 225             unlink $install_flag_fn or warn "could not remove Ceph installation flag - $!";
 226             die "apt failed during ceph installation ($?)\n";
 227         }
 228
 229         print "\ninstalled ceph $cephver successfully!\n";
 230         # done: drop flag file so that the PVE::Ceph::Tools check returns Ok now.
 231         unlink $install_flag_fn or warn "could not remove Ceph installation flag - $!";
 232
 233         print "\nreloading API to load new Ceph RADOS library...\n";
 234         run_command([
 235             'systemctl', 'try-reload-or-restart', 'pvedaemon.service', 'pveproxy.service'
 236         ]);
 237
 238         return undef;
 239     }});
 240
 241 __PACKAGE__->register_method ({
 242     name => 'status',
 243     path => 'status',
 244     method => 'GET',
 245     description => "Get Ceph Status.",
 246     parameters => {
 247         additionalProperties => 0,
 248     },
 249     returns => { type => 'null' },
 250     code => sub {
 251         PVE::Ceph::Tools::check_ceph_inited();
 252
 253         run_command(
 254             ['ceph', '-s'],
 255             outfunc => sub { print "$_[0]\n" },
 256             errfunc => sub { print STDERR "$_[0]\n" },
 257             timeout => 15,
 258         );
 259         return undef;
 260     }});
 261
 262 my $get_storages = sub {
 263     my ($fs, $is_default) = @_;
 264
 265     my $cfg = PVE::Storage::config();
 266
 267     my $storages = $cfg->{ids};
 268     my $res = {};
 269     foreach my $storeid (keys %$storages) {
 270         my $curr = $storages->{$storeid};
 271         next if $curr->{type} ne 'cephfs';
 272         my $cur_fs = $curr->{'fs-name'};
 273         $res->{$storeid} = $storages->{$storeid}
 274             if (!defined($cur_fs) && $is_default) || (defined($cur_fs) && $fs eq $cur_fs);
 275     }
 276
 277     return $res;
 278 };
 279
 280 __PACKAGE__->register_method ({
 281     name => 'destroyfs',
 282     path => 'destroyfs',
 283     method => 'DELETE',
 284     description => "Destroy a Ceph filesystem",
 285     parameters => {
 286         additionalProperties => 0,
 287         properties => {
 288             node => get_standard_option('pve-node'),
 289             name => {
 290                 description => "The ceph filesystem name.",
 291                 type => 'string',
 292             },
 293             'remove-storages' => {
 294                 description => "Remove all pveceph-managed storages configured for this fs.",
 295                 type => 'boolean',
 296                 optional => 1,
 297                 default => 0,
 298             },
 299             'remove-pools' => {
 300                 description => "Remove data and metadata pools configured for this fs.",
 301                 type => 'boolean',
 302                 optional => 1,
 303                 default => 0,
 304             },
 305         },
 306     },
 307     returns => { type => 'string' },
 308     code => sub {
 309         my ($param) = @_;
 310
 311         PVE::Ceph::Tools::check_ceph_inited();
 312
 313         my $rpcenv = PVE::RPCEnvironment::get();
 314         my $user = $rpcenv->get_user();
 315
 316         my $fs_name = $param->{name};
 317
 318         my $fs;
 319         my $fs_list = PVE::Ceph::Tools::ls_fs();
 320         for my $entry (@$fs_list) {
 321             next if $entry->{name} ne $fs_name;
 322             $fs = $entry;
 323             last;
 324         }
 325         die "no such cephfs '$fs_name'\n" if !$fs;
 326
 327         my $worker = sub {
 328             my $rados = PVE::RADOS->new();
 329
 330             if ($param->{'remove-storages'}) {
 331                 my $defaultfs;
 332                 my $fs_dump = $rados->mon_command({ prefix => "fs dump" });
 333                 for my $fs ($fs_dump->{filesystems}->@*) {
 334                     next if $fs->{id} != $fs_dump->{default_fscid};
 335                     $defaultfs = $fs->{mdsmap}->{fs_name};
 336                 }
 337                 warn "no default fs found, maybe not all relevant storages are removed\n"
 338                     if !defined($defaultfs);
 339
 340                 my $storages = $get_storages->($fs_name, $fs_name eq ($defaultfs // ''));
 341                 for my $storeid (keys %$storages) {
 342                     my $store = $storages->{$storeid};
 343                     if (!$store->{disable}) {
 344                         die "storage '$storeid' is not disabled, make sure to disable ".
 345                             "and unmount the storage first\n";
 346                     }
 347                 }
 348
 349                 my $err;
 350                 for my $storeid (keys %$storages) {
 351                     # skip external clusters, not managed by pveceph
 352                     next if $storages->{$storeid}->{monhost};
 353                     eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
 354                     if ($@) {
 355                         warn "failed to remove storage '$storeid': $@\n";
 356                         $err = 1;
 357                     }
 358                 }
 359                 die "failed to remove (some) storages - check log and remove manually!\n"
 360                     if $err;
 361             }
 362
 363             PVE::Ceph::Tools::destroy_fs($fs_name, $rados);
 364
 365             if ($param->{'remove-pools'}) {
 366                 warn "removing metadata pool '$fs->{metadata_pool}'\n";
 367                 eval { PVE::Ceph::Tools::destroy_pool($fs->{metadata_pool}, $rados) };
 368                 warn "$@\n" if $@;
 369
 370                 foreach my $pool ($fs->{data_pools}->@*) {
 371                     warn "removing data pool '$pool'\n";
 372                     eval { PVE::Ceph::Tools::destroy_pool($pool, $rados) };
 373                     warn "$@\n" if $@;
 374                 }
 375             }
 376
 377         };
 378         return $rpcenv->fork_worker('cephdestroyfs', $fs_name,  $user, $worker);
 379     }});
 380
 381 __PACKAGE__->register_method ({
 382     name => 'osd-details',
 383     path => 'osd-details',
 384     method => 'GET',
 385     description => "Get OSD details.",
 386     parameters => {
 387         additionalProperties => 0,
 388         properties => {
 389             node => get_standard_option('pve-node'),
 390             osdid => {
 391                 description => "ID of the OSD",
 392                 type => 'string',
 393             },
 394             verbose => {
 395                 description => "Print verbose information, same as json-pretty output format.",
 396                 type => 'boolean',
 397                 default => 0,
 398                 optional => 1,
 399             },
 400         },
 401     },
 402     returns => { type => 'object' },
 403     code => sub {
 404         my ($param) = @_;
 405
 406         PVE::Ceph::Tools::check_ceph_inited();
 407
 408         my $res = PVE::API2::Ceph::OSD->osddetails({
 409             osdid => $param->{osdid},
 410             node => $param->{node},
 411         });
 412
 413         for my $dev ($res->{devices}->@*) {
 414             $dev->{"lv-info"} = PVE::API2::Ceph::OSD->osdvolume({
 415                 osdid => $param->{osdid},
 416                 node => $param->{node},
 417                 type => $dev->{device},
 418             });
 419         }
 420         $res->{verbose} = 1 if $param->{verbose};
 421         return $res;
 422     }});
 423
 424 my $format_osddetails = sub {
 425     my ($data, $schema, $options) = @_;
 426
 427     $options->{"output-format"} //= "text";
 428
 429     if ($data->{verbose}) {
 430         $options->{"output-format"} = "json-pretty";
 431         delete $data->{verbose};
 432     }
 433
 434     if ($options->{"output-format"} eq "text") {
 435         for my $dev ($data->{devices}->@*) {
 436             my ($disk, $type, $device) = $dev->@{'physical_device', 'type', 'device'};
 437             my ($lv_size, $lv_ctime) = $dev->{'lv-info'}->@{'lv_size', 'creation_time'};
 438
 439             $data->{osd}->{$device} = "Disk: $disk, Type: $type, LV Size: $lv_size, LV Creation Time: $lv_ctime";
 440         }
 441         PVE::CLIFormatter::print_api_result($data->{osd}, $schema, undef, $options);
 442     } else {
 443         PVE::CLIFormatter::print_api_result($data, $schema, undef, $options);
 444     }
 445 };
 446
 447 our $cmddef = {
 448     init => [ 'PVE::API2::Ceph', 'init', [], { node => $nodename } ],
 449     pool => {
 450         ls => [ 'PVE::API2::Ceph::Pool', 'lspools', [], { node => $nodename }, sub {
 451             my ($data, $schema, $options) = @_;
 452             PVE::CLIFormatter::print_api_result($data, $schema,
 453                 [
 454                     'pool_name',
 455                     'size',
 456                     'min_size',
 457                     'pg_num',
 458                     'pg_num_min',
 459                     'pg_num_final',
 460                     'pg_autoscale_mode',
 461                     'target_size',
 462                     'target_size_ratio',
 463                     'crush_rule_name',
 464                     'percent_used',
 465                     'bytes_used',
 466                 ],
 467                 $options);
 468         }, $PVE::RESTHandler::standard_output_options],
 469         create => [ 'PVE::API2::Ceph::Pool', 'createpool', ['name'], { node => $nodename }],
 470         destroy => [ 'PVE::API2::Ceph::Pool', 'destroypool', ['name'], { node => $nodename } ],
 471         set => [ 'PVE::API2::Ceph::Pool', 'setpool', ['name'], { node => $nodename } ],
 472         get => [ 'PVE::API2::Ceph::Pool', 'getpool', ['name'], { node => $nodename }, sub {
 473             my ($data, $schema, $options) = @_;
 474             PVE::CLIFormatter::print_api_result($data, $schema, undef, $options);
 475         }, $PVE::RESTHandler::standard_output_options],
 476     },
 477     lspools => { alias => 'pool ls' },
 478     createpool => { alias => 'pool create' },
 479     destroypool => { alias => 'pool destroy' },
 480     fs => {
 481         create => [ 'PVE::API2::Ceph::FS', 'createfs', [], { node => $nodename }],
 482         destroy => [ __PACKAGE__, 'destroyfs', ['name'], { node => $nodename }],
 483     },
 484     osd => {
 485         create => [ 'PVE::API2::Ceph::OSD', 'createosd', ['dev'], { node => $nodename }, $upid_exit],
 486         destroy => [ 'PVE::API2::Ceph::OSD', 'destroyosd', ['osdid'], { node => $nodename }, $upid_exit],
 487         details => [
 488             __PACKAGE__, 'osd-details', ['osdid'], { node => $nodename }, $format_osddetails,
 489             $PVE::RESTHandler::standard_output_options,
 490         ],
 491     },
 492     createosd => { alias => 'osd create' },
 493     destroyosd => { alias => 'osd destroy' },
 494     mon => {
 495         create => [ 'PVE::API2::Ceph::MON', 'createmon', [], { node => $nodename }, $upid_exit],
 496         destroy => [ 'PVE::API2::Ceph::MON', 'destroymon', ['monid'], { node => $nodename }, $upid_exit],
 497     },
 498     createmon => { alias => 'mon create' },
 499     destroymon => { alias => 'mon destroy' },
 500     mgr => {
 501         create => [ 'PVE::API2::Ceph::MGR', 'createmgr', [], { node => $nodename }, $upid_exit],
 502         destroy => [ 'PVE::API2::Ceph::MGR', 'destroymgr', ['id'], { node => $nodename }, $upid_exit],
 503     },
 504     createmgr => { alias => 'mgr create' },
 505     destroymgr => { alias => 'mgr destroy' },
 506     mds => {
 507         create => [ 'PVE::API2::Ceph::MDS', 'createmds', [], { node => $nodename }, $upid_exit],
 508         destroy => [ 'PVE::API2::Ceph::MDS', 'destroymds', ['name'], { node => $nodename }, $upid_exit],
 509     },
 510     start => [ 'PVE::API2::Ceph', 'start', [], { node => $nodename }, $upid_exit],
 511     stop => [ 'PVE::API2::Ceph', 'stop', [], { node => $nodename }, $upid_exit],
 512     install => [ __PACKAGE__, 'install', [] ],
 513     purge => [  __PACKAGE__, 'purge', [] ],
 514     status => [ __PACKAGE__, 'status', []],
 515 };
 516
 517 1;