PVE/CLI/pveceph.pm

   1 package PVE::CLI::pveceph;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Path;
   8 use IO::File;
   9 use JSON;
  10 use Data::Dumper;
  11 use LWP::UserAgent;
  12
  13 use Proxmox::RS::Subscription;
  14
  15 use PVE::SafeSyslog;
  16 use PVE::Cluster;
  17 use PVE::INotify;
  18 use PVE::RPCEnvironment;
  19 use PVE::Storage;
  20 use PVE::Tools qw(run_command);
  21 use PVE::JSONSchema qw(get_standard_option);
  22 use PVE::Ceph::Tools;
  23 use PVE::Ceph::Services;
  24 use PVE::API2::Ceph;
  25 use PVE::API2::Ceph::FS;
  26 use PVE::API2::Ceph::MDS;
  27 use PVE::API2::Ceph::MGR;
  28 use PVE::API2::Ceph::MON;
  29 use PVE::API2::Ceph::OSD;
  30
  31 use PVE::CLIHandler;
  32
  33 use base qw(PVE::CLIHandler);
  34
  35 my $nodename = PVE::INotify::nodename();
  36
  37 my $upid_exit = sub {
  38     my $upid = shift;
  39     my $status = PVE::Tools::upid_read_status($upid);
  40     exit(PVE::Tools::upid_status_is_error($status) ? -1 : 0);
  41 };
  42
  43 sub setup_environment {
  44     PVE::RPCEnvironment->setup_default_cli_env();
  45 }
  46
  47 __PACKAGE__->register_method ({
  48     name => 'purge',
  49     path => 'purge',
  50     method => 'POST',
  51     description => "Destroy ceph related data and configuration files.",
  52     parameters => {
  53         additionalProperties => 0,
  54         properties => {
  55             logs => {
  56                 description => 'Additionally purge Ceph logs, /var/log/ceph.',
  57                 type => 'boolean',
  58                 optional => 1,
  59             },
  60             crash => {
  61                 description => 'Additionally purge Ceph crash logs, /var/lib/ceph/crash.',
  62                 type => 'boolean',
  63                 optional => 1,
  64             },
  65         },
  66     },
  67     returns => { type => 'null' },
  68     code => sub {
  69         my ($param) = @_;
  70
  71         my $message;
  72         my $pools = [];
  73         my $monstat = {};
  74         my $mdsstat = {};
  75         my $osdstat = [];
  76
  77         eval {
  78             my $rados = PVE::RADOS->new();
  79             $pools = PVE::Ceph::Tools::ls_pools(undef, $rados);
  80             $monstat = PVE::Ceph::Services::get_services_info('mon', undef, $rados);
  81             $mdsstat = PVE::Ceph::Services::get_services_info('mds', undef, $rados);
  82             $osdstat = $rados->mon_command({ prefix => 'osd metadata' });
  83         };
  84         warn "Error gathering ceph info, already purged? Message: $@" if $@;
  85
  86         my $osd = grep { $_->{hostname} eq $nodename } @$osdstat;
  87         my $mds = grep { $mdsstat->{$_}->{host} eq $nodename } keys %$mdsstat;
  88         my $mon = grep { $monstat->{$_}->{host} eq $nodename } keys %$monstat;
  89
  90         # no pools = no data
  91         $message .= "- remove pools, this will !!DESTROY DATA!!\n" if @$pools;
  92         $message .= "- remove active OSD on $nodename\n" if $osd;
  93         $message .= "- remove active MDS on $nodename\n" if $mds;
  94         $message .= "- remove other MONs, $nodename is not the last MON\n"
  95             if scalar(keys %$monstat) > 1 && $mon;
  96
  97         # display all steps at once
  98         die "Unable to purge Ceph!\n\nTo continue:\n$message" if $message;
  99
 100         my $services = PVE::Ceph::Services::get_local_services();
 101         $services->{mon} = $monstat if $mon;
 102         $services->{crash}->{$nodename} = { direxists => 1 } if $param->{crash};
 103         $services->{logs}->{$nodename} = { direxists => 1 } if $param->{logs};
 104
 105         PVE::Ceph::Tools::purge_all_ceph_services($services);
 106         PVE::Ceph::Tools::purge_all_ceph_files($services);
 107
 108         return undef;
 109     }});
 110
 111 my sub has_valid_subscription {
 112     my $info = eval { Proxmox::RS::Subscription::read_subscription('/etc/subscription') } // {};
 113     warn "couldn't check subscription info - $@" if $@;
 114     return $info->{status} && $info->{status} eq 'active'; # age check?
 115 }
 116
 117 my $supported_ceph_versions = ['quincy'];
 118 my $default_ceph_version = 'quincy';
 119
 120 __PACKAGE__->register_method ({
 121     name => 'install',
 122     path => 'install',
 123     method => 'POST',
 124     description => "Install ceph related packages.",
 125     parameters => {
 126         additionalProperties => 0,
 127         properties => {
 128             version => {
 129                 type => 'string',
 130                 enum => $supported_ceph_versions,
 131                 default => $default_ceph_version,
 132                 description => "Ceph version to install.",
 133                 optional => 1,
 134             },
 135             repository => {
 136                 type => 'string',
 137                 enum => ['enterprise', 'no-subscription', 'test'],
 138                 default => 'enterprise',
 139                 description => "Ceph repository to use.",
 140                 optional => 1,
 141             },
 142             'allow-experimental' => {
 143                 type => 'boolean',
 144                 default => 0,
 145                 optional => 1,
 146                 description => "Allow experimental versions. Use with care!",
 147             },
 148         },
 149     },
 150     returns => { type => 'null' },
 151     code => sub {
 152         my ($param) = @_;
 153
 154         my $cephver = $param->{version} || $default_ceph_version;
 155
 156         my $repo = $param->{'repository'} // 'enterprise';
 157         my $enterprise_repo = $repo eq 'enterprise';
 158         my $cdn = $enterprise_repo ? 'https://enterprise.proxmox.com' : 'http://download.proxmox.com';
 159
 160         if (has_valid_subscription()) {
 161             warn "\nNOTE: The node has an active subscription but a non-production Ceph repository selected.\n\n"
 162                 if !$enterprise_repo;
 163         } elsif ($enterprise_repo) {
 164             warn "\nWARN: Enterprise repository selected, but no active subscription!\n\n";
 165         } elsif ($repo eq 'no-subscription') {
 166             warn "\nHINT: The no-subscription repository is not the best choice for production setups.\n"
 167                 ."Proxmox recommends using the enterprise repository with a valid subscription.\n";
 168         } else {
 169             warn "\nWARN: The test repository should only be used for test setups or after consulting"
 170                 ." the official Proxmox support!\n\n"
 171         }
 172
 173         my $repolist;
 174         if ($cephver eq 'quincy') {
 175             $repolist = "deb ${cdn}/debian/ceph-quincy bookworm $repo\n";
 176         } else {
 177             die "unsupported ceph version: $cephver";
 178         }
 179         PVE::Tools::file_set_contents("/etc/apt/sources.list.d/ceph.list", $repolist);
 180
 181         my $supported_re = join('|', $supported_ceph_versions->@*);
 182         warn "WARNING: installing non-default ceph release '$cephver'!\n" if $cephver !~ qr/^(?:$supported_re)$/;
 183
 184         local $ENV{DEBIAN_FRONTEND} = 'noninteractive';
 185         print "update available package list\n";
 186         eval {
 187             run_command(
 188                 ['apt-get', '-q', 'update'],
 189                 outfunc => sub {},
 190                 errfunc => sub { print STDERR "$_[0]\n" },
 191             )
 192         };
 193
 194         my @apt_install = qw(apt-get --no-install-recommends -o Dpkg::Options::=--force-confnew install --);
 195         my @ceph_packages = qw(
 196             ceph
 197             ceph-common
 198             ceph-fuse
 199             ceph-mds
 200             ceph-volume
 201             gdisk
 202             nvme-cli
 203         );
 204
 205         print "start installation\n";
 206
 207         # this flag helps to determine when apt is actually done installing (vs. partial extracing)
 208         my $install_flag_fn = PVE::Ceph::Tools::ceph_install_flag_file();
 209         open(my $install_flag, '>', $install_flag_fn) or die "could not create install flag - $!\n";
 210         close $install_flag;
 211
 212         if (system(@apt_install, @ceph_packages) != 0) {
 213             unlink $install_flag_fn or warn "could not remove Ceph installation flag - $!";
 214             die "apt failed during ceph installation ($?)\n";
 215         }
 216
 217         print "\ninstalled ceph $cephver successfully!\n";
 218         # done: drop flag file so that the PVE::Ceph::Tools check returns Ok now.
 219         unlink $install_flag_fn or warn "could not remove Ceph installation flag - $!";
 220
 221         print "\nreloading API to load new Ceph RADOS library...\n";
 222         run_command([
 223             'systemctl', 'try-reload-or-restart', 'pvedaemon.service', 'pveproxy.service'
 224         ]);
 225
 226         return undef;
 227     }});
 228
 229 __PACKAGE__->register_method ({
 230     name => 'status',
 231     path => 'status',
 232     method => 'GET',
 233     description => "Get Ceph Status.",
 234     parameters => {
 235         additionalProperties => 0,
 236     },
 237     returns => { type => 'null' },
 238     code => sub {
 239         PVE::Ceph::Tools::check_ceph_inited();
 240
 241         run_command(
 242             ['ceph', '-s'],
 243             outfunc => sub { print "$_[0]\n" },
 244             errfunc => sub { print STDERR "$_[0]\n" },
 245             timeout => 15,
 246         );
 247         return undef;
 248     }});
 249
 250 my $get_storages = sub {
 251     my ($fs, $is_default) = @_;
 252
 253     my $cfg = PVE::Storage::config();
 254
 255     my $storages = $cfg->{ids};
 256     my $res = {};
 257     foreach my $storeid (keys %$storages) {
 258         my $curr = $storages->{$storeid};
 259         next if $curr->{type} ne 'cephfs';
 260         my $cur_fs = $curr->{'fs-name'};
 261         $res->{$storeid} = $storages->{$storeid}
 262             if (!defined($cur_fs) && $is_default) || (defined($cur_fs) && $fs eq $cur_fs);
 263     }
 264
 265     return $res;
 266 };
 267
 268 __PACKAGE__->register_method ({
 269     name => 'destroyfs',
 270     path => 'destroyfs',
 271     method => 'DELETE',
 272     description => "Destroy a Ceph filesystem",
 273     parameters => {
 274         additionalProperties => 0,
 275         properties => {
 276             node => get_standard_option('pve-node'),
 277             name => {
 278                 description => "The ceph filesystem name.",
 279                 type => 'string',
 280             },
 281             'remove-storages' => {
 282                 description => "Remove all pveceph-managed storages configured for this fs.",
 283                 type => 'boolean',
 284                 optional => 1,
 285                 default => 0,
 286             },
 287             'remove-pools' => {
 288                 description => "Remove data and metadata pools configured for this fs.",
 289                 type => 'boolean',
 290                 optional => 1,
 291                 default => 0,
 292             },
 293         },
 294     },
 295     returns => { type => 'string' },
 296     code => sub {
 297         my ($param) = @_;
 298
 299         PVE::Ceph::Tools::check_ceph_inited();
 300
 301         my $rpcenv = PVE::RPCEnvironment::get();
 302         my $user = $rpcenv->get_user();
 303
 304         my $fs_name = $param->{name};
 305
 306         my $fs;
 307         my $fs_list = PVE::Ceph::Tools::ls_fs();
 308         for my $entry (@$fs_list) {
 309             next if $entry->{name} ne $fs_name;
 310             $fs = $entry;
 311             last;
 312         }
 313         die "no such cephfs '$fs_name'\n" if !$fs;
 314
 315         my $worker = sub {
 316             my $rados = PVE::RADOS->new();
 317
 318             if ($param->{'remove-storages'}) {
 319                 my $defaultfs;
 320                 my $fs_dump = $rados->mon_command({ prefix => "fs dump" });
 321                 for my $fs ($fs_dump->{filesystems}->@*) {
 322                     next if $fs->{id} != $fs_dump->{default_fscid};
 323                     $defaultfs = $fs->{mdsmap}->{fs_name};
 324                 }
 325                 warn "no default fs found, maybe not all relevant storages are removed\n"
 326                     if !defined($defaultfs);
 327
 328                 my $storages = $get_storages->($fs_name, $fs_name eq ($defaultfs // ''));
 329                 for my $storeid (keys %$storages) {
 330                     my $store = $storages->{$storeid};
 331                     if (!$store->{disable}) {
 332                         die "storage '$storeid' is not disabled, make sure to disable ".
 333                             "and unmount the storage first\n";
 334                     }
 335                 }
 336
 337                 my $err;
 338                 for my $storeid (keys %$storages) {
 339                     # skip external clusters, not managed by pveceph
 340                     next if $storages->{$storeid}->{monhost};
 341                     eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
 342                     if ($@) {
 343                         warn "failed to remove storage '$storeid': $@\n";
 344                         $err = 1;
 345                     }
 346                 }
 347                 die "failed to remove (some) storages - check log and remove manually!\n"
 348                     if $err;
 349             }
 350
 351             PVE::Ceph::Tools::destroy_fs($fs_name, $rados);
 352
 353             if ($param->{'remove-pools'}) {
 354                 warn "removing metadata pool '$fs->{metadata_pool}'\n";
 355                 eval { PVE::Ceph::Tools::destroy_pool($fs->{metadata_pool}, $rados) };
 356                 warn "$@\n" if $@;
 357
 358                 foreach my $pool ($fs->{data_pools}->@*) {
 359                     warn "removing data pool '$pool'\n";
 360                     eval { PVE::Ceph::Tools::destroy_pool($pool, $rados) };
 361                     warn "$@\n" if $@;
 362                 }
 363             }
 364
 365         };
 366         return $rpcenv->fork_worker('cephdestroyfs', $fs_name,  $user, $worker);
 367     }});
 368
 369 our $cmddef = {
 370     init => [ 'PVE::API2::Ceph', 'init', [], { node => $nodename } ],
 371     pool => {
 372         ls => [ 'PVE::API2::Ceph::Pool', 'lspools', [], { node => $nodename }, sub {
 373             my ($data, $schema, $options) = @_;
 374             PVE::CLIFormatter::print_api_result($data, $schema,
 375                 [
 376                     'pool_name',
 377                     'size',
 378                     'min_size',
 379                     'pg_num',
 380                     'pg_num_min',
 381                     'pg_num_final',
 382                     'pg_autoscale_mode',
 383                     'target_size',
 384                     'target_size_ratio',
 385                     'crush_rule_name',
 386                     'percent_used',
 387                     'bytes_used',
 388                 ],
 389                 $options);
 390         }, $PVE::RESTHandler::standard_output_options],
 391         create => [ 'PVE::API2::Ceph::Pool', 'createpool', ['name'], { node => $nodename }],
 392         destroy => [ 'PVE::API2::Ceph::Pool', 'destroypool', ['name'], { node => $nodename } ],
 393         set => [ 'PVE::API2::Ceph::Pool', 'setpool', ['name'], { node => $nodename } ],
 394         get => [ 'PVE::API2::Ceph::Pool', 'getpool', ['name'], { node => $nodename }, sub {
 395             my ($data, $schema, $options) = @_;
 396             PVE::CLIFormatter::print_api_result($data, $schema, undef, $options);
 397         }, $PVE::RESTHandler::standard_output_options],
 398     },
 399     lspools => { alias => 'pool ls' },
 400     createpool => { alias => 'pool create' },
 401     destroypool => { alias => 'pool destroy' },
 402     fs => {
 403         create => [ 'PVE::API2::Ceph::FS', 'createfs', [], { node => $nodename }],
 404         destroy => [ __PACKAGE__, 'destroyfs', ['name'], { node => $nodename }],
 405     },
 406     osd => {
 407         create => [ 'PVE::API2::Ceph::OSD', 'createosd', ['dev'], { node => $nodename }, $upid_exit],
 408         destroy => [ 'PVE::API2::Ceph::OSD', 'destroyosd', ['osdid'], { node => $nodename }, $upid_exit],
 409     },
 410     createosd => { alias => 'osd create' },
 411     destroyosd => { alias => 'osd destroy' },
 412     mon => {
 413         create => [ 'PVE::API2::Ceph::MON', 'createmon', [], { node => $nodename }, $upid_exit],
 414         destroy => [ 'PVE::API2::Ceph::MON', 'destroymon', ['monid'], { node => $nodename }, $upid_exit],
 415     },
 416     createmon => { alias => 'mon create' },
 417     destroymon => { alias => 'mon destroy' },
 418     mgr => {
 419         create => [ 'PVE::API2::Ceph::MGR', 'createmgr', [], { node => $nodename }, $upid_exit],
 420         destroy => [ 'PVE::API2::Ceph::MGR', 'destroymgr', ['id'], { node => $nodename }, $upid_exit],
 421     },
 422     createmgr => { alias => 'mgr create' },
 423     destroymgr => { alias => 'mgr destroy' },
 424     mds => {
 425         create => [ 'PVE::API2::Ceph::MDS', 'createmds', [], { node => $nodename }, $upid_exit],
 426         destroy => [ 'PVE::API2::Ceph::MDS', 'destroymds', ['name'], { node => $nodename }, $upid_exit],
 427     },
 428     start => [ 'PVE::API2::Ceph', 'start', [], { node => $nodename }, $upid_exit],
 429     stop => [ 'PVE::API2::Ceph', 'stop', [], { node => $nodename }, $upid_exit],
 430     install => [ __PACKAGE__, 'install', [] ],
 431     purge => [  __PACKAGE__, 'purge', [] ],
 432     status => [ __PACKAGE__, 'status', []],
 433 };
 434
 435 1;