PVE/CLI/pveceph.pm

   1 package PVE::CLI::pveceph;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Path;
   8 use IO::File;
   9 use JSON;
  10 use Data::Dumper;
  11 use LWP::UserAgent;
  12
  13 use PVE::SafeSyslog;
  14 use PVE::Cluster;
  15 use PVE::INotify;
  16 use PVE::RPCEnvironment;
  17 use PVE::Storage;
  18 use PVE::Tools qw(run_command);
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::Ceph::Tools;
  21 use PVE::Ceph::Services;
  22 use PVE::API2::Ceph;
  23 use PVE::API2::Ceph::FS;
  24 use PVE::API2::Ceph::MDS;
  25 use PVE::API2::Ceph::MGR;
  26 use PVE::API2::Ceph::MON;
  27 use PVE::API2::Ceph::OSD;
  28
  29 use PVE::CLIHandler;
  30
  31 use base qw(PVE::CLIHandler);
  32
  33 my $nodename = PVE::INotify::nodename();
  34
  35 my $upid_exit = sub {
  36     my $upid = shift;
  37     my $status = PVE::Tools::upid_read_status($upid);
  38     exit(PVE::Tools::upid_status_is_error($status) ? -1 : 0);
  39 };
  40
  41 sub setup_environment {
  42     PVE::RPCEnvironment->setup_default_cli_env();
  43 }
  44
  45 __PACKAGE__->register_method ({
  46     name => 'purge',
  47     path => 'purge',
  48     method => 'POST',
  49     description => "Destroy ceph related data and configuration files.",
  50     parameters => {
  51         additionalProperties => 0,
  52         properties => {
  53             logs => {
  54                 description => 'Additionally purge Ceph logs, /var/log/ceph.',
  55                 type => 'boolean',
  56                 optional => 1,
  57             },
  58             crash => {
  59                 description => 'Additionally purge Ceph crash logs, /var/lib/ceph/crash.',
  60                 type => 'boolean',
  61                 optional => 1,
  62             },
  63         },
  64     },
  65     returns => { type => 'null' },
  66     code => sub {
  67         my ($param) = @_;
  68
  69         my $message;
  70         my $pools = [];
  71         my $monstat = {};
  72         my $mdsstat = {};
  73         my $osdstat = [];
  74
  75         eval {
  76             my $rados = PVE::RADOS->new();
  77             $pools = PVE::Ceph::Tools::ls_pools(undef, $rados);
  78             $monstat = PVE::Ceph::Services::get_services_info('mon', undef, $rados);
  79             $mdsstat = PVE::Ceph::Services::get_services_info('mds', undef, $rados);
  80             $osdstat = $rados->mon_command({ prefix => 'osd metadata' });
  81         };
  82         warn "Error gathering ceph info, already purged? Message: $@" if $@;
  83
  84         my $osd = grep { $_->{hostname} eq $nodename } @$osdstat;
  85         my $mds = grep { $mdsstat->{$_}->{host} eq $nodename } keys %$mdsstat;
  86         my $mon = grep { $monstat->{$_}->{host} eq $nodename } keys %$monstat;
  87
  88         # no pools = no data
  89         $message .= "- remove pools, this will !!DESTROY DATA!!\n" if @$pools;
  90         $message .= "- remove active OSD on $nodename\n" if $osd;
  91         $message .= "- remove active MDS on $nodename\n" if $mds;
  92         $message .= "- remove other MONs, $nodename is not the last MON\n"
  93             if scalar(keys %$monstat) > 1 && $mon;
  94
  95         # display all steps at once
  96         die "Unable to purge Ceph!\n\nTo continue:\n$message" if $message;
  97
  98         my $services = PVE::Ceph::Services::get_local_services();
  99         $services->{mon} = $monstat if $mon;
 100         $services->{crash}->{$nodename} = { direxists => 1 } if $param->{crash};
 101         $services->{logs}->{$nodename} = { direxists => 1 } if $param->{logs};
 102
 103         PVE::Ceph::Tools::purge_all_ceph_services($services);
 104         PVE::Ceph::Tools::purge_all_ceph_files($services);
 105
 106         return undef;
 107     }});
 108
 109 my $supported_ceph_versions = ['octopus', 'pacific', 'quincy'];
 110 my $default_ceph_version = 'pacific';
 111
 112 __PACKAGE__->register_method ({
 113     name => 'install',
 114     path => 'install',
 115     method => 'POST',
 116     description => "Install ceph related packages.",
 117     parameters => {
 118         additionalProperties => 0,
 119         properties => {
 120             version => {
 121                 type => 'string',
 122                 enum => $supported_ceph_versions,
 123                 default => $default_ceph_version,
 124                 description => "Ceph version to install.",
 125                 optional => 1,
 126             },
 127             'allow-experimental' => {
 128                 type => 'boolean',
 129                 default => 0,
 130                 optional => 1,
 131                 description => "Allow experimental versions. Use with care!",
 132             },
 133             'test-repository' => {
 134                 type => 'boolean',
 135                 default => 0,
 136                 optional => 1,
 137                 description => "Use the test, not the main repository. Use with care!",
 138             },
 139         },
 140     },
 141     returns => { type => 'null' },
 142     code => sub {
 143         my ($param) = @_;
 144
 145         my $cephver = $param->{version} || $default_ceph_version;
 146
 147         my $repo = $param->{'test-repository'} ? 'test' : 'main';
 148
 149         my $repolist;
 150         if ($cephver eq 'octopus') {
 151             warn "Ceph Octopus will go EOL after 2022-07\n";
 152             $repolist = "deb http://download.proxmox.com/debian/ceph-octopus bullseye $repo\n";
 153         } elsif ($cephver eq 'pacific') {
 154             $repolist = "deb http://download.proxmox.com/debian/ceph-pacific bullseye $repo\n";
 155         } elsif ($cephver eq 'quincy') {
 156             $repolist = "deb http://download.proxmox.com/debian/ceph-quincy bullseye $repo\n";
 157         } else {
 158             die "unsupported ceph version: $cephver";
 159         }
 160         PVE::Tools::file_set_contents("/etc/apt/sources.list.d/ceph.list", $repolist);
 161
 162         my $supported_re = join('|', $supported_ceph_versions->@*);
 163         warn "WARNING: installing non-default ceph release '$cephver'!\n" if $cephver !~ qr/^(?:$supported_re)$/;
 164
 165         local $ENV{DEBIAN_FRONTEND} = 'noninteractive';
 166         print "update available package list\n";
 167         eval {
 168             run_command(
 169                 ['apt-get', '-q', 'update'],
 170                 outfunc => sub {},
 171                 errfunc => sub { print STDERR "$_[0]\n" },
 172             )
 173         };
 174
 175         my @apt_install = qw(apt-get --no-install-recommends -o Dpkg::Options::=--force-confnew install --);
 176         my @ceph_packages = qw(
 177             ceph
 178             ceph-common
 179             ceph-mds
 180             ceph-fuse
 181             gdisk
 182             nvme-cli
 183         );
 184
 185         # got split out with quincy and is required by PVE tooling, conditionally exclude it for older
 186         # FIXME: remove condition with PVE 8.0, i.e., once we only support quincy+ new installations
 187         if ($cephver ne 'octopus' and $cephver ne 'pacific') {
 188             push @ceph_packages, 'ceph-volume';
 189         }
 190
 191         print "start installation\n";
 192
 193         # this flag helps to determine when apt is actually done installing (vs. partial extracing)
 194         my $install_flag_fn = PVE::Ceph::Tools::ceph_install_flag_file();
 195         open(my $install_flag, '>', $install_flag_fn) or die "could not create install flag - $!\n";
 196         close $install_flag;
 197
 198         if (system(@apt_install, @ceph_packages) != 0) {
 199             unlink $install_flag_fn or warn "could not remove Ceph installation flag - $!";
 200             die "apt failed during ceph installation ($?)\n";
 201         }
 202
 203         print "\ninstalled ceph $cephver successfully!\n";
 204         # done: drop flag file so that the PVE::Ceph::Tools check returns Ok now.
 205         unlink $install_flag_fn or warn "could not remove Ceph installation flag - $!";
 206
 207         print "\nreloading API to load new Ceph RADOS library...\n";
 208         run_command([
 209             'systemctl', 'try-reload-or-restart', 'pvedaemon.service', 'pveproxy.service'
 210         ]);
 211
 212         return undef;
 213     }});
 214
 215 __PACKAGE__->register_method ({
 216     name => 'status',
 217     path => 'status',
 218     method => 'GET',
 219     description => "Get Ceph Status.",
 220     parameters => {
 221         additionalProperties => 0,
 222     },
 223     returns => { type => 'null' },
 224     code => sub {
 225         PVE::Ceph::Tools::check_ceph_inited();
 226
 227         run_command(
 228             ['ceph', '-s'],
 229             outfunc => sub { print "$_[0]\n" },
 230             errfunc => sub { print STDERR "$_[0]\n" },
 231             timeout => 15,
 232         );
 233         return undef;
 234     }});
 235
 236 my $get_storages = sub {
 237     my ($fs, $is_default) = @_;
 238
 239     my $cfg = PVE::Storage::config();
 240
 241     my $storages = $cfg->{ids};
 242     my $res = {};
 243     foreach my $storeid (keys %$storages) {
 244         my $curr = $storages->{$storeid};
 245         next if $curr->{type} ne 'cephfs';
 246         my $cur_fs = $curr->{'fs-name'};
 247         $res->{$storeid} = $storages->{$storeid}
 248             if (!defined($cur_fs) && $is_default) || (defined($cur_fs) && $fs eq $cur_fs);
 249     }
 250
 251     return $res;
 252 };
 253
 254 __PACKAGE__->register_method ({
 255     name => 'destroyfs',
 256     path => 'destroyfs',
 257     method => 'DELETE',
 258     description => "Destroy a Ceph filesystem",
 259     parameters => {
 260         additionalProperties => 0,
 261         properties => {
 262             node => get_standard_option('pve-node'),
 263             name => {
 264                 description => "The ceph filesystem name.",
 265                 type => 'string',
 266             },
 267             'remove-storages' => {
 268                 description => "Remove all pveceph-managed storages configured for this fs.",
 269                 type => 'boolean',
 270                 optional => 1,
 271                 default => 0,
 272             },
 273             'remove-pools' => {
 274                 description => "Remove data and metadata pools configured for this fs.",
 275                 type => 'boolean',
 276                 optional => 1,
 277                 default => 0,
 278             },
 279         },
 280     },
 281     returns => { type => 'string' },
 282     code => sub {
 283         my ($param) = @_;
 284
 285         PVE::Ceph::Tools::check_ceph_inited();
 286
 287         my $rpcenv = PVE::RPCEnvironment::get();
 288         my $user = $rpcenv->get_user();
 289
 290         my $fs_name = $param->{name};
 291
 292         my $fs;
 293         my $fs_list = PVE::Ceph::Tools::ls_fs();
 294         for my $entry (@$fs_list) {
 295             next if $entry->{name} ne $fs_name;
 296             $fs = $entry;
 297             last;
 298         }
 299         die "no such cephfs '$fs_name'\n" if !$fs;
 300
 301         my $worker = sub {
 302             my $rados = PVE::RADOS->new();
 303
 304             if ($param->{'remove-storages'}) {
 305                 my $defaultfs;
 306                 my $fs_dump = $rados->mon_command({ prefix => "fs dump" });
 307                 for my $fs ($fs_dump->{filesystems}->@*) {
 308                     next if $fs->{id} != $fs_dump->{default_fscid};
 309                     $defaultfs = $fs->{mdsmap}->{fs_name};
 310                 }
 311                 warn "no default fs found, maybe not all relevant storages are removed\n"
 312                     if !defined($defaultfs);
 313
 314                 my $storages = $get_storages->($fs_name, $fs_name eq ($defaultfs // ''));
 315                 for my $storeid (keys %$storages) {
 316                     my $store = $storages->{$storeid};
 317                     if (!$store->{disable}) {
 318                         die "storage '$storeid' is not disabled, make sure to disable ".
 319                             "and unmount the storage first\n";
 320                     }
 321                 }
 322
 323                 my $err;
 324                 for my $storeid (keys %$storages) {
 325                     # skip external clusters, not managed by pveceph
 326                     next if $storages->{$storeid}->{monhost};
 327                     eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
 328                     if ($@) {
 329                         warn "failed to remove storage '$storeid': $@\n";
 330                         $err = 1;
 331                     }
 332                 }
 333                 die "failed to remove (some) storages - check log and remove manually!\n"
 334                     if $err;
 335             }
 336
 337             PVE::Ceph::Tools::destroy_fs($fs_name, $rados);
 338
 339             if ($param->{'remove-pools'}) {
 340                 warn "removing metadata pool '$fs->{metadata_pool}'\n";
 341                 eval { PVE::Ceph::Tools::destroy_pool($fs->{metadata_pool}, $rados) };
 342                 warn "$@\n" if $@;
 343
 344                 foreach my $pool ($fs->{data_pools}->@*) {
 345                     warn "removing data pool '$pool'\n";
 346                     eval { PVE::Ceph::Tools::destroy_pool($pool, $rados) };
 347                     warn "$@\n" if $@;
 348                 }
 349             }
 350
 351         };
 352         return $rpcenv->fork_worker('cephdestroyfs', $fs_name,  $user, $worker);
 353     }});
 354
 355 our $cmddef = {
 356     init => [ 'PVE::API2::Ceph', 'init', [], { node => $nodename } ],
 357     pool => {
 358         ls => [ 'PVE::API2::Ceph::Pool', 'lspools', [], { node => $nodename }, sub {
 359             my ($data, $schema, $options) = @_;
 360             PVE::CLIFormatter::print_api_result($data, $schema,
 361                 [
 362                     'pool_name',
 363                     'size',
 364                     'min_size',
 365                     'pg_num',
 366                     'pg_num_min',
 367                     'pg_num_final',
 368                     'pg_autoscale_mode',
 369                     'target_size',
 370                     'target_size_ratio',
 371                     'crush_rule_name',
 372                     'percent_used',
 373                     'bytes_used',
 374                 ],
 375                 $options);
 376         }, $PVE::RESTHandler::standard_output_options],
 377         create => [ 'PVE::API2::Ceph::Pool', 'createpool', ['name'], { node => $nodename }],
 378         destroy => [ 'PVE::API2::Ceph::Pool', 'destroypool', ['name'], { node => $nodename } ],
 379         set => [ 'PVE::API2::Ceph::Pool', 'setpool', ['name'], { node => $nodename } ],
 380         get => [ 'PVE::API2::Ceph::Pool', 'getpool', ['name'], { node => $nodename }, sub {
 381             my ($data, $schema, $options) = @_;
 382             PVE::CLIFormatter::print_api_result($data, $schema, undef, $options);
 383         }, $PVE::RESTHandler::standard_output_options],
 384     },
 385     lspools => { alias => 'pool ls' },
 386     createpool => { alias => 'pool create' },
 387     destroypool => { alias => 'pool destroy' },
 388     fs => {
 389         create => [ 'PVE::API2::Ceph::FS', 'createfs', [], { node => $nodename }],
 390         destroy => [ __PACKAGE__, 'destroyfs', ['name'], { node => $nodename }],
 391     },
 392     osd => {
 393         create => [ 'PVE::API2::Ceph::OSD', 'createosd', ['dev'], { node => $nodename }, $upid_exit],
 394         destroy => [ 'PVE::API2::Ceph::OSD', 'destroyosd', ['osdid'], { node => $nodename }, $upid_exit],
 395     },
 396     createosd => { alias => 'osd create' },
 397     destroyosd => { alias => 'osd destroy' },
 398     mon => {
 399         create => [ 'PVE::API2::Ceph::MON', 'createmon', [], { node => $nodename }, $upid_exit],
 400         destroy => [ 'PVE::API2::Ceph::MON', 'destroymon', ['monid'], { node => $nodename }, $upid_exit],
 401     },
 402     createmon => { alias => 'mon create' },
 403     destroymon => { alias => 'mon destroy' },
 404     mgr => {
 405         create => [ 'PVE::API2::Ceph::MGR', 'createmgr', [], { node => $nodename }, $upid_exit],
 406         destroy => [ 'PVE::API2::Ceph::MGR', 'destroymgr', ['id'], { node => $nodename }, $upid_exit],
 407     },
 408     createmgr => { alias => 'mgr create' },
 409     destroymgr => { alias => 'mgr destroy' },
 410     mds => {
 411         create => [ 'PVE::API2::Ceph::MDS', 'createmds', [], { node => $nodename }, $upid_exit],
 412         destroy => [ 'PVE::API2::Ceph::MDS', 'destroymds', ['name'], { node => $nodename }, $upid_exit],
 413     },
 414     start => [ 'PVE::API2::Ceph', 'start', [], { node => $nodename }, $upid_exit],
 415     stop => [ 'PVE::API2::Ceph', 'stop', [], { node => $nodename }, $upid_exit],
 416     install => [ __PACKAGE__, 'install', [] ],
 417     purge => [  __PACKAGE__, 'purge', [] ],
 418     status => [ __PACKAGE__, 'status', []],
 419 };
 420
 421 1;