PVE/CLI/pveceph.pm

   1 package PVE::CLI::pveceph;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Path;
   8 use IO::File;
   9 use JSON;
  10 use Data::Dumper;
  11 use LWP::UserAgent;
  12
  13 use PVE::SafeSyslog;
  14 use PVE::Cluster;
  15 use PVE::INotify;
  16 use PVE::RPCEnvironment;
  17 use PVE::Storage;
  18 use PVE::Tools qw(run_command);
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::Ceph::Tools;
  21 use PVE::Ceph::Services;
  22 use PVE::API2::Ceph;
  23 use PVE::API2::Ceph::FS;
  24 use PVE::API2::Ceph::MDS;
  25 use PVE::API2::Ceph::MGR;
  26 use PVE::API2::Ceph::MON;
  27 use PVE::API2::Ceph::OSD;
  28
  29 use PVE::CLIHandler;
  30
  31 use base qw(PVE::CLIHandler);
  32
  33 my $nodename = PVE::INotify::nodename();
  34
  35 my $upid_exit = sub {
  36     my $upid = shift;
  37     my $status = PVE::Tools::upid_read_status($upid);
  38     exit(PVE::Tools::upid_status_is_error($status) ? -1 : 0);
  39 };
  40
  41 sub setup_environment {
  42     PVE::RPCEnvironment->setup_default_cli_env();
  43 }
  44
  45 __PACKAGE__->register_method ({
  46     name => 'purge',
  47     path => 'purge',
  48     method => 'POST',
  49     description => "Destroy ceph related data and configuration files.",
  50     parameters => {
  51         additionalProperties => 0,
  52         properties => {
  53             logs => {
  54                 description => 'Additionally purge Ceph logs, /var/log/ceph.',
  55                 type => 'boolean',
  56                 optional => 1,
  57             },
  58             crash => {
  59                 description => 'Additionally purge Ceph crash logs, /var/lib/ceph/crash.',
  60                 type => 'boolean',
  61                 optional => 1,
  62             },
  63         },
  64     },
  65     returns => { type => 'null' },
  66     code => sub {
  67         my ($param) = @_;
  68
  69         my $message;
  70         my $pools = [];
  71         my $monstat = {};
  72         my $mdsstat = {};
  73         my $osdstat = [];
  74
  75         eval {
  76             my $rados = PVE::RADOS->new();
  77             $pools = PVE::Ceph::Tools::ls_pools(undef, $rados);
  78             $monstat = PVE::Ceph::Services::get_services_info('mon', undef, $rados);
  79             $mdsstat = PVE::Ceph::Services::get_services_info('mds', undef, $rados);
  80             $osdstat = $rados->mon_command({ prefix => 'osd metadata' });
  81         };
  82         warn "Error gathering ceph info, already purged? Message: $@" if $@;
  83
  84         my $osd = grep { $_->{hostname} eq $nodename } @$osdstat;
  85         my $mds = grep { $mdsstat->{$_}->{host} eq $nodename } keys %$mdsstat;
  86         my $mon = grep { $monstat->{$_}->{host} eq $nodename } keys %$monstat;
  87
  88         # no pools = no data
  89         $message .= "- remove pools, this will !!DESTROY DATA!!\n" if @$pools;
  90         $message .= "- remove active OSD on $nodename\n" if $osd;
  91         $message .= "- remove active MDS on $nodename\n" if $mds;
  92         $message .= "- remove other MONs, $nodename is not the last MON\n"
  93             if scalar(keys %$monstat) > 1 && $mon;
  94
  95         # display all steps at once
  96         die "Unable to purge Ceph!\n\nTo continue:\n$message" if $message;
  97
  98         my $services = PVE::Ceph::Services::get_local_services();
  99         $services->{mon} = $monstat if $mon;
 100         $services->{crash}->{$nodename} = { direxists => 1 } if $param->{crash};
 101         $services->{logs}->{$nodename} = { direxists => 1 } if $param->{logs};
 102
 103         PVE::Ceph::Tools::purge_all_ceph_services($services);
 104         PVE::Ceph::Tools::purge_all_ceph_files($services);
 105
 106         return undef;
 107     }});
 108
 109 my $supported_ceph_versions = ['quincy'];
 110 my $default_ceph_version = 'quincy';
 111
 112 __PACKAGE__->register_method ({
 113     name => 'install',
 114     path => 'install',
 115     method => 'POST',
 116     description => "Install ceph related packages.",
 117     parameters => {
 118         additionalProperties => 0,
 119         properties => {
 120             version => {
 121                 type => 'string',
 122                 enum => $supported_ceph_versions,
 123                 default => $default_ceph_version,
 124                 description => "Ceph version to install.",
 125                 optional => 1,
 126             },
 127             'allow-experimental' => {
 128                 type => 'boolean',
 129                 default => 0,
 130                 optional => 1,
 131                 description => "Allow experimental versions. Use with care!",
 132             },
 133             'test-repository' => {
 134                 type => 'boolean',
 135                 default => 0,
 136                 optional => 1,
 137                 description => "Use the test, not the main repository. Use with care!",
 138             },
 139         },
 140     },
 141     returns => { type => 'null' },
 142     code => sub {
 143         my ($param) = @_;
 144
 145         my $cephver = $param->{version} || $default_ceph_version;
 146
 147         my $repo = $param->{'test-repository'} ? 'test' : 'main';
 148
 149         my $repolist;
 150         if ($cephver eq 'quincy') {
 151             $repolist = "deb http://download.proxmox.com/debian/ceph-quincy bookworm $repo\n";
 152         } else {
 153             die "unsupported ceph version: $cephver";
 154         }
 155         PVE::Tools::file_set_contents("/etc/apt/sources.list.d/ceph.list", $repolist);
 156
 157         my $supported_re = join('|', $supported_ceph_versions->@*);
 158         warn "WARNING: installing non-default ceph release '$cephver'!\n" if $cephver !~ qr/^(?:$supported_re)$/;
 159
 160         local $ENV{DEBIAN_FRONTEND} = 'noninteractive';
 161         print "update available package list\n";
 162         eval {
 163             run_command(
 164                 ['apt-get', '-q', 'update'],
 165                 outfunc => sub {},
 166                 errfunc => sub { print STDERR "$_[0]\n" },
 167             )
 168         };
 169
 170         my @apt_install = qw(apt-get --no-install-recommends -o Dpkg::Options::=--force-confnew install --);
 171         my @ceph_packages = qw(
 172             ceph
 173             ceph-common
 174             ceph-mds
 175             ceph-fuse
 176             gdisk
 177             nvme-cli
 178         );
 179
 180         # got split out with quincy and is required by PVE tooling, conditionally exclude it for older
 181         # FIXME: remove condition with PVE 8.0, i.e., once we only support quincy+ new installations
 182         if ($cephver ne 'octopus' and $cephver ne 'pacific') {
 183             push @ceph_packages, 'ceph-volume';
 184         }
 185
 186         print "start installation\n";
 187
 188         # this flag helps to determine when apt is actually done installing (vs. partial extracing)
 189         my $install_flag_fn = PVE::Ceph::Tools::ceph_install_flag_file();
 190         open(my $install_flag, '>', $install_flag_fn) or die "could not create install flag - $!\n";
 191         close $install_flag;
 192
 193         if (system(@apt_install, @ceph_packages) != 0) {
 194             unlink $install_flag_fn or warn "could not remove Ceph installation flag - $!";
 195             die "apt failed during ceph installation ($?)\n";
 196         }
 197
 198         print "\ninstalled ceph $cephver successfully!\n";
 199         # done: drop flag file so that the PVE::Ceph::Tools check returns Ok now.
 200         unlink $install_flag_fn or warn "could not remove Ceph installation flag - $!";
 201
 202         print "\nreloading API to load new Ceph RADOS library...\n";
 203         run_command([
 204             'systemctl', 'try-reload-or-restart', 'pvedaemon.service', 'pveproxy.service'
 205         ]);
 206
 207         return undef;
 208     }});
 209
 210 __PACKAGE__->register_method ({
 211     name => 'status',
 212     path => 'status',
 213     method => 'GET',
 214     description => "Get Ceph Status.",
 215     parameters => {
 216         additionalProperties => 0,
 217     },
 218     returns => { type => 'null' },
 219     code => sub {
 220         PVE::Ceph::Tools::check_ceph_inited();
 221
 222         run_command(
 223             ['ceph', '-s'],
 224             outfunc => sub { print "$_[0]\n" },
 225             errfunc => sub { print STDERR "$_[0]\n" },
 226             timeout => 15,
 227         );
 228         return undef;
 229     }});
 230
 231 my $get_storages = sub {
 232     my ($fs, $is_default) = @_;
 233
 234     my $cfg = PVE::Storage::config();
 235
 236     my $storages = $cfg->{ids};
 237     my $res = {};
 238     foreach my $storeid (keys %$storages) {
 239         my $curr = $storages->{$storeid};
 240         next if $curr->{type} ne 'cephfs';
 241         my $cur_fs = $curr->{'fs-name'};
 242         $res->{$storeid} = $storages->{$storeid}
 243             if (!defined($cur_fs) && $is_default) || (defined($cur_fs) && $fs eq $cur_fs);
 244     }
 245
 246     return $res;
 247 };
 248
 249 __PACKAGE__->register_method ({
 250     name => 'destroyfs',
 251     path => 'destroyfs',
 252     method => 'DELETE',
 253     description => "Destroy a Ceph filesystem",
 254     parameters => {
 255         additionalProperties => 0,
 256         properties => {
 257             node => get_standard_option('pve-node'),
 258             name => {
 259                 description => "The ceph filesystem name.",
 260                 type => 'string',
 261             },
 262             'remove-storages' => {
 263                 description => "Remove all pveceph-managed storages configured for this fs.",
 264                 type => 'boolean',
 265                 optional => 1,
 266                 default => 0,
 267             },
 268             'remove-pools' => {
 269                 description => "Remove data and metadata pools configured for this fs.",
 270                 type => 'boolean',
 271                 optional => 1,
 272                 default => 0,
 273             },
 274         },
 275     },
 276     returns => { type => 'string' },
 277     code => sub {
 278         my ($param) = @_;
 279
 280         PVE::Ceph::Tools::check_ceph_inited();
 281
 282         my $rpcenv = PVE::RPCEnvironment::get();
 283         my $user = $rpcenv->get_user();
 284
 285         my $fs_name = $param->{name};
 286
 287         my $fs;
 288         my $fs_list = PVE::Ceph::Tools::ls_fs();
 289         for my $entry (@$fs_list) {
 290             next if $entry->{name} ne $fs_name;
 291             $fs = $entry;
 292             last;
 293         }
 294         die "no such cephfs '$fs_name'\n" if !$fs;
 295
 296         my $worker = sub {
 297             my $rados = PVE::RADOS->new();
 298
 299             if ($param->{'remove-storages'}) {
 300                 my $defaultfs;
 301                 my $fs_dump = $rados->mon_command({ prefix => "fs dump" });
 302                 for my $fs ($fs_dump->{filesystems}->@*) {
 303                     next if $fs->{id} != $fs_dump->{default_fscid};
 304                     $defaultfs = $fs->{mdsmap}->{fs_name};
 305                 }
 306                 warn "no default fs found, maybe not all relevant storages are removed\n"
 307                     if !defined($defaultfs);
 308
 309                 my $storages = $get_storages->($fs_name, $fs_name eq ($defaultfs // ''));
 310                 for my $storeid (keys %$storages) {
 311                     my $store = $storages->{$storeid};
 312                     if (!$store->{disable}) {
 313                         die "storage '$storeid' is not disabled, make sure to disable ".
 314                             "and unmount the storage first\n";
 315                     }
 316                 }
 317
 318                 my $err;
 319                 for my $storeid (keys %$storages) {
 320                     # skip external clusters, not managed by pveceph
 321                     next if $storages->{$storeid}->{monhost};
 322                     eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
 323                     if ($@) {
 324                         warn "failed to remove storage '$storeid': $@\n";
 325                         $err = 1;
 326                     }
 327                 }
 328                 die "failed to remove (some) storages - check log and remove manually!\n"
 329                     if $err;
 330             }
 331
 332             PVE::Ceph::Tools::destroy_fs($fs_name, $rados);
 333
 334             if ($param->{'remove-pools'}) {
 335                 warn "removing metadata pool '$fs->{metadata_pool}'\n";
 336                 eval { PVE::Ceph::Tools::destroy_pool($fs->{metadata_pool}, $rados) };
 337                 warn "$@\n" if $@;
 338
 339                 foreach my $pool ($fs->{data_pools}->@*) {
 340                     warn "removing data pool '$pool'\n";
 341                     eval { PVE::Ceph::Tools::destroy_pool($pool, $rados) };
 342                     warn "$@\n" if $@;
 343                 }
 344             }
 345
 346         };
 347         return $rpcenv->fork_worker('cephdestroyfs', $fs_name,  $user, $worker);
 348     }});
 349
 350 our $cmddef = {
 351     init => [ 'PVE::API2::Ceph', 'init', [], { node => $nodename } ],
 352     pool => {
 353         ls => [ 'PVE::API2::Ceph::Pool', 'lspools', [], { node => $nodename }, sub {
 354             my ($data, $schema, $options) = @_;
 355             PVE::CLIFormatter::print_api_result($data, $schema,
 356                 [
 357                     'pool_name',
 358                     'size',
 359                     'min_size',
 360                     'pg_num',
 361                     'pg_num_min',
 362                     'pg_num_final',
 363                     'pg_autoscale_mode',
 364                     'target_size',
 365                     'target_size_ratio',
 366                     'crush_rule_name',
 367                     'percent_used',
 368                     'bytes_used',
 369                 ],
 370                 $options);
 371         }, $PVE::RESTHandler::standard_output_options],
 372         create => [ 'PVE::API2::Ceph::Pool', 'createpool', ['name'], { node => $nodename }],
 373         destroy => [ 'PVE::API2::Ceph::Pool', 'destroypool', ['name'], { node => $nodename } ],
 374         set => [ 'PVE::API2::Ceph::Pool', 'setpool', ['name'], { node => $nodename } ],
 375         get => [ 'PVE::API2::Ceph::Pool', 'getpool', ['name'], { node => $nodename }, sub {
 376             my ($data, $schema, $options) = @_;
 377             PVE::CLIFormatter::print_api_result($data, $schema, undef, $options);
 378         }, $PVE::RESTHandler::standard_output_options],
 379     },
 380     lspools => { alias => 'pool ls' },
 381     createpool => { alias => 'pool create' },
 382     destroypool => { alias => 'pool destroy' },
 383     fs => {
 384         create => [ 'PVE::API2::Ceph::FS', 'createfs', [], { node => $nodename }],
 385         destroy => [ __PACKAGE__, 'destroyfs', ['name'], { node => $nodename }],
 386     },
 387     osd => {
 388         create => [ 'PVE::API2::Ceph::OSD', 'createosd', ['dev'], { node => $nodename }, $upid_exit],
 389         destroy => [ 'PVE::API2::Ceph::OSD', 'destroyosd', ['osdid'], { node => $nodename }, $upid_exit],
 390     },
 391     createosd => { alias => 'osd create' },
 392     destroyosd => { alias => 'osd destroy' },
 393     mon => {
 394         create => [ 'PVE::API2::Ceph::MON', 'createmon', [], { node => $nodename }, $upid_exit],
 395         destroy => [ 'PVE::API2::Ceph::MON', 'destroymon', ['monid'], { node => $nodename }, $upid_exit],
 396     },
 397     createmon => { alias => 'mon create' },
 398     destroymon => { alias => 'mon destroy' },
 399     mgr => {
 400         create => [ 'PVE::API2::Ceph::MGR', 'createmgr', [], { node => $nodename }, $upid_exit],
 401         destroy => [ 'PVE::API2::Ceph::MGR', 'destroymgr', ['id'], { node => $nodename }, $upid_exit],
 402     },
 403     createmgr => { alias => 'mgr create' },
 404     destroymgr => { alias => 'mgr destroy' },
 405     mds => {
 406         create => [ 'PVE::API2::Ceph::MDS', 'createmds', [], { node => $nodename }, $upid_exit],
 407         destroy => [ 'PVE::API2::Ceph::MDS', 'destroymds', ['name'], { node => $nodename }, $upid_exit],
 408     },
 409     start => [ 'PVE::API2::Ceph', 'start', [], { node => $nodename }, $upid_exit],
 410     stop => [ 'PVE::API2::Ceph', 'stop', [], { node => $nodename }, $upid_exit],
 411     install => [ __PACKAGE__, 'install', [] ],
 412     purge => [  __PACKAGE__, 'purge', [] ],
 413     status => [ __PACKAGE__, 'status', []],
 414 };
 415
 416 1;