PVE/API2/Ceph.pm

   1 package PVE::API2::CephOSD;
   2
   3 use strict;
   4 use warnings;
   5 use Cwd qw(abs_path);
   6 use Net::IP;
   7
   8 use PVE::SafeSyslog;
   9 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
  10 use PVE::Exception qw(raise raise_param_exc);
  11 use PVE::INotify;
  12 use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
  13 use PVE::AccessControl;
  14 use PVE::Storage;
  15 use PVE::API2::Storage::Config;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::JSONSchema qw(get_standard_option);
  19 use PVE::RADOS;
  20 use PVE::CephTools;
  21 use PVE::Diskmanage;
  22
  23 use base qw(PVE::RESTHandler);
  24
  25 use Data::Dumper; # fixme: remove
  26
  27 my $get_osd_status = sub {
  28     my ($rados, $osdid) = @_;
  29
  30     my $stat = $rados->mon_command({ prefix => 'osd dump' });
  31
  32     my $osdlist = $stat->{osds} || [];
  33
  34     my $flags = $stat->{flags} || undef;
  35
  36     my $osdstat;
  37     foreach my $d (@$osdlist) {
  38         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  39     }
  40     if (defined($osdid)) {
  41         die "no such OSD '$osdid'\n" if !$osdstat->{$osdid};
  42         return $osdstat->{$osdid};
  43     }
  44
  45     return wantarray? ($osdstat, $flags):$osdstat;
  46 };
  47
  48 my $get_osd_usage = sub {
  49     my ($rados) = @_;
  50
  51     my $osdlist = $rados->mon_command({ prefix => 'pg dump',
  52                                         dumpcontents => [ 'osds' ]}) || [];
  53
  54     my $osdstat;
  55     foreach my $d (@$osdlist) {
  56         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  57     }
  58
  59     return $osdstat;
  60 };
  61
  62 __PACKAGE__->register_method ({
  63     name => 'index',
  64     path => '',
  65     method => 'GET',
  66     description => "Get Ceph osd list/tree.",
  67     proxyto => 'node',
  68     protected => 1,
  69     permissions => {
  70         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  71     },
  72     parameters => {
  73         additionalProperties => 0,
  74         properties => {
  75             node => get_standard_option('pve-node'),
  76         },
  77     },
  78     # fixme: return a list instead of extjs tree format ?
  79     returns => {
  80         type => "object",
  81     },
  82     code => sub {
  83         my ($param) = @_;
  84
  85         PVE::CephTools::check_ceph_inited();
  86
  87         my $rados = PVE::RADOS->new();
  88         my $res = $rados->mon_command({ prefix => 'osd tree' });
  89
  90         die "no tree nodes found\n" if !($res && $res->{nodes});
  91
  92         my ($osdhash, $flags) = &$get_osd_status($rados);
  93
  94         my $usagehash = &$get_osd_usage($rados);
  95
  96         my $osdmetadata_tmp = $rados->mon_command({ prefix => 'osd metadata' });
  97
  98         my $osdmetadata = {};
  99         foreach my $osd (@$osdmetadata_tmp) {
 100             $osdmetadata->{$osd->{id}} = $osd;
 101         }
 102
 103         my $nodes = {};
 104         my $newnodes = {};
 105         foreach my $e (@{$res->{nodes}}) {
 106             $nodes->{$e->{id}} = $e;
 107
 108             my $new = {
 109                 id => $e->{id},
 110                 name => $e->{name},
 111                 type => $e->{type}
 112             };
 113
 114             foreach my $opt (qw(status crush_weight reweight device_class)) {
 115                 $new->{$opt} = $e->{$opt} if defined($e->{$opt});
 116             }
 117
 118             if (my $stat = $osdhash->{$e->{id}}) {
 119                 $new->{in} = $stat->{in} if defined($stat->{in});
 120             }
 121
 122             if (my $stat = $usagehash->{$e->{id}}) {
 123                 $new->{total_space} = ($stat->{kb} || 1) * 1024;
 124                 $new->{bytes_used} = ($stat->{kb_used} || 0) * 1024;
 125                 $new->{percent_used} = ($new->{bytes_used}*100)/$new->{total_space};
 126                 if (my $d = $stat->{fs_perf_stat}) {
 127                     $new->{commit_latency_ms} = $d->{commit_latency_ms};
 128                     $new->{apply_latency_ms} = $d->{apply_latency_ms};
 129                 }
 130             }
 131
 132             my $osdmd = $osdmetadata->{$e->{id}};
 133             if ($e->{type} eq 'osd' && $osdmd) {
 134                 if ($osdmd->{bluefs}) {
 135                     $new->{osdtype} = 'bluestore';
 136                     $new->{blfsdev} = $osdmd->{bluestore_bdev_dev_node};
 137                     $new->{dbdev} = $osdmd->{bluefs_db_dev_node};
 138                     $new->{waldev} = $osdmd->{bluefs_wal_dev_node};
 139                 } else {
 140                     $new->{osdtype} = 'filestore';
 141                 }
 142             }
 143
 144             $newnodes->{$e->{id}} = $new;
 145         }
 146
 147         foreach my $e (@{$res->{nodes}}) {
 148             my $new = $newnodes->{$e->{id}};
 149             if ($e->{children} && scalar(@{$e->{children}})) {
 150                 $new->{children} = [];
 151                 $new->{leaf} = 0;
 152                 foreach my $cid (@{$e->{children}}) {
 153                     $nodes->{$cid}->{parent} = $e->{id};
 154                     if ($nodes->{$cid}->{type} eq 'osd' &&
 155                         $e->{type} eq 'host') {
 156                         $newnodes->{$cid}->{host} = $e->{name};
 157                     }
 158                     push @{$new->{children}}, $newnodes->{$cid};
 159                 }
 160             } else {
 161                 $new->{leaf} = ($e->{id} >= 0) ? 1 : 0;
 162             }
 163         }
 164
 165         my $roots = [];
 166         foreach my $e (@{$res->{nodes}}) {
 167             if (!$nodes->{$e->{id}}->{parent}) {
 168                 push @$roots, $newnodes->{$e->{id}};
 169             }
 170         }
 171
 172         die "no root node\n" if !@$roots;
 173
 174         my $data = { root => { leaf =>  0, children => $roots } };
 175
 176         # we want this for the noout flag
 177         $data->{flags} = $flags if $flags;
 178
 179         return $data;
 180     }});
 181
 182 __PACKAGE__->register_method ({
 183     name => 'createosd',
 184     path => '',
 185     method => 'POST',
 186     description => "Create OSD",
 187     proxyto => 'node',
 188     protected => 1,
 189     parameters => {
 190         additionalProperties => 0,
 191         properties => {
 192             node => get_standard_option('pve-node'),
 193             dev => {
 194                 description => "Block device name.",
 195                 type => 'string',
 196             },
 197             journal_dev => {
 198                 description => "Block device name for journal (filestore) or block.db (bluestore).",
 199                 optional => 1,
 200                 type => 'string',
 201             },
 202             wal_dev => {
 203                 description => "Block device name for block.wal (bluestore only).",
 204                 optional => 1,
 205                 type => 'string',
 206             },
 207             fstype => {
 208                 description => "File system type (filestore only).",
 209                 type => 'string',
 210                 enum => ['xfs', 'ext4', 'btrfs'],
 211                 default => 'xfs',
 212                 optional => 1,
 213             },
 214             bluestore => {
 215                 description => "Use bluestore instead of filestore.",
 216                 type => 'boolean',
 217                 default => 0,
 218                 optional => 1,
 219             },
 220         },
 221     },
 222     returns => { type => 'string' },
 223     code => sub {
 224         my ($param) = @_;
 225
 226         my $rpcenv = PVE::RPCEnvironment::get();
 227
 228         my $authuser = $rpcenv->get_user();
 229
 230         raise_param_exc({ 'bluestore' => "conflicts with parameter 'fstype'" })
 231             if (defined($param->{fstype}) && defined($param->{bluestore}) && $param->{bluestore});
 232
 233         PVE::CephTools::check_ceph_inited();
 234
 235         PVE::CephTools::setup_pve_symlinks();
 236
 237         PVE::CephTools::check_ceph_installed('ceph_osd');
 238
 239         my $bluestore = $param->{bluestore} // 0;
 240
 241         my $journal_dev;
 242         my $wal_dev;
 243
 244         if ($param->{journal_dev} && ($param->{journal_dev} ne $param->{dev})) {
 245             $journal_dev = PVE::Diskmanage::verify_blockdev_path($param->{journal_dev});
 246             # if only journal is given, also put the wal there
 247             $wal_dev = $journal_dev;
 248         }
 249
 250         if ($param->{wal_dev} &&
 251             ($param->{wal_dev} ne $param->{dev}) &&
 252             (!$param->{journal_dev} || $param->{wal_dev} ne $param->{journal_dev})) {
 253             raise_param_exc({ 'wal_dev' => "can only be set with paramater 'bluestore'"})
 254                 if !$bluestore;
 255             $wal_dev = PVE::Diskmanage::verify_blockdev_path($param->{wal_dev});
 256         }
 257
 258         $param->{dev} = PVE::Diskmanage::verify_blockdev_path($param->{dev});
 259
 260         my $devname = $param->{dev};
 261         $devname =~ s|/dev/||;
 262
 263         my $disklist = PVE::Diskmanage::get_disks($devname, 1);
 264
 265         my $diskinfo = $disklist->{$devname};
 266         die "unable to get device info for '$devname'\n"
 267             if !$diskinfo;
 268
 269         die "device '$param->{dev}' is in use\n"
 270             if $diskinfo->{used};
 271
 272         my $devpath = $diskinfo->{devpath};
 273         my $rados = PVE::RADOS->new();
 274         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 275         die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid};
 276
 277         my $fsid = $monstat->{monmap}->{fsid};
 278         $fsid = $1 if $fsid =~ m/^([0-9a-f\-]+)$/;
 279
 280         my $ceph_bootstrap_osd_keyring = PVE::CephTools::get_config('ceph_bootstrap_osd_keyring');
 281
 282         if (! -f $ceph_bootstrap_osd_keyring) {
 283             my $bindata = $rados->mon_command({ prefix => 'auth get', entity => 'client.bootstrap-osd', format => 'plain' });
 284             PVE::Tools::file_set_contents($ceph_bootstrap_osd_keyring, $bindata);
 285         };
 286
 287         my $worker = sub {
 288             my $upid = shift;
 289
 290             my $fstype = $param->{fstype} || 'xfs';
 291
 292
 293             my $ccname = PVE::CephTools::get_config('ccname');
 294
 295             my $cmd = ['ceph-disk', 'prepare', '--zap-disk',
 296                        '--cluster', $ccname, '--cluster-uuid', $fsid ];
 297
 298             if ($bluestore) {
 299                 print "create OSD on $devpath (bluestore)\n";
 300                 push @$cmd, '--bluestore';
 301
 302                 if ($journal_dev) {
 303                     print "using device '$journal_dev' for block.db\n";
 304                     push @$cmd, '--block.db', $journal_dev;
 305                 }
 306
 307                 if ($wal_dev) {
 308                     print "using device '$wal_dev' for block.wal\n";
 309                     push @$cmd, '--block.wal', $wal_dev;
 310                 }
 311
 312                 push @$cmd, $devpath;
 313             } else {
 314                 print "create OSD on $devpath ($fstype)\n";
 315                 push @$cmd, '--filestore', '--fs-type', $fstype;
 316                 if ($journal_dev) {
 317                     print "using device '$journal_dev' for journal\n";
 318                     push @$cmd, '--journal-dev', $devpath, $journal_dev;
 319                 } else {
 320                     push @$cmd, $devpath;
 321                 }
 322             }
 323
 324
 325             run_command($cmd);
 326         };
 327
 328         return $rpcenv->fork_worker('cephcreateosd', $devname,  $authuser, $worker);
 329     }});
 330
 331 __PACKAGE__->register_method ({
 332     name => 'destroyosd',
 333     path => '{osdid}',
 334     method => 'DELETE',
 335     description => "Destroy OSD",
 336     proxyto => 'node',
 337     protected => 1,
 338     parameters => {
 339         additionalProperties => 0,
 340         properties => {
 341             node => get_standard_option('pve-node'),
 342             osdid => {
 343                 description => 'OSD ID',
 344                 type => 'integer',
 345             },
 346             cleanup => {
 347                 description => "If set, we remove partition table entries.",
 348                 type => 'boolean',
 349                 optional => 1,
 350                 default => 0,
 351             },
 352         },
 353     },
 354     returns => { type => 'string' },
 355     code => sub {
 356         my ($param) = @_;
 357
 358         my $rpcenv = PVE::RPCEnvironment::get();
 359
 360         my $authuser = $rpcenv->get_user();
 361
 362         PVE::CephTools::check_ceph_inited();
 363
 364         my $osdid = $param->{osdid};
 365
 366         my $rados = PVE::RADOS->new();
 367         my $osdstat = &$get_osd_status($rados, $osdid);
 368
 369         die "osd is in use (in == 1)\n" if $osdstat->{in};
 370         #&$run_ceph_cmd(['osd', 'out', $osdid]);
 371
 372         die "osd is still runnung (up == 1)\n" if $osdstat->{up};
 373
 374         my $osdsection = "osd.$osdid";
 375
 376         my $worker = sub {
 377             my $upid = shift;
 378
 379             # reopen with longer timeout
 380             $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
 381
 382             print "destroy OSD $osdsection\n";
 383
 384             eval { PVE::CephTools::ceph_service_cmd('stop', $osdsection); };
 385             warn $@ if $@;
 386
 387             print "Remove $osdsection from the CRUSH map\n";
 388             $rados->mon_command({ prefix => "osd crush remove", name => $osdsection, format => 'plain' });
 389
 390             print "Remove the $osdsection authentication key.\n";
 391             $rados->mon_command({ prefix => "auth del", entity => $osdsection, format => 'plain' });
 392
 393             print "Remove OSD $osdsection\n";
 394             $rados->mon_command({ prefix => "osd rm", ids => [ $osdsection ], format => 'plain' });
 395
 396             # try to unmount from standard mount point
 397             my $mountpoint = "/var/lib/ceph/osd/ceph-$osdid";
 398
 399             my $remove_partition = sub {
 400                 my ($part) = @_;
 401
 402                 return if !$part || (! -b $part );
 403                 my $partnum = PVE::Diskmanage::get_partnum($part);
 404                 my $devpath = PVE::Diskmanage::get_blockdev($part);
 405
 406                 print "remove partition $part (disk '${devpath}', partnum $partnum)\n";
 407                 eval { run_command(['/sbin/sgdisk', '-d', $partnum, "${devpath}"]); };
 408                 warn $@ if $@;
 409             };
 410
 411             my $partitions_to_remove = [];
 412
 413             if ($param->{cleanup}) {
 414                 if (my $fd = IO::File->new("/proc/mounts", "r")) {
 415                     while (defined(my $line = <$fd>)) {
 416                         my ($dev, $path, $fstype) = split(/\s+/, $line);
 417                         next if !($dev && $path && $fstype);
 418                         next if $dev !~ m|^/dev/|;
 419                         if ($path eq $mountpoint) {
 420                             my $data_part = abs_path($dev);
 421                             push @$partitions_to_remove, $data_part;
 422                             last;
 423                         }
 424                     }
 425                     close($fd);
 426                 }
 427
 428                 foreach my $path (qw(journal block block.db block.wal)) {
 429                     my $part = abs_path("$mountpoint/$path");
 430                     if ($part) {
 431                         push @$partitions_to_remove, $part;
 432                     }
 433                 }
 434             }
 435
 436             print "Unmount OSD $osdsection from  $mountpoint\n";
 437             eval { run_command(['/bin/umount', $mountpoint]); };
 438             if (my $err = $@) {
 439                 warn $err;
 440             } elsif ($param->{cleanup}) {
 441                 #be aware of the ceph udev rules which can remount.
 442                 foreach my $part (@$partitions_to_remove) {
 443                     $remove_partition->($part);
 444                 }
 445             }
 446         };
 447
 448         return $rpcenv->fork_worker('cephdestroyosd', $osdsection,  $authuser, $worker);
 449     }});
 450
 451 __PACKAGE__->register_method ({
 452     name => 'in',
 453     path => '{osdid}/in',
 454     method => 'POST',
 455     description => "ceph osd in",
 456     proxyto => 'node',
 457     protected => 1,
 458     permissions => {
 459         check => ['perm', '/', [ 'Sys.Modify' ]],
 460     },
 461     parameters => {
 462         additionalProperties => 0,
 463         properties => {
 464             node => get_standard_option('pve-node'),
 465             osdid => {
 466                 description => 'OSD ID',
 467                 type => 'integer',
 468             },
 469         },
 470     },
 471     returns => { type => "null" },
 472     code => sub {
 473         my ($param) = @_;
 474
 475         PVE::CephTools::check_ceph_inited();
 476
 477         my $osdid = $param->{osdid};
 478
 479         my $rados = PVE::RADOS->new();
 480
 481         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 482
 483         my $osdsection = "osd.$osdid";
 484
 485         $rados->mon_command({ prefix => "osd in", ids => [ $osdsection ], format => 'plain' });
 486
 487         return undef;
 488     }});
 489
 490 __PACKAGE__->register_method ({
 491     name => 'out',
 492     path => '{osdid}/out',
 493     method => 'POST',
 494     description => "ceph osd out",
 495     proxyto => 'node',
 496     protected => 1,
 497     permissions => {
 498         check => ['perm', '/', [ 'Sys.Modify' ]],
 499     },
 500     parameters => {
 501         additionalProperties => 0,
 502         properties => {
 503             node => get_standard_option('pve-node'),
 504             osdid => {
 505                 description => 'OSD ID',
 506                 type => 'integer',
 507             },
 508         },
 509     },
 510     returns => { type => "null" },
 511     code => sub {
 512         my ($param) = @_;
 513
 514         PVE::CephTools::check_ceph_inited();
 515
 516         my $osdid = $param->{osdid};
 517
 518         my $rados = PVE::RADOS->new();
 519
 520         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 521
 522         my $osdsection = "osd.$osdid";
 523
 524         $rados->mon_command({ prefix => "osd out", ids => [ $osdsection ], format => 'plain' });
 525
 526         return undef;
 527     }});
 528
 529 package PVE::API2::Ceph;
 530
 531 use strict;
 532 use warnings;
 533 use File::Basename;
 534 use File::Path;
 535 use POSIX qw (LONG_MAX);
 536 use Cwd qw(abs_path);
 537 use IO::Dir;
 538 use UUID;
 539 use Net::IP;
 540
 541 use PVE::SafeSyslog;
 542 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
 543 use PVE::Exception qw(raise raise_param_exc);
 544 use PVE::INotify;
 545 use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
 546 use PVE::AccessControl;
 547 use PVE::Storage;
 548 use PVE::RESTHandler;
 549 use PVE::RPCEnvironment;
 550 use PVE::JSONSchema qw(get_standard_option);
 551 use JSON;
 552 use PVE::RADOS;
 553 use PVE::CephTools;
 554
 555 use base qw(PVE::RESTHandler);
 556
 557 use Data::Dumper; # fixme: remove
 558
 559 my $pve_osd_default_journal_size = 1024*5;
 560
 561 __PACKAGE__->register_method ({
 562     subclass => "PVE::API2::CephOSD",
 563     path => 'osd',
 564 });
 565
 566 __PACKAGE__->register_method ({
 567     name => 'index',
 568     path => '',
 569     method => 'GET',
 570     description => "Directory index.",
 571     permissions => { user => 'all' },
 572     permissions => {
 573         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 574     },
 575     parameters => {
 576         additionalProperties => 0,
 577         properties => {
 578             node => get_standard_option('pve-node'),
 579         },
 580     },
 581     returns => {
 582         type => 'array',
 583         items => {
 584             type => "object",
 585             properties => {},
 586         },
 587         links => [ { rel => 'child', href => "{name}" } ],
 588     },
 589     code => sub {
 590         my ($param) = @_;
 591
 592         my $result = [
 593             { name => 'init' },
 594             { name => 'mon' },
 595             { name => 'osd' },
 596             { name => 'pools' },
 597             { name => 'stop' },
 598             { name => 'start' },
 599             { name => 'status' },
 600             { name => 'crush' },
 601             { name => 'config' },
 602             { name => 'log' },
 603             { name => 'disks' },
 604             { name => 'flags' },
 605             { name => 'rules' },
 606         ];
 607
 608         return $result;
 609     }});
 610
 611 __PACKAGE__->register_method ({
 612     name => 'disks',
 613     path => 'disks',
 614     method => 'GET',
 615     description => "List local disks.",
 616     proxyto => 'node',
 617     protected => 1,
 618     permissions => {
 619         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 620     },
 621     parameters => {
 622         additionalProperties => 0,
 623         properties => {
 624             node => get_standard_option('pve-node'),
 625             type => {
 626                 description => "Only list specific types of disks.",
 627                 type => 'string',
 628                 enum => ['unused', 'journal_disks'],
 629                 optional => 1,
 630             },
 631         },
 632     },
 633     returns => {
 634         type => 'array',
 635         items => {
 636             type => "object",
 637             properties => {
 638                 dev => { type => 'string' },
 639                 used => { type => 'string', optional => 1 },
 640                 gpt => { type => 'boolean' },
 641                 size => { type => 'integer' },
 642                 osdid => { type => 'integer' },
 643                 vendor =>  { type => 'string', optional => 1 },
 644                 model =>  { type => 'string', optional => 1 },
 645                 serial =>  { type => 'string', optional => 1 },
 646             },
 647         },
 648         # links => [ { rel => 'child', href => "{}" } ],
 649     },
 650     code => sub {
 651         my ($param) = @_;
 652
 653         PVE::CephTools::check_ceph_inited();
 654
 655         my $disks = PVE::Diskmanage::get_disks(undef, 1);
 656
 657         my $res = [];
 658         foreach my $dev (keys %$disks) {
 659             my $d = $disks->{$dev};
 660             if ($param->{type}) {
 661                 if ($param->{type} eq 'journal_disks') {
 662                     next if $d->{osdid} >= 0;
 663                     next if !$d->{gpt};
 664                 } elsif ($param->{type} eq 'unused') {
 665                     next if $d->{used};
 666                 } else {
 667                     die "internal error"; # should not happen
 668                 }
 669             }
 670
 671             $d->{dev} = "/dev/$dev";
 672             push @$res, $d;
 673         }
 674
 675         return $res;
 676     }});
 677
 678 __PACKAGE__->register_method ({
 679     name => 'config',
 680     path => 'config',
 681     method => 'GET',
 682     permissions => {
 683         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 684     },
 685     description => "Get Ceph configuration.",
 686     parameters => {
 687         additionalProperties => 0,
 688         properties => {
 689             node => get_standard_option('pve-node'),
 690         },
 691     },
 692     returns => { type => 'string' },
 693     code => sub {
 694         my ($param) = @_;
 695
 696         PVE::CephTools::check_ceph_inited();
 697
 698         my $path = PVE::CephTools::get_config('pve_ceph_cfgpath');
 699         return PVE::Tools::file_get_contents($path);
 700
 701     }});
 702
 703 my $add_storage = sub {
 704     my ($pool, $storeid, $krbd) = @_;
 705
 706     my $storage_params = {
 707         type => 'rbd',
 708         pool => $pool,
 709         storage => $storeid,
 710         krbd => $krbd // 0,
 711         content => $krbd ? 'rootdir' : 'images',
 712     };
 713
 714     PVE::API2::Storage::Config->create($storage_params);
 715 };
 716
 717 my $get_storages = sub {
 718     my ($pool) = @_;
 719
 720     my $cfg = PVE::Storage::config();
 721
 722     my $storages = $cfg->{ids};
 723     my $res = {};
 724     foreach my $storeid (keys %$storages) {
 725         my $curr = $storages->{$storeid};
 726         $res->{$storeid} = $storages->{$storeid}
 727             if $curr->{type} eq 'rbd' && $pool eq $curr->{pool};
 728     }
 729
 730     return $res;
 731 };
 732
 733 __PACKAGE__->register_method ({
 734     name => 'listmon',
 735     path => 'mon',
 736     method => 'GET',
 737     description => "Get Ceph monitor list.",
 738     proxyto => 'node',
 739     protected => 1,
 740     permissions => {
 741         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 742     },
 743     parameters => {
 744         additionalProperties => 0,
 745         properties => {
 746             node => get_standard_option('pve-node'),
 747         },
 748     },
 749     returns => {
 750         type => 'array',
 751         items => {
 752             type => "object",
 753             properties => {
 754                 name => { type => 'string' },
 755                 addr => { type => 'string' },
 756             },
 757         },
 758         links => [ { rel => 'child', href => "{name}" } ],
 759     },
 760     code => sub {
 761         my ($param) = @_;
 762
 763         PVE::CephTools::check_ceph_inited();
 764
 765         my $res = [];
 766
 767         my $cfg = PVE::CephTools::parse_ceph_config();
 768
 769         my $monhash = {};
 770         foreach my $section (keys %$cfg) {
 771             my $d = $cfg->{$section};
 772             if ($section =~ m/^mon\.(\S+)$/) {
 773                 my $monid = $1;
 774                 if ($d->{'mon addr'} && $d->{'host'}) {
 775                     $monhash->{$monid} = {
 776                         addr => $d->{'mon addr'},
 777                         host => $d->{'host'},
 778                         name => $monid,
 779                     }
 780                 }
 781             }
 782         }
 783
 784         eval {
 785             my $rados = PVE::RADOS->new();
 786             my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 787             my $mons = $monstat->{monmap}->{mons};
 788             foreach my $d (@$mons) {
 789                 next if !defined($d->{name});
 790                 $monhash->{$d->{name}}->{rank} = $d->{rank};
 791                 $monhash->{$d->{name}}->{addr} = $d->{addr};
 792                 if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
 793                     $monhash->{$d->{name}}->{quorum} = 1;
 794                 }
 795             }
 796         };
 797         warn $@ if $@;
 798
 799         return PVE::RESTHandler::hash_to_array($monhash, 'name');
 800     }});
 801
 802 __PACKAGE__->register_method ({
 803     name => 'init',
 804     path => 'init',
 805     method => 'POST',
 806     description => "Create initial ceph default configuration and setup symlinks.",
 807     proxyto => 'node',
 808     protected => 1,
 809     permissions => {
 810         check => ['perm', '/', [ 'Sys.Modify' ]],
 811     },
 812     parameters => {
 813         additionalProperties => 0,
 814         properties => {
 815             node => get_standard_option('pve-node'),
 816             network => {
 817                 description => "Use specific network for all ceph related traffic",
 818                 type => 'string', format => 'CIDR',
 819                 optional => 1,
 820                 maxLength => 128,
 821             },
 822             size => {
 823                 description => 'Targeted number of replicas per object',
 824                 type => 'integer',
 825                 default => 3,
 826                 optional => 1,
 827                 minimum => 1,
 828                 maximum => 7,
 829             },
 830             min_size => {
 831                 description => 'Minimum number of available replicas per object to allow I/O',
 832                 type => 'integer',
 833                 default => 2,
 834                 optional => 1,
 835                 minimum => 1,
 836                 maximum => 7,
 837             },
 838             pg_bits => {
 839                 description => "Placement group bits, used to specify the " .
 840                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 841                     "default pg num' does not work for default pools.",
 842                 type => 'integer',
 843                 default => 6,
 844                 optional => 1,
 845                 minimum => 6,
 846                 maximum => 14,
 847             },
 848             disable_cephx => {
 849                 description => "Disable cephx authentification.\n\n" .
 850                     "WARNING: cephx is a security feature protecting against " .
 851                     "man-in-the-middle attacks. Only consider disabling cephx ".
 852                     "if your network is private!",
 853                 type => 'boolean',
 854                 optional => 1,
 855                 default => 0,
 856             },
 857         },
 858     },
 859     returns => { type => 'null' },
 860     code => sub {
 861         my ($param) = @_;
 862
 863         my $version = PVE::CephTools::get_local_version(1);
 864
 865         if (!$version || $version < 12) {
 866             die "Ceph Luminous required - please run 'pveceph install'\n";
 867         } else {
 868             PVE::CephTools::check_ceph_installed('ceph_bin');
 869         }
 870
 871         # simply load old config if it already exists
 872         my $cfg = PVE::CephTools::parse_ceph_config();
 873
 874         if (!$cfg->{global}) {
 875
 876             my $fsid;
 877             my $uuid;
 878
 879             UUID::generate($uuid);
 880             UUID::unparse($uuid, $fsid);
 881
 882             my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 883
 884             $cfg->{global} = {
 885                 'fsid' => $fsid,
 886                 'auth cluster required' => $auth,
 887                 'auth service required' => $auth,
 888                 'auth client required' => $auth,
 889                 'osd journal size' => $pve_osd_default_journal_size,
 890                 'osd pool default size' => $param->{size} // 3,
 891                 'osd pool default min size' => $param->{min_size} // 2,
 892                 'mon allow pool delete' => 'true',
 893             };
 894
 895             # this does not work for default pools
 896             #'osd pool default pg num' => $pg_num,
 897             #'osd pool default pgp num' => $pg_num,
 898         }
 899
 900         $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 901         $cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring';
 902
 903         if ($param->{pg_bits}) {
 904             $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 905             $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 906         }
 907
 908         if ($param->{network}) {
 909             $cfg->{global}->{'public network'} = $param->{network};
 910             $cfg->{global}->{'cluster network'} = $param->{network};
 911         }
 912
 913         PVE::CephTools::write_ceph_config($cfg);
 914
 915         PVE::CephTools::setup_pve_symlinks();
 916
 917         return undef;
 918     }});
 919
 920 my $find_node_ip = sub {
 921     my ($cidr) = @_;
 922
 923     my $net = Net::IP->new($cidr) || die Net::IP::Error() . "\n";
 924     my $id = $net->version == 6 ? 'address6' : 'address';
 925
 926     my $config = PVE::INotify::read_file('interfaces');
 927     my $ifaces = $config->{ifaces};
 928
 929     foreach my $iface (keys %$ifaces) {
 930         my $d = $ifaces->{$iface};
 931         next if !$d->{$id};
 932         my $a = Net::IP->new($d->{$id});
 933         next if !$a;
 934         return $d->{$id} if $net->overlaps($a);
 935     }
 936
 937     die "unable to find local address within network '$cidr'\n";
 938 };
 939
 940 my $create_mgr = sub {
 941     my ($rados, $id) = @_;
 942
 943     my $clustername = PVE::CephTools::get_config('ccname');
 944     my $mgrdir = "/var/lib/ceph/mgr/$clustername-$id";
 945     my $mgrkeyring = "$mgrdir/keyring";
 946     my $mgrname = "mgr.$id";
 947
 948     die "ceph manager directory '$mgrdir' already exists\n"
 949         if -d $mgrdir;
 950
 951     print "creating manager directory '$mgrdir'\n";
 952     mkdir $mgrdir;
 953     print "creating keys for '$mgrname'\n";
 954     my $output = $rados->mon_command({ prefix => 'auth get-or-create',
 955                                        entity => $mgrname,
 956                                        caps => [
 957                                            mon => 'allow profile mgr',
 958                                            osd => 'allow *',
 959                                            mds => 'allow *',
 960                                        ],
 961                                        format => 'plain'});
 962     PVE::Tools::file_set_contents($mgrkeyring, $output);
 963
 964     print "setting owner for directory\n";
 965     run_command(["chown", 'ceph:ceph', '-R', $mgrdir]);
 966
 967     print "enabling service 'ceph-mgr\@$id.service'\n";
 968     PVE::CephTools::ceph_service_cmd('enable', $mgrname);
 969     print "starting service 'ceph-mgr\@$id.service'\n";
 970     PVE::CephTools::ceph_service_cmd('start', $mgrname);
 971 };
 972
 973 my $destroy_mgr = sub {
 974     my ($mgrid) = @_;
 975
 976     my $clustername = PVE::CephTools::get_config('ccname');
 977     my $mgrname = "mgr.$mgrid";
 978     my $mgrdir = "/var/lib/ceph/mgr/$clustername-$mgrid";
 979
 980     die "ceph manager directory '$mgrdir' not found\n"
 981         if ! -d $mgrdir;
 982
 983     print "disabling service 'ceph-mgr\@$mgrid.service'\n";
 984     PVE::CephTools::ceph_service_cmd('disable', $mgrname);
 985     print "stopping service 'ceph-mgr\@$mgrid.service'\n";
 986     PVE::CephTools::ceph_service_cmd('stop', $mgrname);
 987
 988     print "removing manager directory '$mgrdir'\n";
 989     File::Path::remove_tree($mgrdir);
 990 };
 991
 992 __PACKAGE__->register_method ({
 993     name => 'createmon',
 994     path => 'mon',
 995     method => 'POST',
 996     description => "Create Ceph Monitor and Manager",
 997     proxyto => 'node',
 998     protected => 1,
 999     permissions => {
1000         check => ['perm', '/', [ 'Sys.Modify' ]],
1001     },
1002     parameters => {
1003         additionalProperties => 0,
1004         properties => {
1005             node => get_standard_option('pve-node'),
1006             id => {
1007                 type => 'string',
1008                 optional => 1,
1009                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1010                 description => "The ID for the monitor, when omitted the same as the nodename",
1011             },
1012             'exclude-manager' => {
1013                 type => 'boolean',
1014                 optional => 1,
1015                 default => 0,
1016                 description => "When set, only a monitor will be created.",
1017             },
1018         },
1019     },
1020     returns => { type => 'string' },
1021     code => sub {
1022         my ($param) = @_;
1023
1024         PVE::CephTools::check_ceph_installed('ceph_mon');
1025
1026         PVE::CephTools::check_ceph_installed('ceph_mgr')
1027             if (!$param->{'exclude-manager'});
1028
1029         PVE::CephTools::check_ceph_inited();
1030
1031         PVE::CephTools::setup_pve_symlinks();
1032
1033         my $rpcenv = PVE::RPCEnvironment::get();
1034
1035         my $authuser = $rpcenv->get_user();
1036
1037         my $cfg = PVE::CephTools::parse_ceph_config();
1038
1039         my $moncount = 0;
1040
1041         my $monaddrhash = {};
1042
1043         my $systemd_managed = PVE::CephTools::systemd_managed();
1044
1045         foreach my $section (keys %$cfg) {
1046             next if $section eq 'global';
1047             my $d = $cfg->{$section};
1048             if ($section =~ m/^mon\./) {
1049                 $moncount++;
1050                 if ($d->{'mon addr'}) {
1051                     $monaddrhash->{$d->{'mon addr'}} = $section;
1052                 }
1053             }
1054         }
1055
1056         my $monid = $param->{id} // $param->{node};
1057
1058         my $monsection = "mon.$monid";
1059         my $ip;
1060         if (my $pubnet = $cfg->{global}->{'public network'}) {
1061             $ip = &$find_node_ip($pubnet);
1062         } else {
1063             $ip = PVE::Cluster::remote_node_ip($param->{node});
1064         }
1065
1066         my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]:6789" : "$ip:6789";
1067         my $monname = $param->{node};
1068
1069         die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
1070         die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n"
1071             if $monaddrhash->{$monaddr};
1072
1073         my $worker = sub  {
1074             my $upid = shift;
1075
1076             my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1077
1078             if (! -f $pve_ckeyring_path) {
1079                 run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
1080                             "--gen-key -n client.admin");
1081             }
1082
1083             my $pve_mon_key_path = PVE::CephTools::get_config('pve_mon_key_path');
1084             if (! -f $pve_mon_key_path) {
1085                 run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
1086                 run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " .
1087                             "--cap mds 'allow' " .
1088                             "--cap osd 'allow *' " .
1089                             "--cap mgr 'allow *' " .
1090                             "--cap mon 'allow *'");
1091                 run_command("cp $pve_mon_key_path.tmp /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
1092                 run_command("chown ceph:ceph /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
1093                 run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
1094                 run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
1095             }
1096
1097             my $ccname = PVE::CephTools::get_config('ccname');
1098
1099             my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
1100             -d $mondir && die "monitor filesystem '$mondir' already exist\n";
1101
1102             my $monmap = "/tmp/monmap";
1103
1104             eval {
1105                 mkdir $mondir;
1106
1107                 run_command("chown ceph:ceph $mondir") if $systemd_managed;
1108
1109                 if ($moncount > 0) {
1110                     my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1111                     my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
1112                     PVE::Tools::file_set_contents($monmap, $mapdata);
1113                 } else {
1114                     run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
1115                 }
1116
1117                 run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
1118                 run_command("chown ceph:ceph -R $mondir") if $systemd_managed;
1119             };
1120             my $err = $@;
1121             unlink $monmap;
1122             if ($err) {
1123                 File::Path::remove_tree($mondir);
1124                 die $err;
1125             }
1126
1127             $cfg->{$monsection} = {
1128                 'host' => $monname,
1129                 'mon addr' => $monaddr,
1130             };
1131
1132             PVE::CephTools::write_ceph_config($cfg);
1133
1134             my $create_keys_pid = fork();
1135             if (!defined($create_keys_pid)) {
1136                 die "Could not spawn ceph-create-keys to create bootstrap keys\n";
1137             } elsif ($create_keys_pid == 0) {
1138                 exit PVE::Tools::run_command(['ceph-create-keys', '-i', $monid]);
1139             } else {
1140                 PVE::CephTools::ceph_service_cmd('start', $monsection);
1141
1142                 if ($systemd_managed) {
1143                     #to ensure we have the correct startup order.
1144                     eval { PVE::Tools::run_command(['/bin/systemctl', 'enable', "ceph-mon\@${monid}.service"]); };
1145                     warn "Enable ceph-mon\@${monid}.service manually"if $@;
1146                 }
1147                 waitpid($create_keys_pid, 0);
1148             }
1149
1150             # create manager
1151             if (!$param->{'exclude-manager'}) {
1152                 my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1153                 $create_mgr->($rados, $monid);
1154             }
1155         };
1156
1157         return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
1158     }});
1159
1160 __PACKAGE__->register_method ({
1161     name => 'destroymon',
1162     path => 'mon/{monid}',
1163     method => 'DELETE',
1164     description => "Destroy Ceph Monitor and Manager.",
1165     proxyto => 'node',
1166     protected => 1,
1167     permissions => {
1168         check => ['perm', '/', [ 'Sys.Modify' ]],
1169     },
1170     parameters => {
1171         additionalProperties => 0,
1172         properties => {
1173             node => get_standard_option('pve-node'),
1174             monid => {
1175                 description => 'Monitor ID',
1176                 type => 'string',
1177                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1178             },
1179             'exclude-manager' => {
1180                 type => 'boolean',
1181                 default => 0,
1182                 optional => 1,
1183                 description => "When set, removes only the monitor, not the manager"
1184             }
1185         },
1186     },
1187     returns => { type => 'string' },
1188     code => sub {
1189         my ($param) = @_;
1190
1191         my $rpcenv = PVE::RPCEnvironment::get();
1192
1193         my $authuser = $rpcenv->get_user();
1194
1195         PVE::CephTools::check_ceph_inited();
1196
1197         my $cfg = PVE::CephTools::parse_ceph_config();
1198
1199         my $monid = $param->{monid};
1200         my $monsection = "mon.$monid";
1201
1202         my $rados = PVE::RADOS->new();
1203         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
1204         my $monlist = $monstat->{monmap}->{mons};
1205
1206         die "no such monitor id '$monid'\n"
1207             if !defined($cfg->{$monsection});
1208
1209         my $ccname = PVE::CephTools::get_config('ccname');
1210
1211         my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
1212         -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n";
1213
1214         die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
1215
1216         my $worker = sub {
1217             my $upid = shift;
1218
1219             # reopen with longer timeout
1220             $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1221
1222             $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });
1223
1224             eval { PVE::CephTools::ceph_service_cmd('stop', $monsection); };
1225             warn $@ if $@;
1226
1227             delete $cfg->{$monsection};
1228             PVE::CephTools::write_ceph_config($cfg);
1229             File::Path::remove_tree($mondir);
1230
1231             # remove manager
1232             if (!$param->{'exclude-manager'}) {
1233                 eval { $destroy_mgr->($monid); };
1234                 warn $@ if $@;
1235             }
1236         };
1237
1238         return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
1239     }});
1240
1241 __PACKAGE__->register_method ({
1242     name => 'createmgr',
1243     path => 'mgr',
1244     method => 'POST',
1245     description => "Create Ceph Manager",
1246     proxyto => 'node',
1247     protected => 1,
1248     permissions => {
1249         check => ['perm', '/', [ 'Sys.Modify' ]],
1250     },
1251     parameters => {
1252         additionalProperties => 0,
1253         properties => {
1254             node => get_standard_option('pve-node'),
1255             id => {
1256                 type => 'string',
1257                 optional => 1,
1258                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1259                 description => "The ID for the manager, when omitted the same as the nodename",
1260             },
1261         },
1262     },
1263     returns => { type => 'string' },
1264     code => sub {
1265         my ($param) = @_;
1266
1267         PVE::CephTools::check_ceph_installed('ceph_mgr');
1268
1269         PVE::CephTools::check_ceph_inited();
1270
1271         my $rpcenv = PVE::RPCEnvironment::get();
1272
1273         my $authuser = $rpcenv->get_user();
1274
1275         my $mgrid = $param->{id} // $param->{node};
1276
1277         my $worker = sub  {
1278             my $upid = shift;
1279
1280             my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1281
1282             $create_mgr->($rados, $mgrid);
1283         };
1284
1285         return $rpcenv->fork_worker('cephcreatemgr', "mgr.$mgrid", $authuser, $worker);
1286     }});
1287
1288 __PACKAGE__->register_method ({
1289     name => 'destroymgr',
1290     path => 'mgr/{id}',
1291     method => 'DELETE',
1292     description => "Destroy Ceph Manager.",
1293     proxyto => 'node',
1294     protected => 1,
1295     permissions => {
1296         check => ['perm', '/', [ 'Sys.Modify' ]],
1297     },
1298     parameters => {
1299         additionalProperties => 0,
1300         properties => {
1301             node => get_standard_option('pve-node'),
1302             id => {
1303                 description => 'The ID of the manager',
1304                 type => 'string',
1305                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1306             },
1307         },
1308     },
1309     returns => { type => 'string' },
1310     code => sub {
1311         my ($param) = @_;
1312
1313         my $rpcenv = PVE::RPCEnvironment::get();
1314
1315         my $authuser = $rpcenv->get_user();
1316
1317         PVE::CephTools::check_ceph_inited();
1318
1319         my $mgrid = $param->{id};
1320
1321         my $worker = sub {
1322             my $upid = shift;
1323
1324             $destroy_mgr->($mgrid);
1325         };
1326
1327         return $rpcenv->fork_worker('cephdestroymgr', "mgr.$mgrid",  $authuser, $worker);
1328     }});
1329
1330 __PACKAGE__->register_method ({
1331     name => 'stop',
1332     path => 'stop',
1333     method => 'POST',
1334     description => "Stop ceph services.",
1335     proxyto => 'node',
1336     protected => 1,
1337     permissions => {
1338         check => ['perm', '/', [ 'Sys.Modify' ]],
1339     },
1340     parameters => {
1341         additionalProperties => 0,
1342         properties => {
1343             node => get_standard_option('pve-node'),
1344             service => {
1345                 description => 'Ceph service name.',
1346                 type => 'string',
1347                 optional => 1,
1348                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1349             },
1350         },
1351     },
1352     returns => { type => 'string' },
1353     code => sub {
1354         my ($param) = @_;
1355
1356         my $rpcenv = PVE::RPCEnvironment::get();
1357
1358         my $authuser = $rpcenv->get_user();
1359
1360         PVE::CephTools::check_ceph_inited();
1361
1362         my $cfg = PVE::CephTools::parse_ceph_config();
1363         scalar(keys %$cfg) || die "no configuration\n";
1364
1365         my $worker = sub {
1366             my $upid = shift;
1367
1368             my $cmd = ['stop'];
1369             if ($param->{service}) {
1370                 push @$cmd, $param->{service};
1371             }
1372
1373             PVE::CephTools::ceph_service_cmd(@$cmd);
1374         };
1375
1376         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
1377                                     $authuser, $worker);
1378     }});
1379
1380 __PACKAGE__->register_method ({
1381     name => 'start',
1382     path => 'start',
1383     method => 'POST',
1384     description => "Start ceph services.",
1385     proxyto => 'node',
1386     protected => 1,
1387     permissions => {
1388         check => ['perm', '/', [ 'Sys.Modify' ]],
1389     },
1390     parameters => {
1391         additionalProperties => 0,
1392         properties => {
1393             node => get_standard_option('pve-node'),
1394             service => {
1395                 description => 'Ceph service name.',
1396                 type => 'string',
1397                 optional => 1,
1398                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1399             },
1400         },
1401     },
1402     returns => { type => 'string' },
1403     code => sub {
1404         my ($param) = @_;
1405
1406         my $rpcenv = PVE::RPCEnvironment::get();
1407
1408         my $authuser = $rpcenv->get_user();
1409
1410         PVE::CephTools::check_ceph_inited();
1411
1412         my $cfg = PVE::CephTools::parse_ceph_config();
1413         scalar(keys %$cfg) || die "no configuration\n";
1414
1415         my $worker = sub {
1416             my $upid = shift;
1417
1418             my $cmd = ['start'];
1419             if ($param->{service}) {
1420                 push @$cmd, $param->{service};
1421             }
1422
1423             PVE::CephTools::ceph_service_cmd(@$cmd);
1424         };
1425
1426         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
1427                                     $authuser, $worker);
1428     }});
1429
1430 __PACKAGE__->register_method ({
1431     name => 'status',
1432     path => 'status',
1433     method => 'GET',
1434     description => "Get ceph status.",
1435     proxyto => 'node',
1436     protected => 1,
1437     permissions => {
1438         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1439     },
1440     parameters => {
1441         additionalProperties => 0,
1442         properties => {
1443             node => get_standard_option('pve-node'),
1444         },
1445     },
1446     returns => { type => 'object' },
1447     code => sub {
1448         my ($param) = @_;
1449
1450         PVE::CephTools::check_ceph_enabled();
1451
1452         my $rados = PVE::RADOS->new();
1453         my $status = $rados->mon_command({ prefix => 'status' });
1454         $status->{health} = $rados->mon_command({ prefix => 'health', detail => 'detail' });
1455         return $status;
1456     }});
1457
1458 __PACKAGE__->register_method ({
1459     name => 'lspools',
1460     path => 'pools',
1461     method => 'GET',
1462     description => "List all pools.",
1463     proxyto => 'node',
1464     protected => 1,
1465     permissions => {
1466         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1467     },
1468     parameters => {
1469         additionalProperties => 0,
1470         properties => {
1471             node => get_standard_option('pve-node'),
1472         },
1473     },
1474     returns => {
1475         type => 'array',
1476         items => {
1477             type => "object",
1478             properties => {
1479                 pool => { type => 'integer' },
1480                 pool_name => { type => 'string' },
1481                 size => { type => 'integer' },
1482             },
1483         },
1484         links => [ { rel => 'child', href => "{pool_name}" } ],
1485     },
1486     code => sub {
1487         my ($param) = @_;
1488
1489         PVE::CephTools::check_ceph_inited();
1490
1491         my $rados = PVE::RADOS->new();
1492
1493         my $stats = {};
1494         my $res = $rados->mon_command({ prefix => 'df' });
1495         my $total = $res->{stats}->{total_avail_bytes} || 0;
1496
1497         foreach my $d (@{$res->{pools}}) {
1498             next if !$d->{stats};
1499             next if !defined($d->{id});
1500             $stats->{$d->{id}} = $d->{stats};
1501         }
1502
1503         $res = $rados->mon_command({ prefix => 'osd dump' });
1504         my $rulestmp = $rados->mon_command({ prefix => 'osd crush rule dump'});
1505
1506         my $rules = {};
1507         for my $rule (@$rulestmp) {
1508             $rules->{$rule->{rule_id}} = $rule->{rule_name};
1509         }
1510
1511         my $data = [];
1512         foreach my $e (@{$res->{pools}}) {
1513             my $d = {};
1514             foreach my $attr (qw(pool pool_name size min_size pg_num crush_rule)) {
1515                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
1516             }
1517
1518             if (defined($d->{crush_rule}) && defined($rules->{$d->{crush_rule}})) {
1519                 $d->{crush_rule_name} = $rules->{$d->{crush_rule}};
1520             }
1521
1522             if (my $s = $stats->{$d->{pool}}) {
1523                 $d->{bytes_used} = $s->{bytes_used};
1524                 $d->{percent_used} = ($s->{bytes_used} / $total)*100
1525                     if $s->{max_avail} && $total;
1526             }
1527             push @$data, $d;
1528         }
1529
1530
1531         return $data;
1532     }});
1533
1534 __PACKAGE__->register_method ({
1535     name => 'createpool',
1536     path => 'pools',
1537     method => 'POST',
1538     description => "Create POOL",
1539     proxyto => 'node',
1540     protected => 1,
1541     permissions => {
1542         check => ['perm', '/', [ 'Sys.Modify' ]],
1543     },
1544     parameters => {
1545         additionalProperties => 0,
1546         properties => {
1547             node => get_standard_option('pve-node'),
1548             name => {
1549                 description => "The name of the pool. It must be unique.",
1550                 type => 'string',
1551             },
1552             size => {
1553                 description => 'Number of replicas per object',
1554                 type => 'integer',
1555                 default => 3,
1556                 optional => 1,
1557                 minimum => 1,
1558                 maximum => 7,
1559             },
1560             min_size => {
1561                 description => 'Minimum number of replicas per object',
1562                 type => 'integer',
1563                 default => 2,
1564                 optional => 1,
1565                 minimum => 1,
1566                 maximum => 7,
1567             },
1568             pg_num => {
1569                 description => "Number of placement groups.",
1570                 type => 'integer',
1571                 default => 64,
1572                 optional => 1,
1573                 minimum => 8,
1574                 maximum => 32768,
1575             },
1576             crush_rule => {
1577                 description => "The rule to use for mapping object placement in the cluster.",
1578                 type => 'string',
1579                 optional => 1,
1580             },
1581             application => {
1582                 description => "The application of the pool, 'rbd' by default.",
1583                 type => 'string',
1584                 enum => ['rbd', 'cephfs', 'rgw'],
1585                 optional => 1,
1586             },
1587             add_storages => {
1588                 description => "Configure VM and CT storages using the new pool.",
1589                 type => 'boolean',
1590                 optional => 1,
1591             },
1592         },
1593     },
1594     returns => { type => 'null' },
1595     code => sub {
1596         my ($param) = @_;
1597
1598         PVE::CephTools::check_ceph_inited();
1599
1600         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1601
1602         die "not fully configured - missing '$pve_ckeyring_path'\n"
1603             if ! -f $pve_ckeyring_path;
1604
1605         my $pool = $param->{name};
1606
1607         if ($param->{add_storages}) {
1608             my $rpcenv = PVE::RPCEnvironment::get();
1609             my $user = $rpcenv->get_user();
1610             $rpcenv->check($user, '/storage', ['Datastore.Allocate']);
1611             die "pool name contains characters which are illegal for storage naming\n"
1612                 if !PVE::JSONSchema::parse_storage_id($pool);
1613         }
1614
1615         my $pg_num = $param->{pg_num} || 64;
1616         my $size = $param->{size} || 3;
1617         my $min_size = $param->{min_size} || 2;
1618         my $rados = PVE::RADOS->new();
1619         my $application = $param->{application} // 'rbd';
1620
1621         $rados->mon_command({
1622             prefix => "osd pool create",
1623             pool => $pool,
1624             pg_num => int($pg_num),
1625             format => 'plain',
1626         });
1627
1628         $rados->mon_command({
1629             prefix => "osd pool set",
1630             pool => $pool,
1631             var => 'min_size',
1632             val => $min_size,
1633             format => 'plain',
1634         });
1635
1636         $rados->mon_command({
1637             prefix => "osd pool set",
1638             pool => $pool,
1639             var => 'size',
1640             val => $size,
1641             format => 'plain',
1642         });
1643
1644         if (defined($param->{crush_rule})) {
1645             $rados->mon_command({
1646                 prefix => "osd pool set",
1647                 pool => $pool,
1648                 var => 'crush_rule',
1649                 val => $param->{crush_rule},
1650                 format => 'plain',
1651             });
1652         }
1653
1654         $rados->mon_command({
1655                 prefix => "osd pool application enable",
1656                 pool => $pool,
1657                 app => $application,
1658         });
1659
1660         if ($param->{add_storages}) {
1661             my $err;
1662             eval { $add_storage->($pool, "${pool}_vm", 0); };
1663             if ($@) {
1664                 warn "failed to add VM storage: $@";
1665                 $err = 1;
1666             }
1667             eval { $add_storage->($pool, "${pool}_ct", 1); };
1668             if ($@) {
1669                 warn "failed to add CT storage: $@";
1670                 $err = 1;
1671             }
1672             die "adding storages for pool '$pool' failed, check log and add manually!\n"
1673                 if $err;
1674         }
1675
1676         return undef;
1677     }});
1678
1679 __PACKAGE__->register_method ({
1680     name => 'get_flags',
1681     path => 'flags',
1682     method => 'GET',
1683     description => "get all set ceph flags",
1684     proxyto => 'node',
1685     protected => 1,
1686     permissions => {
1687         check => ['perm', '/', [ 'Sys.Audit' ]],
1688     },
1689     parameters => {
1690         additionalProperties => 0,
1691         properties => {
1692             node => get_standard_option('pve-node'),
1693         },
1694     },
1695     returns => { type => 'string' },
1696     code => sub {
1697         my ($param) = @_;
1698
1699         PVE::CephTools::check_ceph_inited();
1700
1701         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1702
1703         die "not fully configured - missing '$pve_ckeyring_path'\n"
1704             if ! -f $pve_ckeyring_path;
1705
1706         my $rados = PVE::RADOS->new();
1707
1708         my $stat = $rados->mon_command({ prefix => 'osd dump' });
1709
1710         return $stat->{flags} // '';
1711     }});
1712
1713 __PACKAGE__->register_method ({
1714     name => 'set_flag',
1715     path => 'flags/{flag}',
1716     method => 'POST',
1717     description => "Set a ceph flag",
1718     proxyto => 'node',
1719     protected => 1,
1720     permissions => {
1721         check => ['perm', '/', [ 'Sys.Modify' ]],
1722     },
1723     parameters => {
1724         additionalProperties => 0,
1725         properties => {
1726             node => get_standard_option('pve-node'),
1727             flag => {
1728                 description => 'The ceph flag to set/unset',
1729                 type => 'string',
1730                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1731             },
1732         },
1733     },
1734     returns => { type => 'null' },
1735     code => sub {
1736         my ($param) = @_;
1737
1738         PVE::CephTools::check_ceph_inited();
1739
1740         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1741
1742         die "not fully configured - missing '$pve_ckeyring_path'\n"
1743             if ! -f $pve_ckeyring_path;
1744
1745         my $set = $param->{set} // !$param->{unset};
1746         my $rados = PVE::RADOS->new();
1747
1748         $rados->mon_command({
1749             prefix => "osd set",
1750             key => $param->{flag},
1751         });
1752
1753         return undef;
1754     }});
1755
1756 __PACKAGE__->register_method ({
1757     name => 'unset_flag',
1758     path => 'flags/{flag}',
1759     method => 'DELETE',
1760     description => "Unset a ceph flag",
1761     proxyto => 'node',
1762     protected => 1,
1763     permissions => {
1764         check => ['perm', '/', [ 'Sys.Modify' ]],
1765     },
1766     parameters => {
1767         additionalProperties => 0,
1768         properties => {
1769             node => get_standard_option('pve-node'),
1770             flag => {
1771                 description => 'The ceph flag to set/unset',
1772                 type => 'string',
1773                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1774             },
1775         },
1776     },
1777     returns => { type => 'null' },
1778     code => sub {
1779         my ($param) = @_;
1780
1781         PVE::CephTools::check_ceph_inited();
1782
1783         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1784
1785         die "not fully configured - missing '$pve_ckeyring_path'\n"
1786             if ! -f $pve_ckeyring_path;
1787
1788         my $set = $param->{set} // !$param->{unset};
1789         my $rados = PVE::RADOS->new();
1790
1791         $rados->mon_command({
1792             prefix => "osd unset",
1793             key => $param->{flag},
1794         });
1795
1796         return undef;
1797     }});
1798
1799 __PACKAGE__->register_method ({
1800     name => 'destroypool',
1801     path => 'pools/{name}',
1802     method => 'DELETE',
1803     description => "Destroy pool",
1804     proxyto => 'node',
1805     protected => 1,
1806     permissions => {
1807         check => ['perm', '/', [ 'Sys.Modify' ]],
1808     },
1809     parameters => {
1810         additionalProperties => 0,
1811         properties => {
1812             node => get_standard_option('pve-node'),
1813             name => {
1814                 description => "The name of the pool. It must be unique.",
1815                 type => 'string',
1816             },
1817             force => {
1818                 description => "If true, destroys pool even if in use",
1819                 type => 'boolean',
1820                 optional => 1,
1821                 default => 0,
1822             }
1823         },
1824     },
1825     returns => { type => 'null' },
1826     code => sub {
1827         my ($param) = @_;
1828
1829         PVE::CephTools::check_ceph_inited();
1830
1831         my $pool = $param->{name};
1832
1833         # if not forced, destroy ceph pool only when no
1834         # vm disks are on it anymore
1835         if (!$param->{force}) {
1836             my $storagecfg = PVE::Storage::config();
1837             foreach my $storageid (keys %{$storagecfg->{ids}}) {
1838                 my $storage = $storagecfg->{ids}->{$storageid};
1839                 next if $storage->{type} ne 'rbd';
1840                 next if $storage->{pool} ne $pool;
1841
1842                 # check if any vm disks are on the pool
1843                 my $res = PVE::Storage::vdisk_list($storagecfg, $storageid);
1844                 die "ceph pool '$pool' still in use by storage '$storageid'\n"
1845                     if @{$res->{$storageid}} != 0;
1846             }
1847         }
1848
1849         my $rados = PVE::RADOS->new();
1850         # fixme: '--yes-i-really-really-mean-it'
1851         $rados->mon_command({
1852             prefix => "osd pool delete",
1853             pool => $pool,
1854             pool2 => $pool,
1855             sure => '--yes-i-really-really-mean-it',
1856             format => 'plain',
1857         });
1858
1859         return undef;
1860     }});
1861
1862
1863 __PACKAGE__->register_method ({
1864     name => 'crush',
1865     path => 'crush',
1866     method => 'GET',
1867     description => "Get OSD crush map",
1868     proxyto => 'node',
1869     protected => 1,
1870     permissions => {
1871         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1872     },
1873     parameters => {
1874         additionalProperties => 0,
1875         properties => {
1876             node => get_standard_option('pve-node'),
1877         },
1878     },
1879     returns => { type => 'string' },
1880     code => sub {
1881         my ($param) = @_;
1882
1883         PVE::CephTools::check_ceph_inited();
1884
1885         # this produces JSON (difficult to read for the user)
1886         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
1887
1888         my $txt = '';
1889
1890         my $mapfile = "/var/tmp/ceph-crush.map.$$";
1891         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
1892
1893         my $rados = PVE::RADOS->new();
1894
1895         eval {
1896             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
1897             PVE::Tools::file_set_contents($mapfile, $bindata);
1898             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
1899             $txt = PVE::Tools::file_get_contents($mapdata);
1900         };
1901         my $err = $@;
1902
1903         unlink $mapfile;
1904         unlink $mapdata;
1905
1906         die $err if $err;
1907
1908         return $txt;
1909     }});
1910
1911 __PACKAGE__->register_method({
1912     name => 'log',
1913     path => 'log',
1914     method => 'GET',
1915     description => "Read ceph log",
1916     proxyto => 'node',
1917     permissions => {
1918         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
1919     },
1920     protected => 1,
1921     parameters => {
1922         additionalProperties => 0,
1923         properties => {
1924             node => get_standard_option('pve-node'),
1925             start => {
1926                 type => 'integer',
1927                 minimum => 0,
1928                 optional => 1,
1929             },
1930             limit => {
1931                 type => 'integer',
1932                 minimum => 0,
1933                 optional => 1,
1934             },
1935         },
1936     },
1937     returns => {
1938         type => 'array',
1939         items => {
1940             type => "object",
1941             properties => {
1942                 n => {
1943                   description=>  "Line number",
1944                   type=> 'integer',
1945                 },
1946                 t => {
1947                   description=>  "Line text",
1948                   type => 'string',
1949                 }
1950             }
1951         }
1952     },
1953     code => sub {
1954         my ($param) = @_;
1955
1956         my $rpcenv = PVE::RPCEnvironment::get();
1957         my $user = $rpcenv->get_user();
1958         my $node = $param->{node};
1959
1960         my $logfile = "/var/log/ceph/ceph.log";
1961         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
1962
1963         $rpcenv->set_result_attrib('total', $count);
1964
1965         return $lines;
1966     }});
1967
1968 __PACKAGE__->register_method ({
1969     name => 'rules',
1970     path => 'rules',
1971     method => 'GET',
1972     description => "List ceph rules.",
1973     proxyto => 'node',
1974     protected => 1,
1975     permissions => {
1976         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1977     },
1978     parameters => {
1979         additionalProperties => 0,
1980         properties => {
1981             node => get_standard_option('pve-node'),
1982         },
1983     },
1984     returns => {
1985         type => 'array',
1986         items => {
1987             type => "object",
1988             properties => {},
1989         },
1990         links => [ { rel => 'child', href => "{name}" } ],
1991     },
1992     code => sub {
1993         my ($param) = @_;
1994
1995         PVE::CephTools::check_ceph_inited();
1996
1997         my $rados = PVE::RADOS->new();
1998
1999         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
2000
2001         my $res = [];
2002
2003         foreach my $rule (@$rules) {
2004             push @$res, { name => $rule };
2005         }
2006
2007         return $res;
2008     }});