PVE/API2/Ceph.pm

   1 package PVE::API2::CephOSD;
   2
   3 use strict;
   4 use warnings;
   5 use Cwd qw(abs_path);
   6 use Net::IP;
   7
   8 use PVE::SafeSyslog;
   9 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
  10 use PVE::Exception qw(raise raise_param_exc);
  11 use PVE::INotify;
  12 use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
  13 use PVE::AccessControl;
  14 use PVE::Storage;
  15 use PVE::API2::Storage::Config;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::JSONSchema qw(get_standard_option);
  19 use PVE::RADOS;
  20 use PVE::CephTools;
  21 use PVE::Diskmanage;
  22
  23 use base qw(PVE::RESTHandler);
  24
  25 use Data::Dumper; # fixme: remove
  26
  27 my $get_osd_status = sub {
  28     my ($rados, $osdid) = @_;
  29
  30     my $stat = $rados->mon_command({ prefix => 'osd dump' });
  31
  32     my $osdlist = $stat->{osds} || [];
  33
  34     my $flags = $stat->{flags} || undef;
  35
  36     my $osdstat;
  37     foreach my $d (@$osdlist) {
  38         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  39     }
  40     if (defined($osdid)) {
  41         die "no such OSD '$osdid'\n" if !$osdstat->{$osdid};
  42         return $osdstat->{$osdid};
  43     }
  44
  45     return wantarray? ($osdstat, $flags):$osdstat;
  46 };
  47
  48 my $get_osd_usage = sub {
  49     my ($rados) = @_;
  50
  51     my $osdlist = $rados->mon_command({ prefix => 'pg dump',
  52                                         dumpcontents => [ 'osds' ]}) || [];
  53
  54     my $osdstat;
  55     foreach my $d (@$osdlist) {
  56         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  57     }
  58
  59     return $osdstat;
  60 };
  61
  62 __PACKAGE__->register_method ({
  63     name => 'index',
  64     path => '',
  65     method => 'GET',
  66     description => "Get Ceph osd list/tree.",
  67     proxyto => 'node',
  68     protected => 1,
  69     permissions => {
  70         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  71     },
  72     parameters => {
  73         additionalProperties => 0,
  74         properties => {
  75             node => get_standard_option('pve-node'),
  76         },
  77     },
  78     # fixme: return a list instead of extjs tree format ?
  79     returns => {
  80         type => "object",
  81     },
  82     code => sub {
  83         my ($param) = @_;
  84
  85         PVE::CephTools::check_ceph_inited();
  86
  87         my $rados = PVE::RADOS->new();
  88         my $res = $rados->mon_command({ prefix => 'osd tree' });
  89
  90         die "no tree nodes found\n" if !($res && $res->{nodes});
  91
  92         my ($osdhash, $flags) = &$get_osd_status($rados);
  93
  94         my $usagehash = &$get_osd_usage($rados);
  95
  96         my $osdmetadata_tmp = $rados->mon_command({ prefix => 'osd metadata' });
  97
  98         my $osdmetadata = {};
  99         foreach my $osd (@$osdmetadata_tmp) {
 100             $osdmetadata->{$osd->{id}} = $osd;
 101         }
 102
 103         my $nodes = {};
 104         my $newnodes = {};
 105         foreach my $e (@{$res->{nodes}}) {
 106             $nodes->{$e->{id}} = $e;
 107
 108             my $new = {
 109                 id => $e->{id},
 110                 name => $e->{name},
 111                 type => $e->{type}
 112             };
 113
 114             foreach my $opt (qw(status crush_weight reweight device_class)) {
 115                 $new->{$opt} = $e->{$opt} if defined($e->{$opt});
 116             }
 117
 118             if (my $stat = $osdhash->{$e->{id}}) {
 119                 $new->{in} = $stat->{in} if defined($stat->{in});
 120             }
 121
 122             if (my $stat = $usagehash->{$e->{id}}) {
 123                 $new->{total_space} = ($stat->{kb} || 1) * 1024;
 124                 $new->{bytes_used} = ($stat->{kb_used} || 0) * 1024;
 125                 $new->{percent_used} = ($new->{bytes_used}*100)/$new->{total_space};
 126                 if (my $d = $stat->{perf_stat}) {
 127                     $new->{commit_latency_ms} = $d->{commit_latency_ms};
 128                     $new->{apply_latency_ms} = $d->{apply_latency_ms};
 129                 }
 130             }
 131
 132             my $osdmd = $osdmetadata->{$e->{id}};
 133             if ($e->{type} eq 'osd' && $osdmd) {
 134                 if ($osdmd->{bluefs}) {
 135                     $new->{osdtype} = 'bluestore';
 136                     $new->{blfsdev} = $osdmd->{bluestore_bdev_dev_node};
 137                     $new->{dbdev} = $osdmd->{bluefs_db_dev_node};
 138                     $new->{waldev} = $osdmd->{bluefs_wal_dev_node};
 139                 } else {
 140                     $new->{osdtype} = 'filestore';
 141                 }
 142             }
 143
 144             $newnodes->{$e->{id}} = $new;
 145         }
 146
 147         foreach my $e (@{$res->{nodes}}) {
 148             my $new = $newnodes->{$e->{id}};
 149             if ($e->{children} && scalar(@{$e->{children}})) {
 150                 $new->{children} = [];
 151                 $new->{leaf} = 0;
 152                 foreach my $cid (@{$e->{children}}) {
 153                     $nodes->{$cid}->{parent} = $e->{id};
 154                     if ($nodes->{$cid}->{type} eq 'osd' &&
 155                         $e->{type} eq 'host') {
 156                         $newnodes->{$cid}->{host} = $e->{name};
 157                     }
 158                     push @{$new->{children}}, $newnodes->{$cid};
 159                 }
 160             } else {
 161                 $new->{leaf} = ($e->{id} >= 0) ? 1 : 0;
 162             }
 163         }
 164
 165         my $roots = [];
 166         foreach my $e (@{$res->{nodes}}) {
 167             if (!$nodes->{$e->{id}}->{parent}) {
 168                 push @$roots, $newnodes->{$e->{id}};
 169             }
 170         }
 171
 172         die "no root node\n" if !@$roots;
 173
 174         my $data = { root => { leaf =>  0, children => $roots } };
 175
 176         # we want this for the noout flag
 177         $data->{flags} = $flags if $flags;
 178
 179         return $data;
 180     }});
 181
 182 __PACKAGE__->register_method ({
 183     name => 'createosd',
 184     path => '',
 185     method => 'POST',
 186     description => "Create OSD",
 187     proxyto => 'node',
 188     protected => 1,
 189     parameters => {
 190         additionalProperties => 0,
 191         properties => {
 192             node => get_standard_option('pve-node'),
 193             dev => {
 194                 description => "Block device name.",
 195                 type => 'string',
 196             },
 197             journal_dev => {
 198                 description => "Block device name for journal (filestore) or block.db (bluestore).",
 199                 optional => 1,
 200                 type => 'string',
 201             },
 202             wal_dev => {
 203                 description => "Block device name for block.wal (bluestore only).",
 204                 optional => 1,
 205                 type => 'string',
 206             },
 207             fstype => {
 208                 description => "File system type (filestore only).",
 209                 type => 'string',
 210                 enum => ['xfs', 'ext4'],
 211                 default => 'xfs',
 212                 optional => 1,
 213             },
 214             bluestore => {
 215                 description => "Use bluestore instead of filestore. This is the default.",
 216                 type => 'boolean',
 217                 default => 1,
 218                 optional => 1,
 219             },
 220         },
 221     },
 222     returns => { type => 'string' },
 223     code => sub {
 224         my ($param) = @_;
 225
 226         my $rpcenv = PVE::RPCEnvironment::get();
 227
 228         my $authuser = $rpcenv->get_user();
 229
 230         raise_param_exc({ 'bluestore' => "conflicts with parameter 'fstype'" })
 231             if (defined($param->{fstype}) && defined($param->{bluestore}) && $param->{bluestore});
 232
 233         PVE::CephTools::check_ceph_inited();
 234
 235         PVE::CephTools::setup_pve_symlinks();
 236
 237         PVE::CephTools::check_ceph_installed('ceph_osd');
 238
 239         my $bluestore = $param->{bluestore} // 1;
 240
 241         my $journal_dev;
 242         my $wal_dev;
 243
 244         if ($param->{journal_dev} && ($param->{journal_dev} ne $param->{dev})) {
 245             $journal_dev = PVE::Diskmanage::verify_blockdev_path($param->{journal_dev});
 246         }
 247
 248         if ($param->{wal_dev} &&
 249             ($param->{wal_dev} ne $param->{dev}) &&
 250             (!$param->{journal_dev} || $param->{wal_dev} ne $param->{journal_dev})) {
 251             raise_param_exc({ 'wal_dev' => "can only be set with paramater 'bluestore'"})
 252                 if !$bluestore;
 253             $wal_dev = PVE::Diskmanage::verify_blockdev_path($param->{wal_dev});
 254         }
 255
 256         $param->{dev} = PVE::Diskmanage::verify_blockdev_path($param->{dev});
 257
 258         my $devname = $param->{dev};
 259         $devname =~ s|/dev/||;
 260
 261         my $disklist = PVE::Diskmanage::get_disks($devname, 1);
 262
 263         my $diskinfo = $disklist->{$devname};
 264         die "unable to get device info for '$devname'\n"
 265             if !$diskinfo;
 266
 267         die "device '$param->{dev}' is in use\n"
 268             if $diskinfo->{used};
 269
 270         my $devpath = $diskinfo->{devpath};
 271         my $rados = PVE::RADOS->new();
 272         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 273         die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid};
 274
 275         my $fsid = $monstat->{monmap}->{fsid};
 276         $fsid = $1 if $fsid =~ m/^([0-9a-f\-]+)$/;
 277
 278         my $ceph_bootstrap_osd_keyring = PVE::CephTools::get_config('ceph_bootstrap_osd_keyring');
 279
 280         if (! -f $ceph_bootstrap_osd_keyring) {
 281             my $bindata = $rados->mon_command({ prefix => 'auth get', entity => 'client.bootstrap-osd', format => 'plain' });
 282             PVE::Tools::file_set_contents($ceph_bootstrap_osd_keyring, $bindata);
 283         };
 284
 285         my $worker = sub {
 286             my $upid = shift;
 287
 288             my $fstype = $param->{fstype} || 'xfs';
 289
 290
 291             my $ccname = PVE::CephTools::get_config('ccname');
 292
 293             my $cmd = ['ceph-disk', 'prepare', '--zap-disk',
 294                        '--cluster', $ccname, '--cluster-uuid', $fsid ];
 295
 296             if ($bluestore) {
 297                 print "create OSD on $devpath (bluestore)\n";
 298                 push @$cmd, '--bluestore';
 299
 300                 if ($journal_dev) {
 301                     print "using device '$journal_dev' for block.db\n";
 302                     push @$cmd, '--block.db', $journal_dev;
 303                 }
 304
 305                 if ($wal_dev) {
 306                     print "using device '$wal_dev' for block.wal\n";
 307                     push @$cmd, '--block.wal', $wal_dev;
 308                 }
 309
 310                 push @$cmd, $devpath;
 311             } else {
 312                 print "create OSD on $devpath ($fstype)\n";
 313                 push @$cmd, '--filestore', '--fs-type', $fstype;
 314                 if ($journal_dev) {
 315                     print "using device '$journal_dev' for journal\n";
 316                     push @$cmd, '--journal-dev', $devpath, $journal_dev;
 317                 } else {
 318                     push @$cmd, $devpath;
 319                 }
 320             }
 321
 322
 323             run_command($cmd);
 324         };
 325
 326         return $rpcenv->fork_worker('cephcreateosd', $devname,  $authuser, $worker);
 327     }});
 328
 329 __PACKAGE__->register_method ({
 330     name => 'destroyosd',
 331     path => '{osdid}',
 332     method => 'DELETE',
 333     description => "Destroy OSD",
 334     proxyto => 'node',
 335     protected => 1,
 336     parameters => {
 337         additionalProperties => 0,
 338         properties => {
 339             node => get_standard_option('pve-node'),
 340             osdid => {
 341                 description => 'OSD ID',
 342                 type => 'integer',
 343             },
 344             cleanup => {
 345                 description => "If set, we remove partition table entries.",
 346                 type => 'boolean',
 347                 optional => 1,
 348                 default => 0,
 349             },
 350         },
 351     },
 352     returns => { type => 'string' },
 353     code => sub {
 354         my ($param) = @_;
 355
 356         my $rpcenv = PVE::RPCEnvironment::get();
 357
 358         my $authuser = $rpcenv->get_user();
 359
 360         PVE::CephTools::check_ceph_inited();
 361
 362         my $osdid = $param->{osdid};
 363
 364         my $rados = PVE::RADOS->new();
 365         my $osdstat = &$get_osd_status($rados, $osdid);
 366
 367         die "osd is in use (in == 1)\n" if $osdstat->{in};
 368         #&$run_ceph_cmd(['osd', 'out', $osdid]);
 369
 370         die "osd is still runnung (up == 1)\n" if $osdstat->{up};
 371
 372         my $osdsection = "osd.$osdid";
 373
 374         my $worker = sub {
 375             my $upid = shift;
 376
 377             # reopen with longer timeout
 378             $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
 379
 380             print "destroy OSD $osdsection\n";
 381
 382             eval {
 383                 PVE::CephTools::ceph_service_cmd('stop', $osdsection);
 384                 PVE::CephTools::ceph_service_cmd('disable', $osdsection);
 385             };
 386             warn $@ if $@;
 387
 388             print "Remove $osdsection from the CRUSH map\n";
 389             $rados->mon_command({ prefix => "osd crush remove", name => $osdsection, format => 'plain' });
 390
 391             print "Remove the $osdsection authentication key.\n";
 392             $rados->mon_command({ prefix => "auth del", entity => $osdsection, format => 'plain' });
 393
 394             print "Remove OSD $osdsection\n";
 395             $rados->mon_command({ prefix => "osd rm", ids => [ $osdsection ], format => 'plain' });
 396
 397             # try to unmount from standard mount point
 398             my $mountpoint = "/var/lib/ceph/osd/ceph-$osdid";
 399
 400             my $disks_to_wipe = {};
 401             my $remove_partition = sub {
 402                 my ($part) = @_;
 403
 404                 return if !$part || (! -b $part );
 405                 my $partnum = PVE::Diskmanage::get_partnum($part);
 406                 my $devpath = PVE::Diskmanage::get_blockdev($part);
 407
 408                 print "remove partition $part (disk '${devpath}', partnum $partnum)\n";
 409                 eval { run_command(['/sbin/sgdisk', '-d', $partnum, "${devpath}"]); };
 410                 warn $@ if $@;
 411
 412                 $disks_to_wipe->{$devpath} = 1;
 413             };
 414
 415             my $partitions_to_remove = [];
 416
 417             if ($param->{cleanup}) {
 418                 if (my $fd = IO::File->new("/proc/mounts", "r")) {
 419                     while (defined(my $line = <$fd>)) {
 420                         my ($dev, $path, $fstype) = split(/\s+/, $line);
 421                         next if !($dev && $path && $fstype);
 422                         next if $dev !~ m|^/dev/|;
 423                         if ($path eq $mountpoint) {
 424                             my $data_part = abs_path($dev);
 425                             push @$partitions_to_remove, $data_part;
 426                             last;
 427                         }
 428                     }
 429                     close($fd);
 430                 }
 431
 432                 foreach my $path (qw(journal block block.db block.wal)) {
 433                     my $part = abs_path("$mountpoint/$path");
 434                     if ($part) {
 435                         push @$partitions_to_remove, $part;
 436                     }
 437                 }
 438             }
 439
 440             print "Unmount OSD $osdsection from  $mountpoint\n";
 441             eval { run_command(['/bin/umount', $mountpoint]); };
 442             if (my $err = $@) {
 443                 warn $err;
 444             } elsif ($param->{cleanup}) {
 445                 #be aware of the ceph udev rules which can remount.
 446                 foreach my $part (@$partitions_to_remove) {
 447                     $remove_partition->($part);
 448                 }
 449                 my @wipe_cmd = qw(/bin/dd if=/dev/zero bs=1M count=200 conv=fdatasync);
 450                 foreach my $devpath (keys %$disks_to_wipe) {
 451                     print "wipe disk: $devpath\n";
 452                     eval { run_command([@wipe_cmd, "of=${devpath}"]) };
 453                     warn $@ if $@;
 454                 }
 455             }
 456         };
 457
 458         return $rpcenv->fork_worker('cephdestroyosd', $osdsection,  $authuser, $worker);
 459     }});
 460
 461 __PACKAGE__->register_method ({
 462     name => 'in',
 463     path => '{osdid}/in',
 464     method => 'POST',
 465     description => "ceph osd in",
 466     proxyto => 'node',
 467     protected => 1,
 468     permissions => {
 469         check => ['perm', '/', [ 'Sys.Modify' ]],
 470     },
 471     parameters => {
 472         additionalProperties => 0,
 473         properties => {
 474             node => get_standard_option('pve-node'),
 475             osdid => {
 476                 description => 'OSD ID',
 477                 type => 'integer',
 478             },
 479         },
 480     },
 481     returns => { type => "null" },
 482     code => sub {
 483         my ($param) = @_;
 484
 485         PVE::CephTools::check_ceph_inited();
 486
 487         my $osdid = $param->{osdid};
 488
 489         my $rados = PVE::RADOS->new();
 490
 491         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 492
 493         my $osdsection = "osd.$osdid";
 494
 495         $rados->mon_command({ prefix => "osd in", ids => [ $osdsection ], format => 'plain' });
 496
 497         return undef;
 498     }});
 499
 500 __PACKAGE__->register_method ({
 501     name => 'out',
 502     path => '{osdid}/out',
 503     method => 'POST',
 504     description => "ceph osd out",
 505     proxyto => 'node',
 506     protected => 1,
 507     permissions => {
 508         check => ['perm', '/', [ 'Sys.Modify' ]],
 509     },
 510     parameters => {
 511         additionalProperties => 0,
 512         properties => {
 513             node => get_standard_option('pve-node'),
 514             osdid => {
 515                 description => 'OSD ID',
 516                 type => 'integer',
 517             },
 518         },
 519     },
 520     returns => { type => "null" },
 521     code => sub {
 522         my ($param) = @_;
 523
 524         PVE::CephTools::check_ceph_inited();
 525
 526         my $osdid = $param->{osdid};
 527
 528         my $rados = PVE::RADOS->new();
 529
 530         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 531
 532         my $osdsection = "osd.$osdid";
 533
 534         $rados->mon_command({ prefix => "osd out", ids => [ $osdsection ], format => 'plain' });
 535
 536         return undef;
 537     }});
 538
 539 package PVE::API2::Ceph;
 540
 541 use strict;
 542 use warnings;
 543 use File::Basename;
 544 use File::Path;
 545 use POSIX qw (LONG_MAX);
 546 use Cwd qw(abs_path);
 547 use IO::Dir;
 548 use UUID;
 549 use Net::IP;
 550
 551 use PVE::SafeSyslog;
 552 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
 553 use PVE::Exception qw(raise raise_param_exc);
 554 use PVE::INotify;
 555 use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
 556 use PVE::AccessControl;
 557 use PVE::Storage;
 558 use PVE::RESTHandler;
 559 use PVE::RPCEnvironment;
 560 use PVE::JSONSchema qw(get_standard_option);
 561 use JSON;
 562 use PVE::RADOS;
 563 use PVE::CephTools;
 564 use PVE::Network;
 565
 566 use base qw(PVE::RESTHandler);
 567
 568 use Data::Dumper; # fixme: remove
 569
 570 my $pve_osd_default_journal_size = 1024*5;
 571
 572 __PACKAGE__->register_method ({
 573     subclass => "PVE::API2::CephOSD",
 574     path => 'osd',
 575 });
 576
 577 __PACKAGE__->register_method ({
 578     name => 'index',
 579     path => '',
 580     method => 'GET',
 581     description => "Directory index.",
 582     permissions => { user => 'all' },
 583     permissions => {
 584         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 585     },
 586     parameters => {
 587         additionalProperties => 0,
 588         properties => {
 589             node => get_standard_option('pve-node'),
 590         },
 591     },
 592     returns => {
 593         type => 'array',
 594         items => {
 595             type => "object",
 596             properties => {},
 597         },
 598         links => [ { rel => 'child', href => "{name}" } ],
 599     },
 600     code => sub {
 601         my ($param) = @_;
 602
 603         my $result = [
 604             { name => 'init' },
 605             { name => 'mon' },
 606             { name => 'osd' },
 607             { name => 'pools' },
 608             { name => 'stop' },
 609             { name => 'start' },
 610             { name => 'status' },
 611             { name => 'crush' },
 612             { name => 'config' },
 613             { name => 'log' },
 614             { name => 'disks' },
 615             { name => 'flags' },
 616             { name => 'rules' },
 617         ];
 618
 619         return $result;
 620     }});
 621
 622 __PACKAGE__->register_method ({
 623     name => 'disks',
 624     path => 'disks',
 625     method => 'GET',
 626     description => "List local disks.",
 627     proxyto => 'node',
 628     protected => 1,
 629     permissions => {
 630         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 631     },
 632     parameters => {
 633         additionalProperties => 0,
 634         properties => {
 635             node => get_standard_option('pve-node'),
 636             type => {
 637                 description => "Only list specific types of disks.",
 638                 type => 'string',
 639                 enum => ['unused', 'journal_disks'],
 640                 optional => 1,
 641             },
 642         },
 643     },
 644     returns => {
 645         type => 'array',
 646         items => {
 647             type => "object",
 648             properties => {
 649                 dev => { type => 'string' },
 650                 used => { type => 'string', optional => 1 },
 651                 gpt => { type => 'boolean' },
 652                 size => { type => 'integer' },
 653                 osdid => { type => 'integer' },
 654                 vendor =>  { type => 'string', optional => 1 },
 655                 model =>  { type => 'string', optional => 1 },
 656                 serial =>  { type => 'string', optional => 1 },
 657             },
 658         },
 659         # links => [ { rel => 'child', href => "{}" } ],
 660     },
 661     code => sub {
 662         my ($param) = @_;
 663
 664         PVE::CephTools::check_ceph_inited();
 665
 666         my $disks = PVE::Diskmanage::get_disks(undef, 1);
 667
 668         my $res = [];
 669         foreach my $dev (keys %$disks) {
 670             my $d = $disks->{$dev};
 671             if ($param->{type}) {
 672                 if ($param->{type} eq 'journal_disks') {
 673                     next if $d->{osdid} >= 0;
 674                     next if !$d->{gpt};
 675                 } elsif ($param->{type} eq 'unused') {
 676                     next if $d->{used};
 677                 } else {
 678                     die "internal error"; # should not happen
 679                 }
 680             }
 681
 682             $d->{dev} = "/dev/$dev";
 683             push @$res, $d;
 684         }
 685
 686         return $res;
 687     }});
 688
 689 __PACKAGE__->register_method ({
 690     name => 'config',
 691     path => 'config',
 692     method => 'GET',
 693     permissions => {
 694         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 695     },
 696     description => "Get Ceph configuration.",
 697     parameters => {
 698         additionalProperties => 0,
 699         properties => {
 700             node => get_standard_option('pve-node'),
 701         },
 702     },
 703     returns => { type => 'string' },
 704     code => sub {
 705         my ($param) = @_;
 706
 707         PVE::CephTools::check_ceph_inited();
 708
 709         my $path = PVE::CephTools::get_config('pve_ceph_cfgpath');
 710         return PVE::Tools::file_get_contents($path);
 711
 712     }});
 713
 714 my $add_storage = sub {
 715     my ($pool, $storeid) = @_;
 716
 717     my $storage_params = {
 718         type => 'rbd',
 719         pool => $pool,
 720         storage => $storeid,
 721         krbd => 0,
 722         content => 'rootdir,images',
 723     };
 724
 725     PVE::API2::Storage::Config->create($storage_params);
 726 };
 727
 728 my $get_storages = sub {
 729     my ($pool) = @_;
 730
 731     my $cfg = PVE::Storage::config();
 732
 733     my $storages = $cfg->{ids};
 734     my $res = {};
 735     foreach my $storeid (keys %$storages) {
 736         my $curr = $storages->{$storeid};
 737         $res->{$storeid} = $storages->{$storeid}
 738             if $curr->{type} eq 'rbd' && $pool eq $curr->{pool};
 739     }
 740
 741     return $res;
 742 };
 743
 744 __PACKAGE__->register_method ({
 745     name => 'listmon',
 746     path => 'mon',
 747     method => 'GET',
 748     description => "Get Ceph monitor list.",
 749     proxyto => 'node',
 750     protected => 1,
 751     permissions => {
 752         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 753     },
 754     parameters => {
 755         additionalProperties => 0,
 756         properties => {
 757             node => get_standard_option('pve-node'),
 758         },
 759     },
 760     returns => {
 761         type => 'array',
 762         items => {
 763             type => "object",
 764             properties => {
 765                 name => { type => 'string' },
 766                 addr => { type => 'string' },
 767             },
 768         },
 769         links => [ { rel => 'child', href => "{name}" } ],
 770     },
 771     code => sub {
 772         my ($param) = @_;
 773
 774         PVE::CephTools::check_ceph_inited();
 775
 776         my $res = [];
 777
 778         my $cfg = PVE::CephTools::parse_ceph_config();
 779
 780         my $monhash = {};
 781         foreach my $section (keys %$cfg) {
 782             my $d = $cfg->{$section};
 783             if ($section =~ m/^mon\.(\S+)$/) {
 784                 my $monid = $1;
 785                 if ($d->{'mon addr'} && $d->{'host'}) {
 786                     $monhash->{$monid} = {
 787                         addr => $d->{'mon addr'},
 788                         host => $d->{'host'},
 789                         name => $monid,
 790                     }
 791                 }
 792             }
 793         }
 794
 795         eval {
 796             my $rados = PVE::RADOS->new();
 797             my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 798             my $mons = $monstat->{monmap}->{mons};
 799             foreach my $d (@$mons) {
 800                 next if !defined($d->{name});
 801                 $monhash->{$d->{name}}->{rank} = $d->{rank};
 802                 $monhash->{$d->{name}}->{addr} = $d->{addr};
 803                 if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
 804                     $monhash->{$d->{name}}->{quorum} = 1;
 805                 }
 806             }
 807         };
 808         warn $@ if $@;
 809
 810         return PVE::RESTHandler::hash_to_array($monhash, 'name');
 811     }});
 812
 813 __PACKAGE__->register_method ({
 814     name => 'init',
 815     path => 'init',
 816     method => 'POST',
 817     description => "Create initial ceph default configuration and setup symlinks.",
 818     proxyto => 'node',
 819     protected => 1,
 820     permissions => {
 821         check => ['perm', '/', [ 'Sys.Modify' ]],
 822     },
 823     parameters => {
 824         additionalProperties => 0,
 825         properties => {
 826             node => get_standard_option('pve-node'),
 827             network => {
 828                 description => "Use specific network for all ceph related traffic",
 829                 type => 'string', format => 'CIDR',
 830                 optional => 1,
 831                 maxLength => 128,
 832             },
 833             size => {
 834                 description => 'Targeted number of replicas per object',
 835                 type => 'integer',
 836                 default => 3,
 837                 optional => 1,
 838                 minimum => 1,
 839                 maximum => 7,
 840             },
 841             min_size => {
 842                 description => 'Minimum number of available replicas per object to allow I/O',
 843                 type => 'integer',
 844                 default => 2,
 845                 optional => 1,
 846                 minimum => 1,
 847                 maximum => 7,
 848             },
 849             pg_bits => {
 850                 description => "Placement group bits, used to specify the " .
 851                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 852                     "default pg num' does not work for default pools.",
 853                 type => 'integer',
 854                 default => 6,
 855                 optional => 1,
 856                 minimum => 6,
 857                 maximum => 14,
 858             },
 859             disable_cephx => {
 860                 description => "Disable cephx authentification.\n\n" .
 861                     "WARNING: cephx is a security feature protecting against " .
 862                     "man-in-the-middle attacks. Only consider disabling cephx ".
 863                     "if your network is private!",
 864                 type => 'boolean',
 865                 optional => 1,
 866                 default => 0,
 867             },
 868         },
 869     },
 870     returns => { type => 'null' },
 871     code => sub {
 872         my ($param) = @_;
 873
 874         my $version = PVE::CephTools::get_local_version(1);
 875
 876         if (!$version || $version < 12) {
 877             die "Ceph Luminous required - please run 'pveceph install'\n";
 878         } else {
 879             PVE::CephTools::check_ceph_installed('ceph_bin');
 880         }
 881
 882         # simply load old config if it already exists
 883         my $cfg = PVE::CephTools::parse_ceph_config();
 884
 885         if (!$cfg->{global}) {
 886
 887             my $fsid;
 888             my $uuid;
 889
 890             UUID::generate($uuid);
 891             UUID::unparse($uuid, $fsid);
 892
 893             my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 894
 895             $cfg->{global} = {
 896                 'fsid' => $fsid,
 897                 'auth cluster required' => $auth,
 898                 'auth service required' => $auth,
 899                 'auth client required' => $auth,
 900                 'osd journal size' => $pve_osd_default_journal_size,
 901                 'osd pool default size' => $param->{size} // 3,
 902                 'osd pool default min size' => $param->{min_size} // 2,
 903                 'mon allow pool delete' => 'true',
 904             };
 905
 906             # this does not work for default pools
 907             #'osd pool default pg num' => $pg_num,
 908             #'osd pool default pgp num' => $pg_num,
 909         }
 910
 911         $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 912         $cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring';
 913
 914         if ($param->{pg_bits}) {
 915             $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 916             $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 917         }
 918
 919         if ($param->{network}) {
 920             $cfg->{global}->{'public network'} = $param->{network};
 921             $cfg->{global}->{'cluster network'} = $param->{network};
 922         }
 923
 924         PVE::CephTools::write_ceph_config($cfg);
 925
 926         PVE::CephTools::setup_pve_symlinks();
 927
 928         return undef;
 929     }});
 930
 931 my $find_mon_ip = sub {
 932     my ($pubnet, $node, $overwrite_ip) = @_;
 933
 934     if (!$pubnet) {
 935         return $overwrite_ip // PVE::Cluster::remote_node_ip($node);
 936     }
 937
 938     my $allowed_ips = PVE::Network::get_local_ip_from_cidr($pubnet);
 939     die "No IP configured and up from ceph public network '$pubnet'\n"
 940         if scalar(@$allowed_ips) < 1;
 941
 942     if (!$overwrite_ip) {
 943         if (scalar(@$allowed_ips) == 1) {
 944             return $allowed_ips->[0];
 945         }
 946         die "Multiple IPs for ceph public network '$pubnet' detected on $node:\n".
 947             join("\n", @$allowed_ips) ."\nuse 'mon-address' to specify one of them.\n";
 948     } else {
 949         if (grep { $_ eq $overwrite_ip } @$allowed_ips) {
 950             return $overwrite_ip;
 951         }
 952         die "Monitor IP '$overwrite_ip' not in ceph public network '$pubnet'\n"
 953             if !PVE::Network::is_ip_in_cidr($overwrite_ip, $pubnet);
 954
 955         die "Specified monitor IP '$overwrite_ip' not configured or up on $node!\n";
 956     }
 957 };
 958
 959 my $create_mgr = sub {
 960     my ($rados, $id) = @_;
 961
 962     my $clustername = PVE::CephTools::get_config('ccname');
 963     my $mgrdir = "/var/lib/ceph/mgr/$clustername-$id";
 964     my $mgrkeyring = "$mgrdir/keyring";
 965     my $mgrname = "mgr.$id";
 966
 967     die "ceph manager directory '$mgrdir' already exists\n"
 968         if -d $mgrdir;
 969
 970     print "creating manager directory '$mgrdir'\n";
 971     mkdir $mgrdir;
 972     print "creating keys for '$mgrname'\n";
 973     my $output = $rados->mon_command({ prefix => 'auth get-or-create',
 974                                        entity => $mgrname,
 975                                        caps => [
 976                                            mon => 'allow profile mgr',
 977                                            osd => 'allow *',
 978                                            mds => 'allow *',
 979                                        ],
 980                                        format => 'plain'});
 981     PVE::Tools::file_set_contents($mgrkeyring, $output);
 982
 983     print "setting owner for directory\n";
 984     run_command(["chown", 'ceph:ceph', '-R', $mgrdir]);
 985
 986     print "enabling service 'ceph-mgr\@$id.service'\n";
 987     PVE::CephTools::ceph_service_cmd('enable', $mgrname);
 988     print "starting service 'ceph-mgr\@$id.service'\n";
 989     PVE::CephTools::ceph_service_cmd('start', $mgrname);
 990 };
 991
 992 my $destroy_mgr = sub {
 993     my ($mgrid) = @_;
 994
 995     my $clustername = PVE::CephTools::get_config('ccname');
 996     my $mgrname = "mgr.$mgrid";
 997     my $mgrdir = "/var/lib/ceph/mgr/$clustername-$mgrid";
 998
 999     die "ceph manager directory '$mgrdir' not found\n"
1000         if ! -d $mgrdir;
1001
1002     print "disabling service 'ceph-mgr\@$mgrid.service'\n";
1003     PVE::CephTools::ceph_service_cmd('disable', $mgrname);
1004     print "stopping service 'ceph-mgr\@$mgrid.service'\n";
1005     PVE::CephTools::ceph_service_cmd('stop', $mgrname);
1006
1007     print "removing manager directory '$mgrdir'\n";
1008     File::Path::remove_tree($mgrdir);
1009 };
1010
1011 __PACKAGE__->register_method ({
1012     name => 'createmon',
1013     path => 'mon',
1014     method => 'POST',
1015     description => "Create Ceph Monitor and Manager",
1016     proxyto => 'node',
1017     protected => 1,
1018     permissions => {
1019         check => ['perm', '/', [ 'Sys.Modify' ]],
1020     },
1021     parameters => {
1022         additionalProperties => 0,
1023         properties => {
1024             node => get_standard_option('pve-node'),
1025             id => {
1026                 type => 'string',
1027                 optional => 1,
1028                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1029                 description => "The ID for the monitor, when omitted the same as the nodename",
1030             },
1031             'exclude-manager' => {
1032                 type => 'boolean',
1033                 optional => 1,
1034                 default => 0,
1035                 description => "When set, only a monitor will be created.",
1036             },
1037             'mon-address' => {
1038                 description => 'Overwrites autodetected monitor IP address. ' .
1039                                'Must be in the public network of ceph.',
1040                 type => 'string', format => 'ip',
1041                 optional => 1,
1042             },
1043         },
1044     },
1045     returns => { type => 'string' },
1046     code => sub {
1047         my ($param) = @_;
1048
1049         PVE::CephTools::check_ceph_installed('ceph_mon');
1050
1051         PVE::CephTools::check_ceph_installed('ceph_mgr')
1052             if (!$param->{'exclude-manager'});
1053
1054         PVE::CephTools::check_ceph_inited();
1055
1056         PVE::CephTools::setup_pve_symlinks();
1057
1058         my $rpcenv = PVE::RPCEnvironment::get();
1059
1060         my $authuser = $rpcenv->get_user();
1061
1062         my $cfg = PVE::CephTools::parse_ceph_config();
1063
1064         my $moncount = 0;
1065
1066         my $monaddrhash = {};
1067
1068         my $systemd_managed = PVE::CephTools::systemd_managed();
1069
1070         foreach my $section (keys %$cfg) {
1071             next if $section eq 'global';
1072             my $d = $cfg->{$section};
1073             if ($section =~ m/^mon\./) {
1074                 $moncount++;
1075                 if ($d->{'mon addr'}) {
1076                     $monaddrhash->{$d->{'mon addr'}} = $section;
1077                 }
1078             }
1079         }
1080
1081         my $monid = $param->{id} // $param->{node};
1082
1083         my $monsection = "mon.$monid";
1084         my $pubnet = $cfg->{global}->{'public network'};
1085         my $ip = $find_mon_ip->($pubnet, $param->{node}, $param->{'mon-address'});
1086
1087         my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]:6789" : "$ip:6789";
1088         my $monname = $param->{node};
1089
1090         die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
1091         die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n"
1092             if $monaddrhash->{$monaddr};
1093
1094         my $worker = sub  {
1095             my $upid = shift;
1096
1097             my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1098
1099             if (! -f $pve_ckeyring_path) {
1100                 run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
1101                             "--gen-key -n client.admin");
1102             }
1103
1104             my $pve_mon_key_path = PVE::CephTools::get_config('pve_mon_key_path');
1105             if (! -f $pve_mon_key_path) {
1106                 run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
1107                 run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " .
1108                             "--cap mds 'allow' " .
1109                             "--cap osd 'allow *' " .
1110                             "--cap mgr 'allow *' " .
1111                             "--cap mon 'allow *'");
1112                 run_command("cp $pve_mon_key_path.tmp /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
1113                 run_command("chown ceph:ceph /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
1114                 run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
1115                 run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
1116             }
1117
1118             my $ccname = PVE::CephTools::get_config('ccname');
1119
1120             my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
1121             -d $mondir && die "monitor filesystem '$mondir' already exist\n";
1122
1123             my $monmap = "/tmp/monmap";
1124
1125             eval {
1126                 mkdir $mondir;
1127
1128                 run_command("chown ceph:ceph $mondir") if $systemd_managed;
1129
1130                 if ($moncount > 0) {
1131                     my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1132                     my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
1133                     PVE::Tools::file_set_contents($monmap, $mapdata);
1134                 } else {
1135                     run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
1136                 }
1137
1138                 run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
1139                 run_command("chown ceph:ceph -R $mondir") if $systemd_managed;
1140             };
1141             my $err = $@;
1142             unlink $monmap;
1143             if ($err) {
1144                 File::Path::remove_tree($mondir);
1145                 die $err;
1146             }
1147
1148             $cfg->{$monsection} = {
1149                 'host' => $monname,
1150                 'mon addr' => $monaddr,
1151             };
1152
1153             PVE::CephTools::write_ceph_config($cfg);
1154
1155             my $create_keys_pid = fork();
1156             if (!defined($create_keys_pid)) {
1157                 die "Could not spawn ceph-create-keys to create bootstrap keys\n";
1158             } elsif ($create_keys_pid == 0) {
1159                 exit PVE::Tools::run_command(['ceph-create-keys', '-i', $monid]);
1160             } else {
1161                 PVE::CephTools::ceph_service_cmd('start', $monsection);
1162
1163                 if ($systemd_managed) {
1164                     #to ensure we have the correct startup order.
1165                     eval { PVE::Tools::run_command(['/bin/systemctl', 'enable', "ceph-mon\@${monid}.service"]); };
1166                     warn "Enable ceph-mon\@${monid}.service manually"if $@;
1167                 }
1168                 waitpid($create_keys_pid, 0);
1169             }
1170
1171             # create manager
1172             if (!$param->{'exclude-manager'}) {
1173                 my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1174                 $create_mgr->($rados, $monid);
1175             }
1176         };
1177
1178         return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
1179     }});
1180
1181 __PACKAGE__->register_method ({
1182     name => 'destroymon',
1183     path => 'mon/{monid}',
1184     method => 'DELETE',
1185     description => "Destroy Ceph Monitor and Manager.",
1186     proxyto => 'node',
1187     protected => 1,
1188     permissions => {
1189         check => ['perm', '/', [ 'Sys.Modify' ]],
1190     },
1191     parameters => {
1192         additionalProperties => 0,
1193         properties => {
1194             node => get_standard_option('pve-node'),
1195             monid => {
1196                 description => 'Monitor ID',
1197                 type => 'string',
1198                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1199             },
1200             'exclude-manager' => {
1201                 type => 'boolean',
1202                 default => 0,
1203                 optional => 1,
1204                 description => "When set, removes only the monitor, not the manager"
1205             }
1206         },
1207     },
1208     returns => { type => 'string' },
1209     code => sub {
1210         my ($param) = @_;
1211
1212         my $rpcenv = PVE::RPCEnvironment::get();
1213
1214         my $authuser = $rpcenv->get_user();
1215
1216         PVE::CephTools::check_ceph_inited();
1217
1218         my $cfg = PVE::CephTools::parse_ceph_config();
1219
1220         my $monid = $param->{monid};
1221         my $monsection = "mon.$monid";
1222
1223         my $rados = PVE::RADOS->new();
1224         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
1225         my $monlist = $monstat->{monmap}->{mons};
1226
1227         die "no such monitor id '$monid'\n"
1228             if !defined($cfg->{$monsection});
1229
1230         my $ccname = PVE::CephTools::get_config('ccname');
1231
1232         my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
1233         -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n";
1234
1235         die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
1236
1237         my $worker = sub {
1238             my $upid = shift;
1239
1240             # reopen with longer timeout
1241             $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1242
1243             $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });
1244
1245             eval { PVE::CephTools::ceph_service_cmd('stop', $monsection); };
1246             warn $@ if $@;
1247
1248             delete $cfg->{$monsection};
1249             PVE::CephTools::write_ceph_config($cfg);
1250             File::Path::remove_tree($mondir);
1251
1252             # remove manager
1253             if (!$param->{'exclude-manager'}) {
1254                 eval { $destroy_mgr->($monid); };
1255                 warn $@ if $@;
1256             }
1257         };
1258
1259         return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
1260     }});
1261
1262 __PACKAGE__->register_method ({
1263     name => 'createmgr',
1264     path => 'mgr',
1265     method => 'POST',
1266     description => "Create Ceph Manager",
1267     proxyto => 'node',
1268     protected => 1,
1269     permissions => {
1270         check => ['perm', '/', [ 'Sys.Modify' ]],
1271     },
1272     parameters => {
1273         additionalProperties => 0,
1274         properties => {
1275             node => get_standard_option('pve-node'),
1276             id => {
1277                 type => 'string',
1278                 optional => 1,
1279                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1280                 description => "The ID for the manager, when omitted the same as the nodename",
1281             },
1282         },
1283     },
1284     returns => { type => 'string' },
1285     code => sub {
1286         my ($param) = @_;
1287
1288         PVE::CephTools::check_ceph_installed('ceph_mgr');
1289
1290         PVE::CephTools::check_ceph_inited();
1291
1292         my $rpcenv = PVE::RPCEnvironment::get();
1293
1294         my $authuser = $rpcenv->get_user();
1295
1296         my $mgrid = $param->{id} // $param->{node};
1297
1298         my $worker = sub  {
1299             my $upid = shift;
1300
1301             my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1302
1303             $create_mgr->($rados, $mgrid);
1304         };
1305
1306         return $rpcenv->fork_worker('cephcreatemgr', "mgr.$mgrid", $authuser, $worker);
1307     }});
1308
1309 __PACKAGE__->register_method ({
1310     name => 'destroymgr',
1311     path => 'mgr/{id}',
1312     method => 'DELETE',
1313     description => "Destroy Ceph Manager.",
1314     proxyto => 'node',
1315     protected => 1,
1316     permissions => {
1317         check => ['perm', '/', [ 'Sys.Modify' ]],
1318     },
1319     parameters => {
1320         additionalProperties => 0,
1321         properties => {
1322             node => get_standard_option('pve-node'),
1323             id => {
1324                 description => 'The ID of the manager',
1325                 type => 'string',
1326                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1327             },
1328         },
1329     },
1330     returns => { type => 'string' },
1331     code => sub {
1332         my ($param) = @_;
1333
1334         my $rpcenv = PVE::RPCEnvironment::get();
1335
1336         my $authuser = $rpcenv->get_user();
1337
1338         PVE::CephTools::check_ceph_inited();
1339
1340         my $mgrid = $param->{id};
1341
1342         my $worker = sub {
1343             my $upid = shift;
1344
1345             $destroy_mgr->($mgrid);
1346         };
1347
1348         return $rpcenv->fork_worker('cephdestroymgr', "mgr.$mgrid",  $authuser, $worker);
1349     }});
1350
1351 __PACKAGE__->register_method ({
1352     name => 'stop',
1353     path => 'stop',
1354     method => 'POST',
1355     description => "Stop ceph services.",
1356     proxyto => 'node',
1357     protected => 1,
1358     permissions => {
1359         check => ['perm', '/', [ 'Sys.Modify' ]],
1360     },
1361     parameters => {
1362         additionalProperties => 0,
1363         properties => {
1364             node => get_standard_option('pve-node'),
1365             service => {
1366                 description => 'Ceph service name.',
1367                 type => 'string',
1368                 optional => 1,
1369                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1370             },
1371         },
1372     },
1373     returns => { type => 'string' },
1374     code => sub {
1375         my ($param) = @_;
1376
1377         my $rpcenv = PVE::RPCEnvironment::get();
1378
1379         my $authuser = $rpcenv->get_user();
1380
1381         PVE::CephTools::check_ceph_inited();
1382
1383         my $cfg = PVE::CephTools::parse_ceph_config();
1384         scalar(keys %$cfg) || die "no configuration\n";
1385
1386         my $worker = sub {
1387             my $upid = shift;
1388
1389             my $cmd = ['stop'];
1390             if ($param->{service}) {
1391                 push @$cmd, $param->{service};
1392             }
1393
1394             PVE::CephTools::ceph_service_cmd(@$cmd);
1395         };
1396
1397         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
1398                                     $authuser, $worker);
1399     }});
1400
1401 __PACKAGE__->register_method ({
1402     name => 'start',
1403     path => 'start',
1404     method => 'POST',
1405     description => "Start ceph services.",
1406     proxyto => 'node',
1407     protected => 1,
1408     permissions => {
1409         check => ['perm', '/', [ 'Sys.Modify' ]],
1410     },
1411     parameters => {
1412         additionalProperties => 0,
1413         properties => {
1414             node => get_standard_option('pve-node'),
1415             service => {
1416                 description => 'Ceph service name.',
1417                 type => 'string',
1418                 optional => 1,
1419                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1420             },
1421         },
1422     },
1423     returns => { type => 'string' },
1424     code => sub {
1425         my ($param) = @_;
1426
1427         my $rpcenv = PVE::RPCEnvironment::get();
1428
1429         my $authuser = $rpcenv->get_user();
1430
1431         PVE::CephTools::check_ceph_inited();
1432
1433         my $cfg = PVE::CephTools::parse_ceph_config();
1434         scalar(keys %$cfg) || die "no configuration\n";
1435
1436         my $worker = sub {
1437             my $upid = shift;
1438
1439             my $cmd = ['start'];
1440             if ($param->{service}) {
1441                 push @$cmd, $param->{service};
1442             }
1443
1444             PVE::CephTools::ceph_service_cmd(@$cmd);
1445         };
1446
1447         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
1448                                     $authuser, $worker);
1449     }});
1450
1451 __PACKAGE__->register_method ({
1452     name => 'status',
1453     path => 'status',
1454     method => 'GET',
1455     description => "Get ceph status.",
1456     proxyto => 'node',
1457     protected => 1,
1458     permissions => {
1459         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1460     },
1461     parameters => {
1462         additionalProperties => 0,
1463         properties => {
1464             node => get_standard_option('pve-node'),
1465         },
1466     },
1467     returns => { type => 'object' },
1468     code => sub {
1469         my ($param) = @_;
1470
1471         PVE::CephTools::check_ceph_enabled();
1472
1473         my $rados = PVE::RADOS->new();
1474         my $status = $rados->mon_command({ prefix => 'status' });
1475         $status->{health} = $rados->mon_command({ prefix => 'health', detail => 'detail' });
1476         return $status;
1477     }});
1478
1479 __PACKAGE__->register_method ({
1480     name => 'lspools',
1481     path => 'pools',
1482     method => 'GET',
1483     description => "List all pools.",
1484     proxyto => 'node',
1485     protected => 1,
1486     permissions => {
1487         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1488     },
1489     parameters => {
1490         additionalProperties => 0,
1491         properties => {
1492             node => get_standard_option('pve-node'),
1493         },
1494     },
1495     returns => {
1496         type => 'array',
1497         items => {
1498             type => "object",
1499             properties => {
1500                 pool => { type => 'integer' },
1501                 pool_name => { type => 'string' },
1502                 size => { type => 'integer' },
1503             },
1504         },
1505         links => [ { rel => 'child', href => "{pool_name}" } ],
1506     },
1507     code => sub {
1508         my ($param) = @_;
1509
1510         PVE::CephTools::check_ceph_inited();
1511
1512         my $rados = PVE::RADOS->new();
1513
1514         my $stats = {};
1515         my $res = $rados->mon_command({ prefix => 'df' });
1516
1517         foreach my $d (@{$res->{pools}}) {
1518             next if !$d->{stats};
1519             next if !defined($d->{id});
1520             $stats->{$d->{id}} = $d->{stats};
1521         }
1522
1523         $res = $rados->mon_command({ prefix => 'osd dump' });
1524         my $rulestmp = $rados->mon_command({ prefix => 'osd crush rule dump'});
1525
1526         my $rules = {};
1527         for my $rule (@$rulestmp) {
1528             $rules->{$rule->{rule_id}} = $rule->{rule_name};
1529         }
1530
1531         my $data = [];
1532         foreach my $e (@{$res->{pools}}) {
1533             my $d = {};
1534             foreach my $attr (qw(pool pool_name size min_size pg_num crush_rule)) {
1535                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
1536             }
1537
1538             if (defined($d->{crush_rule}) && defined($rules->{$d->{crush_rule}})) {
1539                 $d->{crush_rule_name} = $rules->{$d->{crush_rule}};
1540             }
1541
1542             if (my $s = $stats->{$d->{pool}}) {
1543                 $d->{bytes_used} = $s->{bytes_used};
1544                 $d->{percent_used} = $s->{percent_used};
1545             }
1546             push @$data, $d;
1547         }
1548
1549
1550         return $data;
1551     }});
1552
1553 __PACKAGE__->register_method ({
1554     name => 'createpool',
1555     path => 'pools',
1556     method => 'POST',
1557     description => "Create POOL",
1558     proxyto => 'node',
1559     protected => 1,
1560     permissions => {
1561         check => ['perm', '/', [ 'Sys.Modify' ]],
1562     },
1563     parameters => {
1564         additionalProperties => 0,
1565         properties => {
1566             node => get_standard_option('pve-node'),
1567             name => {
1568                 description => "The name of the pool. It must be unique.",
1569                 type => 'string',
1570             },
1571             size => {
1572                 description => 'Number of replicas per object',
1573                 type => 'integer',
1574                 default => 3,
1575                 optional => 1,
1576                 minimum => 1,
1577                 maximum => 7,
1578             },
1579             min_size => {
1580                 description => 'Minimum number of replicas per object',
1581                 type => 'integer',
1582                 default => 2,
1583                 optional => 1,
1584                 minimum => 1,
1585                 maximum => 7,
1586             },
1587             pg_num => {
1588                 description => "Number of placement groups.",
1589                 type => 'integer',
1590                 default => 64,
1591                 optional => 1,
1592                 minimum => 8,
1593                 maximum => 32768,
1594             },
1595             crush_rule => {
1596                 description => "The rule to use for mapping object placement in the cluster.",
1597                 type => 'string',
1598                 optional => 1,
1599             },
1600             application => {
1601                 description => "The application of the pool, 'rbd' by default.",
1602                 type => 'string',
1603                 enum => ['rbd', 'cephfs', 'rgw'],
1604                 optional => 1,
1605             },
1606             add_storages => {
1607                 description => "Configure VM and CT storage using the new pool.",
1608                 type => 'boolean',
1609                 optional => 1,
1610             },
1611         },
1612     },
1613     returns => { type => 'string' },
1614     code => sub {
1615         my ($param) = @_;
1616
1617         PVE::Cluster::check_cfs_quorum();
1618         PVE::CephTools::check_ceph_inited();
1619
1620         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1621
1622         die "not fully configured - missing '$pve_ckeyring_path'\n"
1623             if ! -f $pve_ckeyring_path;
1624
1625         my $pool = $param->{name};
1626         my $rpcenv = PVE::RPCEnvironment::get();
1627         my $user = $rpcenv->get_user();
1628
1629         if ($param->{add_storages}) {
1630             $rpcenv->check($user, '/storage', ['Datastore.Allocate']);
1631             die "pool name contains characters which are illegal for storage naming\n"
1632                 if !PVE::JSONSchema::parse_storage_id($pool);
1633         }
1634
1635         my $pg_num = $param->{pg_num} || 64;
1636         my $size = $param->{size} || 3;
1637         my $min_size = $param->{min_size} || 2;
1638         my $application = $param->{application} // 'rbd';
1639
1640         my $worker = sub {
1641
1642             my $rados = PVE::RADOS->new();
1643             $rados->mon_command({
1644                 prefix => "osd pool create",
1645                 pool => $pool,
1646                 pg_num => int($pg_num),
1647                 format => 'plain',
1648             });
1649
1650             $rados->mon_command({
1651                 prefix => "osd pool set",
1652                 pool => $pool,
1653                 var => 'min_size',
1654                 val => $min_size,
1655                 format => 'plain',
1656             });
1657
1658             $rados->mon_command({
1659                 prefix => "osd pool set",
1660                 pool => $pool,
1661                 var => 'size',
1662                 val => $size,
1663                 format => 'plain',
1664             });
1665
1666             if (defined($param->{crush_rule})) {
1667                 $rados->mon_command({
1668                     prefix => "osd pool set",
1669                     pool => $pool,
1670                     var => 'crush_rule',
1671                     val => $param->{crush_rule},
1672                     format => 'plain',
1673                 });
1674             }
1675
1676             $rados->mon_command({
1677                     prefix => "osd pool application enable",
1678                     pool => $pool,
1679                     app => $application,
1680             });
1681
1682             if ($param->{add_storages}) {
1683                 my $err;
1684                 eval { $add_storage->($pool, "${pool}"); };
1685                 if ($@) {
1686                     warn "failed to add storage: $@";
1687                     $err = 1;
1688                 }
1689                 die "adding storage for pool '$pool' failed, check log and add manually!\n"
1690                     if $err;
1691             }
1692         };
1693
1694         return $rpcenv->fork_worker('cephcreatepool', $pool,  $user, $worker);
1695     }});
1696
1697 __PACKAGE__->register_method ({
1698     name => 'get_flags',
1699     path => 'flags',
1700     method => 'GET',
1701     description => "get all set ceph flags",
1702     proxyto => 'node',
1703     protected => 1,
1704     permissions => {
1705         check => ['perm', '/', [ 'Sys.Audit' ]],
1706     },
1707     parameters => {
1708         additionalProperties => 0,
1709         properties => {
1710             node => get_standard_option('pve-node'),
1711         },
1712     },
1713     returns => { type => 'string' },
1714     code => sub {
1715         my ($param) = @_;
1716
1717         PVE::CephTools::check_ceph_inited();
1718
1719         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1720
1721         die "not fully configured - missing '$pve_ckeyring_path'\n"
1722             if ! -f $pve_ckeyring_path;
1723
1724         my $rados = PVE::RADOS->new();
1725
1726         my $stat = $rados->mon_command({ prefix => 'osd dump' });
1727
1728         return $stat->{flags} // '';
1729     }});
1730
1731 __PACKAGE__->register_method ({
1732     name => 'set_flag',
1733     path => 'flags/{flag}',
1734     method => 'POST',
1735     description => "Set a ceph flag",
1736     proxyto => 'node',
1737     protected => 1,
1738     permissions => {
1739         check => ['perm', '/', [ 'Sys.Modify' ]],
1740     },
1741     parameters => {
1742         additionalProperties => 0,
1743         properties => {
1744             node => get_standard_option('pve-node'),
1745             flag => {
1746                 description => 'The ceph flag to set/unset',
1747                 type => 'string',
1748                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1749             },
1750         },
1751     },
1752     returns => { type => 'null' },
1753     code => sub {
1754         my ($param) = @_;
1755
1756         PVE::CephTools::check_ceph_inited();
1757
1758         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1759
1760         die "not fully configured - missing '$pve_ckeyring_path'\n"
1761             if ! -f $pve_ckeyring_path;
1762
1763         my $set = $param->{set} // !$param->{unset};
1764         my $rados = PVE::RADOS->new();
1765
1766         $rados->mon_command({
1767             prefix => "osd set",
1768             key => $param->{flag},
1769         });
1770
1771         return undef;
1772     }});
1773
1774 __PACKAGE__->register_method ({
1775     name => 'unset_flag',
1776     path => 'flags/{flag}',
1777     method => 'DELETE',
1778     description => "Unset a ceph flag",
1779     proxyto => 'node',
1780     protected => 1,
1781     permissions => {
1782         check => ['perm', '/', [ 'Sys.Modify' ]],
1783     },
1784     parameters => {
1785         additionalProperties => 0,
1786         properties => {
1787             node => get_standard_option('pve-node'),
1788             flag => {
1789                 description => 'The ceph flag to set/unset',
1790                 type => 'string',
1791                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1792             },
1793         },
1794     },
1795     returns => { type => 'null' },
1796     code => sub {
1797         my ($param) = @_;
1798
1799         PVE::CephTools::check_ceph_inited();
1800
1801         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1802
1803         die "not fully configured - missing '$pve_ckeyring_path'\n"
1804             if ! -f $pve_ckeyring_path;
1805
1806         my $set = $param->{set} // !$param->{unset};
1807         my $rados = PVE::RADOS->new();
1808
1809         $rados->mon_command({
1810             prefix => "osd unset",
1811             key => $param->{flag},
1812         });
1813
1814         return undef;
1815     }});
1816
1817 __PACKAGE__->register_method ({
1818     name => 'destroypool',
1819     path => 'pools/{name}',
1820     method => 'DELETE',
1821     description => "Destroy pool",
1822     proxyto => 'node',
1823     protected => 1,
1824     permissions => {
1825         check => ['perm', '/', [ 'Sys.Modify' ]],
1826     },
1827     parameters => {
1828         additionalProperties => 0,
1829         properties => {
1830             node => get_standard_option('pve-node'),
1831             name => {
1832                 description => "The name of the pool. It must be unique.",
1833                 type => 'string',
1834             },
1835             force => {
1836                 description => "If true, destroys pool even if in use",
1837                 type => 'boolean',
1838                 optional => 1,
1839                 default => 0,
1840             },
1841             remove_storages => {
1842                 description => "Remove all pveceph-managed storages configured for this pool",
1843                 type => 'boolean',
1844                 optional => 1,
1845                 default => 0,
1846             },
1847         },
1848     },
1849     returns => { type => 'string' },
1850     code => sub {
1851         my ($param) = @_;
1852
1853         PVE::CephTools::check_ceph_inited();
1854
1855         my $rpcenv = PVE::RPCEnvironment::get();
1856         my $user = $rpcenv->get_user();
1857         $rpcenv->check($user, '/storage', ['Datastore.Allocate'])
1858             if $param->{remove_storages};
1859
1860         my $pool = $param->{name};
1861
1862         my $worker = sub {
1863             my $storages = $get_storages->($pool);
1864
1865             # if not forced, destroy ceph pool only when no
1866             # vm disks are on it anymore
1867             if (!$param->{force}) {
1868                 my $storagecfg = PVE::Storage::config();
1869                 foreach my $storeid (keys %$storages) {
1870                     my $storage = $storages->{$storeid};
1871
1872                     # check if any vm disks are on the pool
1873                     print "checking storage '$storeid' for RBD images..\n";
1874                     my $res = PVE::Storage::vdisk_list($storagecfg, $storeid);
1875                     die "ceph pool '$pool' still in use by storage '$storeid'\n"
1876                         if @{$res->{$storeid}} != 0;
1877                 }
1878             }
1879
1880             my $rados = PVE::RADOS->new();
1881             # fixme: '--yes-i-really-really-mean-it'
1882             $rados->mon_command({
1883                 prefix => "osd pool delete",
1884                 pool => $pool,
1885                 pool2 => $pool,
1886                 sure => '--yes-i-really-really-mean-it',
1887                 format => 'plain',
1888             });
1889
1890             if ($param->{remove_storages}) {
1891                 my $err;
1892                 foreach my $storeid (keys %$storages) {
1893                     # skip external clusters, not managed by pveceph
1894                     next if $storages->{$storeid}->{monhost};
1895                     eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
1896                     if ($@) {
1897                         warn "failed to remove storage '$storeid': $@\n";
1898                         $err = 1;
1899                     }
1900                 }
1901                 die "failed to remove (some) storages - check log and remove manually!\n"
1902                     if $err;
1903             }
1904         };
1905         return $rpcenv->fork_worker('cephdestroypool', $pool,  $user, $worker);
1906     }});
1907
1908
1909 __PACKAGE__->register_method ({
1910     name => 'crush',
1911     path => 'crush',
1912     method => 'GET',
1913     description => "Get OSD crush map",
1914     proxyto => 'node',
1915     protected => 1,
1916     permissions => {
1917         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1918     },
1919     parameters => {
1920         additionalProperties => 0,
1921         properties => {
1922             node => get_standard_option('pve-node'),
1923         },
1924     },
1925     returns => { type => 'string' },
1926     code => sub {
1927         my ($param) = @_;
1928
1929         PVE::CephTools::check_ceph_inited();
1930
1931         # this produces JSON (difficult to read for the user)
1932         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
1933
1934         my $txt = '';
1935
1936         my $mapfile = "/var/tmp/ceph-crush.map.$$";
1937         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
1938
1939         my $rados = PVE::RADOS->new();
1940
1941         eval {
1942             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
1943             PVE::Tools::file_set_contents($mapfile, $bindata);
1944             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
1945             $txt = PVE::Tools::file_get_contents($mapdata);
1946         };
1947         my $err = $@;
1948
1949         unlink $mapfile;
1950         unlink $mapdata;
1951
1952         die $err if $err;
1953
1954         return $txt;
1955     }});
1956
1957 __PACKAGE__->register_method({
1958     name => 'log',
1959     path => 'log',
1960     method => 'GET',
1961     description => "Read ceph log",
1962     proxyto => 'node',
1963     permissions => {
1964         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
1965     },
1966     protected => 1,
1967     parameters => {
1968         additionalProperties => 0,
1969         properties => {
1970             node => get_standard_option('pve-node'),
1971             start => {
1972                 type => 'integer',
1973                 minimum => 0,
1974                 optional => 1,
1975             },
1976             limit => {
1977                 type => 'integer',
1978                 minimum => 0,
1979                 optional => 1,
1980             },
1981         },
1982     },
1983     returns => {
1984         type => 'array',
1985         items => {
1986             type => "object",
1987             properties => {
1988                 n => {
1989                   description=>  "Line number",
1990                   type=> 'integer',
1991                 },
1992                 t => {
1993                   description=>  "Line text",
1994                   type => 'string',
1995                 }
1996             }
1997         }
1998     },
1999     code => sub {
2000         my ($param) = @_;
2001
2002         my $rpcenv = PVE::RPCEnvironment::get();
2003         my $user = $rpcenv->get_user();
2004         my $node = $param->{node};
2005
2006         my $logfile = "/var/log/ceph/ceph.log";
2007         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
2008
2009         $rpcenv->set_result_attrib('total', $count);
2010
2011         return $lines;
2012     }});
2013
2014 __PACKAGE__->register_method ({
2015     name => 'rules',
2016     path => 'rules',
2017     method => 'GET',
2018     description => "List ceph rules.",
2019     proxyto => 'node',
2020     protected => 1,
2021     permissions => {
2022         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
2023     },
2024     parameters => {
2025         additionalProperties => 0,
2026         properties => {
2027             node => get_standard_option('pve-node'),
2028         },
2029     },
2030     returns => {
2031         type => 'array',
2032         items => {
2033             type => "object",
2034             properties => {},
2035         },
2036         links => [ { rel => 'child', href => "{name}" } ],
2037     },
2038     code => sub {
2039         my ($param) = @_;
2040
2041         PVE::CephTools::check_ceph_inited();
2042
2043         my $rados = PVE::RADOS->new();
2044
2045         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
2046
2047         my $res = [];
2048
2049         foreach my $rule (@$rules) {
2050             push @$res, { name => $rule };
2051         }
2052
2053         return $res;
2054     }});