PVE/API2/Ceph.pm

   1 package PVE::API2::CephOSD;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd qw(abs_path);
   7 use IO::File;
   8
   9 use PVE::Ceph::Tools;
  10 use PVE::Ceph::Services;
  11 use PVE::CephConfig;
  12 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  13 use PVE::Diskmanage;
  14 use PVE::Exception qw(raise_param_exc);
  15 use PVE::JSONSchema qw(get_standard_option);
  16 use PVE::RADOS;
  17 use PVE::RESTHandler;
  18 use PVE::RPCEnvironment;
  19 use PVE::Tools qw(run_command file_set_contents);
  20
  21 use base qw(PVE::RESTHandler);
  22
  23 my $get_osd_status = sub {
  24     my ($rados, $osdid) = @_;
  25
  26     my $stat = $rados->mon_command({ prefix => 'osd dump' });
  27
  28     my $osdlist = $stat->{osds} || [];
  29
  30     my $flags = $stat->{flags} || undef;
  31
  32     my $osdstat;
  33     foreach my $d (@$osdlist) {
  34         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  35     }
  36     if (defined($osdid)) {
  37         die "no such OSD '$osdid'\n" if !$osdstat->{$osdid};
  38         return $osdstat->{$osdid};
  39     }
  40
  41     return wantarray? ($osdstat, $flags):$osdstat;
  42 };
  43
  44 my $get_osd_usage = sub {
  45     my ($rados) = @_;
  46
  47     my $osdlist = $rados->mon_command({ prefix => 'pg dump',
  48                                         dumpcontents => [ 'osds' ]}) || [];
  49
  50     my $osdstat;
  51     foreach my $d (@$osdlist) {
  52         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  53     }
  54
  55     return $osdstat;
  56 };
  57
  58 __PACKAGE__->register_method ({
  59     name => 'index',
  60     path => '',
  61     method => 'GET',
  62     description => "Get Ceph osd list/tree.",
  63     proxyto => 'node',
  64     protected => 1,
  65     permissions => {
  66         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  67     },
  68     parameters => {
  69         additionalProperties => 0,
  70         properties => {
  71             node => get_standard_option('pve-node'),
  72         },
  73     },
  74     # fixme: return a list instead of extjs tree format ?
  75     returns => {
  76         type => "object",
  77     },
  78     code => sub {
  79         my ($param) = @_;
  80
  81         PVE::Ceph::Tools::check_ceph_inited();
  82
  83         my $rados = PVE::RADOS->new();
  84         my $res = $rados->mon_command({ prefix => 'osd tree' });
  85
  86         die "no tree nodes found\n" if !($res && $res->{nodes});
  87
  88         my ($osdhash, $flags) = &$get_osd_status($rados);
  89
  90         my $usagehash = &$get_osd_usage($rados);
  91
  92         my $osdmetadata_tmp = $rados->mon_command({ prefix => 'osd metadata' });
  93
  94         my $osdmetadata = {};
  95         foreach my $osd (@$osdmetadata_tmp) {
  96             $osdmetadata->{$osd->{id}} = $osd;
  97         }
  98
  99         my $nodes = {};
 100         my $newnodes = {};
 101         foreach my $e (@{$res->{nodes}}) {
 102             $nodes->{$e->{id}} = $e;
 103
 104             my $new = {
 105                 id => $e->{id},
 106                 name => $e->{name},
 107                 type => $e->{type}
 108             };
 109
 110             foreach my $opt (qw(status crush_weight reweight device_class)) {
 111                 $new->{$opt} = $e->{$opt} if defined($e->{$opt});
 112             }
 113
 114             if (my $stat = $osdhash->{$e->{id}}) {
 115                 $new->{in} = $stat->{in} if defined($stat->{in});
 116             }
 117
 118             if (my $stat = $usagehash->{$e->{id}}) {
 119                 $new->{total_space} = ($stat->{kb} || 1) * 1024;
 120                 $new->{bytes_used} = ($stat->{kb_used} || 0) * 1024;
 121                 $new->{percent_used} = ($new->{bytes_used}*100)/$new->{total_space};
 122                 if (my $d = $stat->{perf_stat}) {
 123                     $new->{commit_latency_ms} = $d->{commit_latency_ms};
 124                     $new->{apply_latency_ms} = $d->{apply_latency_ms};
 125                 }
 126             }
 127
 128             my $osdmd = $osdmetadata->{$e->{id}};
 129             if ($e->{type} eq 'osd' && $osdmd) {
 130                 if ($osdmd->{bluefs}) {
 131                     $new->{osdtype} = 'bluestore';
 132                     $new->{blfsdev} = $osdmd->{bluestore_bdev_dev_node};
 133                     $new->{dbdev} = $osdmd->{bluefs_db_dev_node};
 134                     $new->{waldev} = $osdmd->{bluefs_wal_dev_node};
 135                 } else {
 136                     $new->{osdtype} = 'filestore';
 137                 }
 138             }
 139
 140             $newnodes->{$e->{id}} = $new;
 141         }
 142
 143         foreach my $e (@{$res->{nodes}}) {
 144             my $new = $newnodes->{$e->{id}};
 145             if ($e->{children} && scalar(@{$e->{children}})) {
 146                 $new->{children} = [];
 147                 $new->{leaf} = 0;
 148                 foreach my $cid (@{$e->{children}}) {
 149                     $nodes->{$cid}->{parent} = $e->{id};
 150                     if ($nodes->{$cid}->{type} eq 'osd' &&
 151                         $e->{type} eq 'host') {
 152                         $newnodes->{$cid}->{host} = $e->{name};
 153                     }
 154                     push @{$new->{children}}, $newnodes->{$cid};
 155                 }
 156             } else {
 157                 $new->{leaf} = ($e->{id} >= 0) ? 1 : 0;
 158             }
 159         }
 160
 161         my $roots = [];
 162         foreach my $e (@{$res->{nodes}}) {
 163             if (!$nodes->{$e->{id}}->{parent}) {
 164                 push @$roots, $newnodes->{$e->{id}};
 165             }
 166         }
 167
 168         die "no root node\n" if !@$roots;
 169
 170         my $data = { root => { leaf =>  0, children => $roots } };
 171
 172         # we want this for the noout flag
 173         $data->{flags} = $flags if $flags;
 174
 175         return $data;
 176     }});
 177
 178 __PACKAGE__->register_method ({
 179     name => 'createosd',
 180     path => '',
 181     method => 'POST',
 182     description => "Create OSD",
 183     proxyto => 'node',
 184     protected => 1,
 185     parameters => {
 186         additionalProperties => 0,
 187         properties => {
 188             node => get_standard_option('pve-node'),
 189             dev => {
 190                 description => "Block device name.",
 191                 type => 'string',
 192             },
 193             journal_dev => {
 194                 description => "Block device name for journal (filestore) or block.db (bluestore).",
 195                 optional => 1,
 196                 type => 'string',
 197             },
 198             wal_dev => {
 199                 description => "Block device name for block.wal (bluestore only).",
 200                 optional => 1,
 201                 type => 'string',
 202             },
 203             fstype => {
 204                 description => "File system type (filestore only).",
 205                 type => 'string',
 206                 enum => ['xfs', 'ext4'],
 207                 default => 'xfs',
 208                 optional => 1,
 209             },
 210             bluestore => {
 211                 description => "Use bluestore instead of filestore. This is the default.",
 212                 type => 'boolean',
 213                 default => 1,
 214                 optional => 1,
 215             },
 216         },
 217     },
 218     returns => { type => 'string' },
 219     code => sub {
 220         my ($param) = @_;
 221
 222         my $rpcenv = PVE::RPCEnvironment::get();
 223
 224         my $authuser = $rpcenv->get_user();
 225
 226         raise_param_exc({ 'bluestore' => "conflicts with parameter 'fstype'" })
 227             if (defined($param->{fstype}) && defined($param->{bluestore}) && $param->{bluestore});
 228
 229         PVE::Ceph::Tools::check_ceph_inited();
 230
 231         PVE::Ceph::Tools::setup_pve_symlinks();
 232
 233         PVE::Ceph::Tools::check_ceph_installed('ceph_osd');
 234
 235         my $bluestore = $param->{bluestore} // 1;
 236
 237         my $journal_dev;
 238         my $wal_dev;
 239
 240         if ($param->{journal_dev} && ($param->{journal_dev} ne $param->{dev})) {
 241             $journal_dev = PVE::Diskmanage::verify_blockdev_path($param->{journal_dev});
 242         }
 243
 244         if ($param->{wal_dev} &&
 245             ($param->{wal_dev} ne $param->{dev}) &&
 246             (!$param->{journal_dev} || $param->{wal_dev} ne $param->{journal_dev})) {
 247             raise_param_exc({ 'wal_dev' => "can only be set with paramater 'bluestore'"})
 248                 if !$bluestore;
 249             $wal_dev = PVE::Diskmanage::verify_blockdev_path($param->{wal_dev});
 250         }
 251
 252         $param->{dev} = PVE::Diskmanage::verify_blockdev_path($param->{dev});
 253
 254         my $devname = $param->{dev};
 255         $devname =~ s|/dev/||;
 256
 257         my $disklist = PVE::Diskmanage::get_disks($devname, 1);
 258
 259         my $diskinfo = $disklist->{$devname};
 260         die "unable to get device info for '$devname'\n"
 261             if !$diskinfo;
 262
 263         die "device '$param->{dev}' is in use\n"
 264             if $diskinfo->{used};
 265
 266         my $devpath = $diskinfo->{devpath};
 267         my $rados = PVE::RADOS->new();
 268         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 269         die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid};
 270
 271         my $fsid = $monstat->{monmap}->{fsid};
 272         $fsid = $1 if $fsid =~ m/^([0-9a-f\-]+)$/;
 273
 274         my $ceph_bootstrap_osd_keyring = PVE::Ceph::Tools::get_config('ceph_bootstrap_osd_keyring');
 275
 276         if (! -f $ceph_bootstrap_osd_keyring) {
 277             my $bindata = $rados->mon_command({ prefix => 'auth get', entity => 'client.bootstrap-osd', format => 'plain' });
 278             file_set_contents($ceph_bootstrap_osd_keyring, $bindata);
 279         };
 280
 281         my $worker = sub {
 282             my $upid = shift;
 283
 284             my $fstype = $param->{fstype} || 'xfs';
 285
 286
 287             my $ccname = PVE::Ceph::Tools::get_config('ccname');
 288
 289             my $cmd = ['ceph-disk', 'prepare', '--zap-disk',
 290                        '--cluster', $ccname, '--cluster-uuid', $fsid ];
 291
 292             if ($bluestore) {
 293                 print "create OSD on $devpath (bluestore)\n";
 294                 push @$cmd, '--bluestore';
 295
 296                 if ($journal_dev) {
 297                     print "using device '$journal_dev' for block.db\n";
 298                     push @$cmd, '--block.db', $journal_dev;
 299                 }
 300
 301                 if ($wal_dev) {
 302                     print "using device '$wal_dev' for block.wal\n";
 303                     push @$cmd, '--block.wal', $wal_dev;
 304                 }
 305
 306                 push @$cmd, $devpath;
 307             } else {
 308                 print "create OSD on $devpath ($fstype)\n";
 309                 push @$cmd, '--filestore', '--fs-type', $fstype;
 310                 if ($journal_dev) {
 311                     print "using device '$journal_dev' for journal\n";
 312                     push @$cmd, '--journal-dev', $devpath, $journal_dev;
 313                 } else {
 314                     push @$cmd, $devpath;
 315                 }
 316             }
 317
 318             PVE::Ceph::Tools::wipe_disks($devpath);
 319
 320             run_command($cmd);
 321         };
 322
 323         return $rpcenv->fork_worker('cephcreateosd', $devname,  $authuser, $worker);
 324     }});
 325
 326 __PACKAGE__->register_method ({
 327     name => 'destroyosd',
 328     path => '{osdid}',
 329     method => 'DELETE',
 330     description => "Destroy OSD",
 331     proxyto => 'node',
 332     protected => 1,
 333     parameters => {
 334         additionalProperties => 0,
 335         properties => {
 336             node => get_standard_option('pve-node'),
 337             osdid => {
 338                 description => 'OSD ID',
 339                 type => 'integer',
 340             },
 341             cleanup => {
 342                 description => "If set, we remove partition table entries.",
 343                 type => 'boolean',
 344                 optional => 1,
 345                 default => 0,
 346             },
 347         },
 348     },
 349     returns => { type => 'string' },
 350     code => sub {
 351         my ($param) = @_;
 352
 353         my $rpcenv = PVE::RPCEnvironment::get();
 354
 355         my $authuser = $rpcenv->get_user();
 356
 357         PVE::Ceph::Tools::check_ceph_inited();
 358
 359         my $osdid = $param->{osdid};
 360
 361         my $rados = PVE::RADOS->new();
 362         my $osdstat = &$get_osd_status($rados, $osdid);
 363
 364         die "osd is in use (in == 1)\n" if $osdstat->{in};
 365         #&$run_ceph_cmd(['osd', 'out', $osdid]);
 366
 367         die "osd is still runnung (up == 1)\n" if $osdstat->{up};
 368
 369         my $osdsection = "osd.$osdid";
 370
 371         my $worker = sub {
 372             my $upid = shift;
 373
 374             # reopen with longer timeout
 375             $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
 376
 377             print "destroy OSD $osdsection\n";
 378
 379             eval {
 380                 PVE::Ceph::Services::ceph_service_cmd('stop', $osdsection);
 381                 PVE::Ceph::Services::ceph_service_cmd('disable', $osdsection);
 382             };
 383             warn $@ if $@;
 384
 385             print "Remove $osdsection from the CRUSH map\n";
 386             $rados->mon_command({ prefix => "osd crush remove", name => $osdsection, format => 'plain' });
 387
 388             print "Remove the $osdsection authentication key.\n";
 389             $rados->mon_command({ prefix => "auth del", entity => $osdsection, format => 'plain' });
 390
 391             print "Remove OSD $osdsection\n";
 392             $rados->mon_command({ prefix => "osd rm", ids => [ $osdsection ], format => 'plain' });
 393
 394             # try to unmount from standard mount point
 395             my $mountpoint = "/var/lib/ceph/osd/ceph-$osdid";
 396
 397             my $disks_to_wipe = {};
 398             my $remove_partition = sub {
 399                 my ($part) = @_;
 400
 401                 return if !$part || (! -b $part );
 402                 my $partnum = PVE::Diskmanage::get_partnum($part);
 403                 my $devpath = PVE::Diskmanage::get_blockdev($part);
 404
 405                 print "remove partition $part (disk '${devpath}', partnum $partnum)\n";
 406                 eval { run_command(['/sbin/sgdisk', '-d', $partnum, "${devpath}"]); };
 407                 warn $@ if $@;
 408
 409                 $disks_to_wipe->{$devpath} = 1;
 410             };
 411
 412             my $partitions_to_remove = [];
 413
 414             if ($param->{cleanup}) {
 415                 if (my $fd = IO::File->new("/proc/mounts", "r")) {
 416                     while (defined(my $line = <$fd>)) {
 417                         my ($dev, $path, $fstype) = split(/\s+/, $line);
 418                         next if !($dev && $path && $fstype);
 419                         next if $dev !~ m|^/dev/|;
 420                         if ($path eq $mountpoint) {
 421                             my $data_part = abs_path($dev);
 422                             push @$partitions_to_remove, $data_part;
 423                             last;
 424                         }
 425                     }
 426                     close($fd);
 427                 }
 428
 429                 foreach my $path (qw(journal block block.db block.wal)) {
 430                     my $part = abs_path("$mountpoint/$path");
 431                     if ($part) {
 432                         push @$partitions_to_remove, $part;
 433                     }
 434                 }
 435             }
 436
 437             print "Unmount OSD $osdsection from  $mountpoint\n";
 438             eval { run_command(['/bin/umount', $mountpoint]); };
 439             if (my $err = $@) {
 440                 warn $err;
 441             } elsif ($param->{cleanup}) {
 442                 #be aware of the ceph udev rules which can remount.
 443                 foreach my $part (@$partitions_to_remove) {
 444                     $remove_partition->($part);
 445                 }
 446
 447                 PVE::Ceph::Tools::wipe_disks(keys %$disks_to_wipe);
 448             }
 449         };
 450
 451         return $rpcenv->fork_worker('cephdestroyosd', $osdsection,  $authuser, $worker);
 452     }});
 453
 454 __PACKAGE__->register_method ({
 455     name => 'in',
 456     path => '{osdid}/in',
 457     method => 'POST',
 458     description => "ceph osd in",
 459     proxyto => 'node',
 460     protected => 1,
 461     permissions => {
 462         check => ['perm', '/', [ 'Sys.Modify' ]],
 463     },
 464     parameters => {
 465         additionalProperties => 0,
 466         properties => {
 467             node => get_standard_option('pve-node'),
 468             osdid => {
 469                 description => 'OSD ID',
 470                 type => 'integer',
 471             },
 472         },
 473     },
 474     returns => { type => "null" },
 475     code => sub {
 476         my ($param) = @_;
 477
 478         PVE::Ceph::Tools::check_ceph_inited();
 479
 480         my $osdid = $param->{osdid};
 481
 482         my $rados = PVE::RADOS->new();
 483
 484         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 485
 486         my $osdsection = "osd.$osdid";
 487
 488         $rados->mon_command({ prefix => "osd in", ids => [ $osdsection ], format => 'plain' });
 489
 490         return undef;
 491     }});
 492
 493 __PACKAGE__->register_method ({
 494     name => 'out',
 495     path => '{osdid}/out',
 496     method => 'POST',
 497     description => "ceph osd out",
 498     proxyto => 'node',
 499     protected => 1,
 500     permissions => {
 501         check => ['perm', '/', [ 'Sys.Modify' ]],
 502     },
 503     parameters => {
 504         additionalProperties => 0,
 505         properties => {
 506             node => get_standard_option('pve-node'),
 507             osdid => {
 508                 description => 'OSD ID',
 509                 type => 'integer',
 510             },
 511         },
 512     },
 513     returns => { type => "null" },
 514     code => sub {
 515         my ($param) = @_;
 516
 517         PVE::Ceph::Tools::check_ceph_inited();
 518
 519         my $osdid = $param->{osdid};
 520
 521         my $rados = PVE::RADOS->new();
 522
 523         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 524
 525         my $osdsection = "osd.$osdid";
 526
 527         $rados->mon_command({ prefix => "osd out", ids => [ $osdsection ], format => 'plain' });
 528
 529         return undef;
 530     }});
 531
 532 package PVE::API2::Ceph;
 533
 534 use strict;
 535 use warnings;
 536
 537 use File::Path;
 538 use Net::IP;
 539 use UUID;
 540
 541 use PVE::Ceph::Tools;
 542 use PVE::Ceph::Services;
 543 use PVE::Cluster qw(cfs_read_file cfs_write_file);
 544 use PVE::JSONSchema qw(get_standard_option);
 545 use PVE::Network;
 546 use PVE::RADOS;
 547 use PVE::RESTHandler;
 548 use PVE::RPCEnvironment;
 549 use PVE::Storage;
 550 use PVE::Tools qw(run_command file_get_contents file_set_contents);
 551
 552 use PVE::API2::Ceph::FS;
 553 use PVE::API2::Ceph::MDS;
 554 use PVE::API2::Storage::Config;
 555
 556 use base qw(PVE::RESTHandler);
 557
 558 my $pve_osd_default_journal_size = 1024*5;
 559
 560 __PACKAGE__->register_method ({
 561     subclass => "PVE::API2::CephOSD",
 562     path => 'osd',
 563 });
 564
 565 __PACKAGE__->register_method ({
 566     subclass => "PVE::API2::Ceph::MDS",
 567     path => 'mds',
 568 });
 569
 570 __PACKAGE__->register_method ({
 571     subclass => "PVE::API2::Ceph::FS",
 572     path => 'fs',
 573 });
 574
 575 __PACKAGE__->register_method ({
 576     name => 'index',
 577     path => '',
 578     method => 'GET',
 579     description => "Directory index.",
 580     permissions => { user => 'all' },
 581     permissions => {
 582         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 583     },
 584     parameters => {
 585         additionalProperties => 0,
 586         properties => {
 587             node => get_standard_option('pve-node'),
 588         },
 589     },
 590     returns => {
 591         type => 'array',
 592         items => {
 593             type => "object",
 594             properties => {},
 595         },
 596         links => [ { rel => 'child', href => "{name}" } ],
 597     },
 598     code => sub {
 599         my ($param) = @_;
 600
 601         my $result = [
 602             { name => 'init' },
 603             { name => 'mon' },
 604             { name => 'osd' },
 605             { name => 'pools' },
 606             { name => 'fs' },
 607             { name => 'mds' },
 608             { name => 'stop' },
 609             { name => 'start' },
 610             { name => 'status' },
 611             { name => 'crush' },
 612             { name => 'config' },
 613             { name => 'log' },
 614             { name => 'disks' },
 615             { name => 'flags' },
 616             { name => 'rules' },
 617         ];
 618
 619         return $result;
 620     }});
 621
 622 __PACKAGE__->register_method ({
 623     name => 'disks',
 624     path => 'disks',
 625     method => 'GET',
 626     description => "List local disks.",
 627     proxyto => 'node',
 628     protected => 1,
 629     permissions => {
 630         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 631     },
 632     parameters => {
 633         additionalProperties => 0,
 634         properties => {
 635             node => get_standard_option('pve-node'),
 636             type => {
 637                 description => "Only list specific types of disks.",
 638                 type => 'string',
 639                 enum => ['unused', 'journal_disks'],
 640                 optional => 1,
 641             },
 642         },
 643     },
 644     returns => {
 645         type => 'array',
 646         items => {
 647             type => "object",
 648             properties => {
 649                 dev => { type => 'string' },
 650                 used => { type => 'string', optional => 1 },
 651                 gpt => { type => 'boolean' },
 652                 size => { type => 'integer' },
 653                 osdid => { type => 'integer' },
 654                 vendor =>  { type => 'string', optional => 1 },
 655                 model =>  { type => 'string', optional => 1 },
 656                 serial =>  { type => 'string', optional => 1 },
 657             },
 658         },
 659         # links => [ { rel => 'child', href => "{}" } ],
 660     },
 661     code => sub {
 662         my ($param) = @_;
 663
 664         PVE::Ceph::Tools::check_ceph_inited();
 665
 666         my $disks = PVE::Diskmanage::get_disks(undef, 1);
 667
 668         my $res = [];
 669         foreach my $dev (keys %$disks) {
 670             my $d = $disks->{$dev};
 671             if ($param->{type}) {
 672                 if ($param->{type} eq 'journal_disks') {
 673                     next if $d->{osdid} >= 0;
 674                     next if !$d->{gpt};
 675                 } elsif ($param->{type} eq 'unused') {
 676                     next if $d->{used};
 677                 } else {
 678                     die "internal error"; # should not happen
 679                 }
 680             }
 681
 682             $d->{dev} = "/dev/$dev";
 683             push @$res, $d;
 684         }
 685
 686         return $res;
 687     }});
 688
 689 __PACKAGE__->register_method ({
 690     name => 'config',
 691     path => 'config',
 692     method => 'GET',
 693     permissions => {
 694         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 695     },
 696     description => "Get Ceph configuration.",
 697     parameters => {
 698         additionalProperties => 0,
 699         properties => {
 700             node => get_standard_option('pve-node'),
 701         },
 702     },
 703     returns => { type => 'string' },
 704     code => sub {
 705         my ($param) = @_;
 706
 707         PVE::Ceph::Tools::check_ceph_inited();
 708
 709         my $path = PVE::Ceph::Tools::get_config('pve_ceph_cfgpath');
 710         return file_get_contents($path);
 711
 712     }});
 713
 714 my $add_storage = sub {
 715     my ($pool, $storeid) = @_;
 716
 717     my $storage_params = {
 718         type => 'rbd',
 719         pool => $pool,
 720         storage => $storeid,
 721         krbd => 0,
 722         content => 'rootdir,images',
 723     };
 724
 725     PVE::API2::Storage::Config->create($storage_params);
 726 };
 727
 728 my $get_storages = sub {
 729     my ($pool) = @_;
 730
 731     my $cfg = PVE::Storage::config();
 732
 733     my $storages = $cfg->{ids};
 734     my $res = {};
 735     foreach my $storeid (keys %$storages) {
 736         my $curr = $storages->{$storeid};
 737         $res->{$storeid} = $storages->{$storeid}
 738             if $curr->{type} eq 'rbd' && $pool eq $curr->{pool};
 739     }
 740
 741     return $res;
 742 };
 743
 744 __PACKAGE__->register_method ({
 745     name => 'listmon',
 746     path => 'mon',
 747     method => 'GET',
 748     description => "Get Ceph monitor list.",
 749     proxyto => 'node',
 750     protected => 1,
 751     permissions => {
 752         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 753     },
 754     parameters => {
 755         additionalProperties => 0,
 756         properties => {
 757             node => get_standard_option('pve-node'),
 758         },
 759     },
 760     returns => {
 761         type => 'array',
 762         items => {
 763             type => "object",
 764             properties => {
 765                 name => { type => 'string' },
 766                 addr => { type => 'string' },
 767             },
 768         },
 769         links => [ { rel => 'child', href => "{name}" } ],
 770     },
 771     code => sub {
 772         my ($param) = @_;
 773
 774         PVE::Ceph::Tools::check_ceph_inited();
 775
 776         my $res = [];
 777
 778         my $cfg = cfs_read_file('ceph.conf');
 779
 780         my $monhash = {};
 781         foreach my $section (keys %$cfg) {
 782             my $d = $cfg->{$section};
 783             if ($section =~ m/^mon\.(\S+)$/) {
 784                 my $monid = $1;
 785                 if ($d->{'mon addr'} && $d->{'host'}) {
 786                     $monhash->{$monid} = {
 787                         addr => $d->{'mon addr'},
 788                         host => $d->{'host'},
 789                         name => $monid,
 790                     }
 791                 }
 792             }
 793         }
 794
 795         eval {
 796             my $rados = PVE::RADOS->new();
 797             my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 798             my $mons = $monstat->{monmap}->{mons};
 799             foreach my $d (@$mons) {
 800                 next if !defined($d->{name});
 801                 $monhash->{$d->{name}}->{rank} = $d->{rank};
 802                 $monhash->{$d->{name}}->{addr} = $d->{addr};
 803                 if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
 804                     $monhash->{$d->{name}}->{quorum} = 1;
 805                 }
 806             }
 807         };
 808         warn $@ if $@;
 809
 810         return PVE::RESTHandler::hash_to_array($monhash, 'name');
 811     }});
 812
 813 __PACKAGE__->register_method ({
 814     name => 'init',
 815     path => 'init',
 816     method => 'POST',
 817     description => "Create initial ceph default configuration and setup symlinks.",
 818     proxyto => 'node',
 819     protected => 1,
 820     permissions => {
 821         check => ['perm', '/', [ 'Sys.Modify' ]],
 822     },
 823     parameters => {
 824         additionalProperties => 0,
 825         properties => {
 826             node => get_standard_option('pve-node'),
 827             network => {
 828                 description => "Use specific network for all ceph related traffic",
 829                 type => 'string', format => 'CIDR',
 830                 optional => 1,
 831                 maxLength => 128,
 832             },
 833             'cluster-network' => {
 834                 description => "Declare a separate cluster network, OSDs will route" .
 835                     "heartbeat, object replication and recovery traffic over it",
 836                 type => 'string', format => 'CIDR',
 837                 requires => 'network',
 838                 optional => 1,
 839                 maxLength => 128,
 840             },
 841             size => {
 842                 description => 'Targeted number of replicas per object',
 843                 type => 'integer',
 844                 default => 3,
 845                 optional => 1,
 846                 minimum => 1,
 847                 maximum => 7,
 848             },
 849             min_size => {
 850                 description => 'Minimum number of available replicas per object to allow I/O',
 851                 type => 'integer',
 852                 default => 2,
 853                 optional => 1,
 854                 minimum => 1,
 855                 maximum => 7,
 856             },
 857             pg_bits => {
 858                 description => "Placement group bits, used to specify the " .
 859                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 860                     "default pg num' does not work for default pools.",
 861                 type => 'integer',
 862                 default => 6,
 863                 optional => 1,
 864                 minimum => 6,
 865                 maximum => 14,
 866             },
 867             disable_cephx => {
 868                 description => "Disable cephx authentification.\n\n" .
 869                     "WARNING: cephx is a security feature protecting against " .
 870                     "man-in-the-middle attacks. Only consider disabling cephx ".
 871                     "if your network is private!",
 872                 type => 'boolean',
 873                 optional => 1,
 874                 default => 0,
 875             },
 876         },
 877     },
 878     returns => { type => 'null' },
 879     code => sub {
 880         my ($param) = @_;
 881
 882         my $version = PVE::Ceph::Tools::get_local_version(1);
 883
 884         if (!$version || $version < 12) {
 885             die "Ceph Luminous required - please run 'pveceph install'\n";
 886         } else {
 887             PVE::Ceph::Tools::check_ceph_installed('ceph_bin');
 888         }
 889
 890         # simply load old config if it already exists
 891         my $cfg = cfs_read_file('ceph.conf');
 892
 893         if (!$cfg->{global}) {
 894
 895             my $fsid;
 896             my $uuid;
 897
 898             UUID::generate($uuid);
 899             UUID::unparse($uuid, $fsid);
 900
 901             my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 902
 903             $cfg->{global} = {
 904                 'fsid' => $fsid,
 905                 'auth cluster required' => $auth,
 906                 'auth service required' => $auth,
 907                 'auth client required' => $auth,
 908                 'osd journal size' => $pve_osd_default_journal_size,
 909                 'osd pool default size' => $param->{size} // 3,
 910                 'osd pool default min size' => $param->{min_size} // 2,
 911                 'mon allow pool delete' => 'true',
 912             };
 913
 914             # this does not work for default pools
 915             #'osd pool default pg num' => $pg_num,
 916             #'osd pool default pgp num' => $pg_num,
 917         }
 918
 919         $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 920         $cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring';
 921
 922         if ($param->{pg_bits}) {
 923             $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 924             $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 925         }
 926
 927         if ($param->{network}) {
 928             $cfg->{global}->{'public network'} = $param->{network};
 929             $cfg->{global}->{'cluster network'} = $param->{network};
 930         }
 931
 932         if ($param->{'cluster-network'}) {
 933             $cfg->{global}->{'cluster network'} = $param->{'cluster-network'};
 934         }
 935
 936         cfs_write_file('ceph.conf', $cfg);
 937
 938         PVE::Ceph::Tools::setup_pve_symlinks();
 939
 940         return undef;
 941     }});
 942
 943 my $find_mon_ip = sub {
 944     my ($pubnet, $node, $overwrite_ip) = @_;
 945
 946     if (!$pubnet) {
 947         return $overwrite_ip // PVE::Cluster::remote_node_ip($node);
 948     }
 949
 950     my $allowed_ips = PVE::Network::get_local_ip_from_cidr($pubnet);
 951     die "No IP configured and up from ceph public network '$pubnet'\n"
 952         if scalar(@$allowed_ips) < 1;
 953
 954     if (!$overwrite_ip) {
 955         if (scalar(@$allowed_ips) == 1) {
 956             return $allowed_ips->[0];
 957         }
 958         die "Multiple IPs for ceph public network '$pubnet' detected on $node:\n".
 959             join("\n", @$allowed_ips) ."\nuse 'mon-address' to specify one of them.\n";
 960     } else {
 961         if (grep { $_ eq $overwrite_ip } @$allowed_ips) {
 962             return $overwrite_ip;
 963         }
 964         die "Monitor IP '$overwrite_ip' not in ceph public network '$pubnet'\n"
 965             if !PVE::Network::is_ip_in_cidr($overwrite_ip, $pubnet);
 966
 967         die "Specified monitor IP '$overwrite_ip' not configured or up on $node!\n";
 968     }
 969 };
 970
 971 my $create_mgr = sub {
 972     my ($rados, $id) = @_;
 973
 974     my $clustername = PVE::Ceph::Tools::get_config('ccname');
 975     my $mgrdir = "/var/lib/ceph/mgr/$clustername-$id";
 976     my $mgrkeyring = "$mgrdir/keyring";
 977     my $mgrname = "mgr.$id";
 978
 979     die "ceph manager directory '$mgrdir' already exists\n"
 980         if -d $mgrdir;
 981
 982     print "creating manager directory '$mgrdir'\n";
 983     mkdir $mgrdir;
 984     print "creating keys for '$mgrname'\n";
 985     my $output = $rados->mon_command({ prefix => 'auth get-or-create',
 986                                        entity => $mgrname,
 987                                        caps => [
 988                                            mon => 'allow profile mgr',
 989                                            osd => 'allow *',
 990                                            mds => 'allow *',
 991                                        ],
 992                                        format => 'plain'});
 993     file_set_contents($mgrkeyring, $output);
 994
 995     print "setting owner for directory\n";
 996     run_command(["chown", 'ceph:ceph', '-R', $mgrdir]);
 997
 998     print "enabling service 'ceph-mgr\@$id.service'\n";
 999     PVE::Ceph::Services::ceph_service_cmd('enable', $mgrname);
1000     print "starting service 'ceph-mgr\@$id.service'\n";
1001     PVE::Ceph::Services::ceph_service_cmd('start', $mgrname);
1002 };
1003
1004 my $destroy_mgr = sub {
1005     my ($mgrid) = @_;
1006
1007     my $clustername = PVE::Ceph::Tools::get_config('ccname');
1008     my $mgrname = "mgr.$mgrid";
1009     my $mgrdir = "/var/lib/ceph/mgr/$clustername-$mgrid";
1010
1011     die "ceph manager directory '$mgrdir' not found\n"
1012         if ! -d $mgrdir;
1013
1014     print "disabling service 'ceph-mgr\@$mgrid.service'\n";
1015     PVE::Ceph::Services::ceph_service_cmd('disable', $mgrname);
1016     print "stopping service 'ceph-mgr\@$mgrid.service'\n";
1017     PVE::Ceph::Services::ceph_service_cmd('stop', $mgrname);
1018
1019     print "removing manager directory '$mgrdir'\n";
1020     File::Path::remove_tree($mgrdir);
1021 };
1022
1023 __PACKAGE__->register_method ({
1024     name => 'createmon',
1025     path => 'mon',
1026     method => 'POST',
1027     description => "Create Ceph Monitor and Manager",
1028     proxyto => 'node',
1029     protected => 1,
1030     permissions => {
1031         check => ['perm', '/', [ 'Sys.Modify' ]],
1032     },
1033     parameters => {
1034         additionalProperties => 0,
1035         properties => {
1036             node => get_standard_option('pve-node'),
1037             id => {
1038                 type => 'string',
1039                 optional => 1,
1040                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1041                 description => "The ID for the monitor, when omitted the same as the nodename",
1042             },
1043             'exclude-manager' => {
1044                 type => 'boolean',
1045                 optional => 1,
1046                 default => 0,
1047                 description => "When set, only a monitor will be created.",
1048             },
1049             'mon-address' => {
1050                 description => 'Overwrites autodetected monitor IP address. ' .
1051                                'Must be in the public network of ceph.',
1052                 type => 'string', format => 'ip',
1053                 optional => 1,
1054             },
1055         },
1056     },
1057     returns => { type => 'string' },
1058     code => sub {
1059         my ($param) = @_;
1060
1061         PVE::Ceph::Tools::check_ceph_installed('ceph_mon');
1062
1063         PVE::Ceph::Tools::check_ceph_installed('ceph_mgr')
1064             if (!$param->{'exclude-manager'});
1065
1066         PVE::Ceph::Tools::check_ceph_inited();
1067
1068         PVE::Ceph::Tools::setup_pve_symlinks();
1069
1070         my $rpcenv = PVE::RPCEnvironment::get();
1071
1072         my $authuser = $rpcenv->get_user();
1073
1074         my $cfg = cfs_read_file('ceph.conf');
1075
1076         my $moncount = 0;
1077
1078         my $monaddrhash = {};
1079
1080         my $systemd_managed = PVE::Ceph::Tools::systemd_managed();
1081
1082         foreach my $section (keys %$cfg) {
1083             next if $section eq 'global';
1084             my $d = $cfg->{$section};
1085             if ($section =~ m/^mon\./) {
1086                 $moncount++;
1087                 if ($d->{'mon addr'}) {
1088                     $monaddrhash->{$d->{'mon addr'}} = $section;
1089                 }
1090             }
1091         }
1092
1093         my $monid = $param->{id} // $param->{node};
1094
1095         my $monsection = "mon.$monid";
1096         my $pubnet = $cfg->{global}->{'public network'};
1097         my $ip = $find_mon_ip->($pubnet, $param->{node}, $param->{'mon-address'});
1098
1099         my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]:6789" : "$ip:6789";
1100         my $monname = $param->{node};
1101
1102         die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
1103         die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n"
1104             if $monaddrhash->{$monaddr};
1105
1106         my $worker = sub  {
1107             my $upid = shift;
1108
1109             my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
1110
1111             if (! -f $pve_ckeyring_path) {
1112                 run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
1113                             "--gen-key -n client.admin");
1114             }
1115
1116             my $pve_mon_key_path = PVE::Ceph::Tools::get_config('pve_mon_key_path');
1117             if (! -f $pve_mon_key_path) {
1118                 run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
1119                 run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " .
1120                             "--cap mds 'allow' " .
1121                             "--cap osd 'allow *' " .
1122                             "--cap mgr 'allow *' " .
1123                             "--cap mon 'allow *'");
1124                 run_command("cp $pve_mon_key_path.tmp /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
1125                 run_command("chown ceph:ceph /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
1126                 run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
1127                 run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
1128             }
1129
1130             my $ccname = PVE::Ceph::Tools::get_config('ccname');
1131
1132             my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
1133             -d $mondir && die "monitor filesystem '$mondir' already exist\n";
1134
1135             my $monmap = "/tmp/monmap";
1136
1137             eval {
1138                 mkdir $mondir;
1139
1140                 run_command("chown ceph:ceph $mondir") if $systemd_managed;
1141
1142                 if ($moncount > 0) {
1143                     my $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
1144                     my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
1145                     file_set_contents($monmap, $mapdata);
1146                 } else {
1147                     run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
1148                 }
1149
1150                 run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
1151                 run_command("chown ceph:ceph -R $mondir") if $systemd_managed;
1152             };
1153             my $err = $@;
1154             unlink $monmap;
1155             if ($err) {
1156                 File::Path::remove_tree($mondir);
1157                 die $err;
1158             }
1159
1160             $cfg->{$monsection} = {
1161                 'host' => $monname,
1162                 'mon addr' => $monaddr,
1163             };
1164
1165             cfs_write_file('ceph.conf', $cfg);
1166
1167             my $create_keys_pid = fork();
1168             if (!defined($create_keys_pid)) {
1169                 die "Could not spawn ceph-create-keys to create bootstrap keys\n";
1170             } elsif ($create_keys_pid == 0) {
1171                 exit PVE::Tools::run_command(['ceph-create-keys', '-i', $monid]);
1172             } else {
1173                 PVE::Ceph::Services::ceph_service_cmd('start', $monsection);
1174
1175                 if ($systemd_managed) {
1176                     #to ensure we have the correct startup order.
1177                     eval { PVE::Tools::run_command(['/bin/systemctl', 'enable', "ceph-mon\@${monid}.service"]); };
1178                     warn "Enable ceph-mon\@${monid}.service manually"if $@;
1179                 }
1180                 waitpid($create_keys_pid, 0);
1181             }
1182
1183             # create manager
1184             if (!$param->{'exclude-manager'}) {
1185                 my $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
1186                 $create_mgr->($rados, $monid);
1187             }
1188         };
1189
1190         return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
1191     }});
1192
1193 __PACKAGE__->register_method ({
1194     name => 'destroymon',
1195     path => 'mon/{monid}',
1196     method => 'DELETE',
1197     description => "Destroy Ceph Monitor and Manager.",
1198     proxyto => 'node',
1199     protected => 1,
1200     permissions => {
1201         check => ['perm', '/', [ 'Sys.Modify' ]],
1202     },
1203     parameters => {
1204         additionalProperties => 0,
1205         properties => {
1206             node => get_standard_option('pve-node'),
1207             monid => {
1208                 description => 'Monitor ID',
1209                 type => 'string',
1210                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1211             },
1212             'exclude-manager' => {
1213                 type => 'boolean',
1214                 default => 0,
1215                 optional => 1,
1216                 description => "When set, removes only the monitor, not the manager"
1217             }
1218         },
1219     },
1220     returns => { type => 'string' },
1221     code => sub {
1222         my ($param) = @_;
1223
1224         my $rpcenv = PVE::RPCEnvironment::get();
1225
1226         my $authuser = $rpcenv->get_user();
1227
1228         PVE::Ceph::Tools::check_ceph_inited();
1229
1230         my $cfg = cfs_read_file('ceph.conf');
1231
1232         my $monid = $param->{monid};
1233         my $monsection = "mon.$monid";
1234
1235         my $rados = PVE::RADOS->new();
1236         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
1237         my $monlist = $monstat->{monmap}->{mons};
1238
1239         die "no such monitor id '$monid'\n"
1240             if !defined($cfg->{$monsection});
1241
1242         my $ccname = PVE::Ceph::Tools::get_config('ccname');
1243
1244         my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
1245         -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n";
1246
1247         die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
1248
1249         my $worker = sub {
1250             my $upid = shift;
1251
1252             # reopen with longer timeout
1253             $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
1254
1255             $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });
1256
1257             eval { PVE::Ceph::Services::ceph_service_cmd('stop', $monsection); };
1258             warn $@ if $@;
1259
1260             delete $cfg->{$monsection};
1261             cfs_write_file('ceph.conf', $cfg);
1262             File::Path::remove_tree($mondir);
1263
1264             # remove manager
1265             if (!$param->{'exclude-manager'}) {
1266                 eval { $destroy_mgr->($monid); };
1267                 warn $@ if $@;
1268             }
1269         };
1270
1271         return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
1272     }});
1273
1274 __PACKAGE__->register_method ({
1275     name => 'createmgr',
1276     path => 'mgr',
1277     method => 'POST',
1278     description => "Create Ceph Manager",
1279     proxyto => 'node',
1280     protected => 1,
1281     permissions => {
1282         check => ['perm', '/', [ 'Sys.Modify' ]],
1283     },
1284     parameters => {
1285         additionalProperties => 0,
1286         properties => {
1287             node => get_standard_option('pve-node'),
1288             id => {
1289                 type => 'string',
1290                 optional => 1,
1291                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1292                 description => "The ID for the manager, when omitted the same as the nodename",
1293             },
1294         },
1295     },
1296     returns => { type => 'string' },
1297     code => sub {
1298         my ($param) = @_;
1299
1300         PVE::Ceph::Tools::check_ceph_installed('ceph_mgr');
1301
1302         PVE::Ceph::Tools::check_ceph_inited();
1303
1304         my $rpcenv = PVE::RPCEnvironment::get();
1305
1306         my $authuser = $rpcenv->get_user();
1307
1308         my $mgrid = $param->{id} // $param->{node};
1309
1310         my $worker = sub  {
1311             my $upid = shift;
1312
1313             my $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
1314
1315             $create_mgr->($rados, $mgrid);
1316         };
1317
1318         return $rpcenv->fork_worker('cephcreatemgr', "mgr.$mgrid", $authuser, $worker);
1319     }});
1320
1321 __PACKAGE__->register_method ({
1322     name => 'destroymgr',
1323     path => 'mgr/{id}',
1324     method => 'DELETE',
1325     description => "Destroy Ceph Manager.",
1326     proxyto => 'node',
1327     protected => 1,
1328     permissions => {
1329         check => ['perm', '/', [ 'Sys.Modify' ]],
1330     },
1331     parameters => {
1332         additionalProperties => 0,
1333         properties => {
1334             node => get_standard_option('pve-node'),
1335             id => {
1336                 description => 'The ID of the manager',
1337                 type => 'string',
1338                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1339             },
1340         },
1341     },
1342     returns => { type => 'string' },
1343     code => sub {
1344         my ($param) = @_;
1345
1346         my $rpcenv = PVE::RPCEnvironment::get();
1347
1348         my $authuser = $rpcenv->get_user();
1349
1350         PVE::Ceph::Tools::check_ceph_inited();
1351
1352         my $mgrid = $param->{id};
1353
1354         my $worker = sub {
1355             my $upid = shift;
1356
1357             $destroy_mgr->($mgrid);
1358         };
1359
1360         return $rpcenv->fork_worker('cephdestroymgr', "mgr.$mgrid",  $authuser, $worker);
1361     }});
1362
1363 __PACKAGE__->register_method ({
1364     name => 'stop',
1365     path => 'stop',
1366     method => 'POST',
1367     description => "Stop ceph services.",
1368     proxyto => 'node',
1369     protected => 1,
1370     permissions => {
1371         check => ['perm', '/', [ 'Sys.Modify' ]],
1372     },
1373     parameters => {
1374         additionalProperties => 0,
1375         properties => {
1376             node => get_standard_option('pve-node'),
1377             service => {
1378                 description => 'Ceph service name.',
1379                 type => 'string',
1380                 optional => 1,
1381                 default => 'ceph.target',
1382                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1383             },
1384         },
1385     },
1386     returns => { type => 'string' },
1387     code => sub {
1388         my ($param) = @_;
1389
1390         my $rpcenv = PVE::RPCEnvironment::get();
1391
1392         my $authuser = $rpcenv->get_user();
1393
1394         PVE::Ceph::Tools::check_ceph_inited();
1395
1396         my $cfg = cfs_read_file('ceph.conf');
1397         scalar(keys %$cfg) || die "no configuration\n";
1398
1399         my $worker = sub {
1400             my $upid = shift;
1401
1402             my $cmd = ['stop'];
1403             if ($param->{service}) {
1404                 push @$cmd, $param->{service};
1405             }
1406
1407             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
1408         };
1409
1410         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
1411                                     $authuser, $worker);
1412     }});
1413
1414 __PACKAGE__->register_method ({
1415     name => 'start',
1416     path => 'start',
1417     method => 'POST',
1418     description => "Start ceph services.",
1419     proxyto => 'node',
1420     protected => 1,
1421     permissions => {
1422         check => ['perm', '/', [ 'Sys.Modify' ]],
1423     },
1424     parameters => {
1425         additionalProperties => 0,
1426         properties => {
1427             node => get_standard_option('pve-node'),
1428             service => {
1429                 description => 'Ceph service name.',
1430                 type => 'string',
1431                 optional => 1,
1432                 default => 'ceph.target',
1433                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1434             },
1435         },
1436     },
1437     returns => { type => 'string' },
1438     code => sub {
1439         my ($param) = @_;
1440
1441         my $rpcenv = PVE::RPCEnvironment::get();
1442
1443         my $authuser = $rpcenv->get_user();
1444
1445         PVE::Ceph::Tools::check_ceph_inited();
1446
1447         my $cfg = cfs_read_file('ceph.conf');
1448         scalar(keys %$cfg) || die "no configuration\n";
1449
1450         my $worker = sub {
1451             my $upid = shift;
1452
1453             my $cmd = ['start'];
1454             if ($param->{service}) {
1455                 push @$cmd, $param->{service};
1456             }
1457
1458             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
1459         };
1460
1461         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
1462                                     $authuser, $worker);
1463     }});
1464
1465 __PACKAGE__->register_method ({
1466     name => 'restart',
1467     path => 'restart',
1468     method => 'POST',
1469     description => "Restart ceph services.",
1470     proxyto => 'node',
1471     protected => 1,
1472     permissions => {
1473         check => ['perm', '/', [ 'Sys.Modify' ]],
1474     },
1475     parameters => {
1476         additionalProperties => 0,
1477         properties => {
1478             node => get_standard_option('pve-node'),
1479             service => {
1480                 description => 'Ceph service name.',
1481                 type => 'string',
1482                 optional => 1,
1483                 default => 'ceph.target',
1484                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1485             },
1486         },
1487     },
1488     returns => { type => 'string' },
1489     code => sub {
1490         my ($param) = @_;
1491
1492         my $rpcenv = PVE::RPCEnvironment::get();
1493
1494         my $authuser = $rpcenv->get_user();
1495
1496         PVE::Ceph::Tools::check_ceph_inited();
1497
1498         my $cfg = cfs_read_file('ceph.conf');
1499         scalar(keys %$cfg) || die "no configuration\n";
1500
1501         my $worker = sub {
1502             my $upid = shift;
1503
1504             my $cmd = ['restart'];
1505             if ($param->{service}) {
1506                 push @$cmd, $param->{service};
1507             }
1508
1509             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
1510         };
1511
1512         return $rpcenv->fork_worker('srvrestart', $param->{service} || 'ceph',
1513                                     $authuser, $worker);
1514     }});
1515
1516 __PACKAGE__->register_method ({
1517     name => 'status',
1518     path => 'status',
1519     method => 'GET',
1520     description => "Get ceph status.",
1521     proxyto => 'node',
1522     protected => 1,
1523     permissions => {
1524         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1525     },
1526     parameters => {
1527         additionalProperties => 0,
1528         properties => {
1529             node => get_standard_option('pve-node'),
1530         },
1531     },
1532     returns => { type => 'object' },
1533     code => sub {
1534         my ($param) = @_;
1535
1536         PVE::Ceph::Tools::check_ceph_enabled();
1537
1538         my $rados = PVE::RADOS->new();
1539         my $status = $rados->mon_command({ prefix => 'status' });
1540         $status->{health} = $rados->mon_command({ prefix => 'health', detail => 'detail' });
1541         return $status;
1542     }});
1543
1544 __PACKAGE__->register_method ({
1545     name => 'lspools',
1546     path => 'pools',
1547     method => 'GET',
1548     description => "List all pools.",
1549     proxyto => 'node',
1550     protected => 1,
1551     permissions => {
1552         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1553     },
1554     parameters => {
1555         additionalProperties => 0,
1556         properties => {
1557             node => get_standard_option('pve-node'),
1558         },
1559     },
1560     returns => {
1561         type => 'array',
1562         items => {
1563             type => "object",
1564             properties => {
1565                 pool => { type => 'integer' },
1566                 pool_name => { type => 'string' },
1567                 size => { type => 'integer' },
1568             },
1569         },
1570         links => [ { rel => 'child', href => "{pool_name}" } ],
1571     },
1572     code => sub {
1573         my ($param) = @_;
1574
1575         PVE::Ceph::Tools::check_ceph_inited();
1576
1577         my $rados = PVE::RADOS->new();
1578
1579         my $stats = {};
1580         my $res = $rados->mon_command({ prefix => 'df' });
1581
1582         foreach my $d (@{$res->{pools}}) {
1583             next if !$d->{stats};
1584             next if !defined($d->{id});
1585             $stats->{$d->{id}} = $d->{stats};
1586         }
1587
1588         $res = $rados->mon_command({ prefix => 'osd dump' });
1589         my $rulestmp = $rados->mon_command({ prefix => 'osd crush rule dump'});
1590
1591         my $rules = {};
1592         for my $rule (@$rulestmp) {
1593             $rules->{$rule->{rule_id}} = $rule->{rule_name};
1594         }
1595
1596         my $data = [];
1597         foreach my $e (@{$res->{pools}}) {
1598             my $d = {};
1599             foreach my $attr (qw(pool pool_name size min_size pg_num crush_rule)) {
1600                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
1601             }
1602
1603             if (defined($d->{crush_rule}) && defined($rules->{$d->{crush_rule}})) {
1604                 $d->{crush_rule_name} = $rules->{$d->{crush_rule}};
1605             }
1606
1607             if (my $s = $stats->{$d->{pool}}) {
1608                 $d->{bytes_used} = $s->{bytes_used};
1609                 $d->{percent_used} = $s->{percent_used};
1610             }
1611             push @$data, $d;
1612         }
1613
1614
1615         return $data;
1616     }});
1617
1618 __PACKAGE__->register_method ({
1619     name => 'createpool',
1620     path => 'pools',
1621     method => 'POST',
1622     description => "Create POOL",
1623     proxyto => 'node',
1624     protected => 1,
1625     permissions => {
1626         check => ['perm', '/', [ 'Sys.Modify' ]],
1627     },
1628     parameters => {
1629         additionalProperties => 0,
1630         properties => {
1631             node => get_standard_option('pve-node'),
1632             name => {
1633                 description => "The name of the pool. It must be unique.",
1634                 type => 'string',
1635             },
1636             size => {
1637                 description => 'Number of replicas per object',
1638                 type => 'integer',
1639                 default => 3,
1640                 optional => 1,
1641                 minimum => 1,
1642                 maximum => 7,
1643             },
1644             min_size => {
1645                 description => 'Minimum number of replicas per object',
1646                 type => 'integer',
1647                 default => 2,
1648                 optional => 1,
1649                 minimum => 1,
1650                 maximum => 7,
1651             },
1652             pg_num => {
1653                 description => "Number of placement groups.",
1654                 type => 'integer',
1655                 default => 128,
1656                 optional => 1,
1657                 minimum => 8,
1658                 maximum => 32768,
1659             },
1660             crush_rule => {
1661                 description => "The rule to use for mapping object placement in the cluster.",
1662                 type => 'string',
1663                 optional => 1,
1664             },
1665             application => {
1666                 description => "The application of the pool, 'rbd' by default.",
1667                 type => 'string',
1668                 enum => ['rbd', 'cephfs', 'rgw'],
1669                 optional => 1,
1670             },
1671             add_storages => {
1672                 description => "Configure VM and CT storage using the new pool.",
1673                 type => 'boolean',
1674                 optional => 1,
1675             },
1676         },
1677     },
1678     returns => { type => 'string' },
1679     code => sub {
1680         my ($param) = @_;
1681
1682         PVE::Cluster::check_cfs_quorum();
1683         PVE::Ceph::Tools::check_ceph_inited();
1684
1685         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
1686
1687         die "not fully configured - missing '$pve_ckeyring_path'\n"
1688             if ! -f $pve_ckeyring_path;
1689
1690         my $pool = $param->{name};
1691         my $rpcenv = PVE::RPCEnvironment::get();
1692         my $user = $rpcenv->get_user();
1693
1694         if ($param->{add_storages}) {
1695             $rpcenv->check($user, '/storage', ['Datastore.Allocate']);
1696             die "pool name contains characters which are illegal for storage naming\n"
1697                 if !PVE::JSONSchema::parse_storage_id($pool);
1698         }
1699
1700         my $pg_num = $param->{pg_num} || 128;
1701         my $size = $param->{size} || 3;
1702         my $min_size = $param->{min_size} || 2;
1703         my $application = $param->{application} // 'rbd';
1704
1705         my $worker = sub {
1706
1707             PVE::Ceph::Tools::create_pool($pool, $param);
1708
1709             if ($param->{add_storages}) {
1710                 my $err;
1711                 eval { $add_storage->($pool, "${pool}"); };
1712                 if ($@) {
1713                     warn "failed to add storage: $@";
1714                     $err = 1;
1715                 }
1716                 die "adding storage for pool '$pool' failed, check log and add manually!\n"
1717                     if $err;
1718             }
1719         };
1720
1721         return $rpcenv->fork_worker('cephcreatepool', $pool,  $user, $worker);
1722     }});
1723
1724 __PACKAGE__->register_method ({
1725     name => 'get_flags',
1726     path => 'flags',
1727     method => 'GET',
1728     description => "get all set ceph flags",
1729     proxyto => 'node',
1730     protected => 1,
1731     permissions => {
1732         check => ['perm', '/', [ 'Sys.Audit' ]],
1733     },
1734     parameters => {
1735         additionalProperties => 0,
1736         properties => {
1737             node => get_standard_option('pve-node'),
1738         },
1739     },
1740     returns => { type => 'string' },
1741     code => sub {
1742         my ($param) = @_;
1743
1744         PVE::Ceph::Tools::check_ceph_inited();
1745
1746         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
1747
1748         die "not fully configured - missing '$pve_ckeyring_path'\n"
1749             if ! -f $pve_ckeyring_path;
1750
1751         my $rados = PVE::RADOS->new();
1752
1753         my $stat = $rados->mon_command({ prefix => 'osd dump' });
1754
1755         return $stat->{flags} // '';
1756     }});
1757
1758 __PACKAGE__->register_method ({
1759     name => 'set_flag',
1760     path => 'flags/{flag}',
1761     method => 'POST',
1762     description => "Set a ceph flag",
1763     proxyto => 'node',
1764     protected => 1,
1765     permissions => {
1766         check => ['perm', '/', [ 'Sys.Modify' ]],
1767     },
1768     parameters => {
1769         additionalProperties => 0,
1770         properties => {
1771             node => get_standard_option('pve-node'),
1772             flag => {
1773                 description => 'The ceph flag to set/unset',
1774                 type => 'string',
1775                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1776             },
1777         },
1778     },
1779     returns => { type => 'null' },
1780     code => sub {
1781         my ($param) = @_;
1782
1783         PVE::Ceph::Tools::check_ceph_inited();
1784
1785         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
1786
1787         die "not fully configured - missing '$pve_ckeyring_path'\n"
1788             if ! -f $pve_ckeyring_path;
1789
1790         my $set = $param->{set} // !$param->{unset};
1791         my $rados = PVE::RADOS->new();
1792
1793         $rados->mon_command({
1794             prefix => "osd set",
1795             key => $param->{flag},
1796         });
1797
1798         return undef;
1799     }});
1800
1801 __PACKAGE__->register_method ({
1802     name => 'unset_flag',
1803     path => 'flags/{flag}',
1804     method => 'DELETE',
1805     description => "Unset a ceph flag",
1806     proxyto => 'node',
1807     protected => 1,
1808     permissions => {
1809         check => ['perm', '/', [ 'Sys.Modify' ]],
1810     },
1811     parameters => {
1812         additionalProperties => 0,
1813         properties => {
1814             node => get_standard_option('pve-node'),
1815             flag => {
1816                 description => 'The ceph flag to set/unset',
1817                 type => 'string',
1818                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1819             },
1820         },
1821     },
1822     returns => { type => 'null' },
1823     code => sub {
1824         my ($param) = @_;
1825
1826         PVE::Ceph::Tools::check_ceph_inited();
1827
1828         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
1829
1830         die "not fully configured - missing '$pve_ckeyring_path'\n"
1831             if ! -f $pve_ckeyring_path;
1832
1833         my $set = $param->{set} // !$param->{unset};
1834         my $rados = PVE::RADOS->new();
1835
1836         $rados->mon_command({
1837             prefix => "osd unset",
1838             key => $param->{flag},
1839         });
1840
1841         return undef;
1842     }});
1843
1844 __PACKAGE__->register_method ({
1845     name => 'destroypool',
1846     path => 'pools/{name}',
1847     method => 'DELETE',
1848     description => "Destroy pool",
1849     proxyto => 'node',
1850     protected => 1,
1851     permissions => {
1852         check => ['perm', '/', [ 'Sys.Modify' ]],
1853     },
1854     parameters => {
1855         additionalProperties => 0,
1856         properties => {
1857             node => get_standard_option('pve-node'),
1858             name => {
1859                 description => "The name of the pool. It must be unique.",
1860                 type => 'string',
1861             },
1862             force => {
1863                 description => "If true, destroys pool even if in use",
1864                 type => 'boolean',
1865                 optional => 1,
1866                 default => 0,
1867             },
1868             remove_storages => {
1869                 description => "Remove all pveceph-managed storages configured for this pool",
1870                 type => 'boolean',
1871                 optional => 1,
1872                 default => 0,
1873             },
1874         },
1875     },
1876     returns => { type => 'string' },
1877     code => sub {
1878         my ($param) = @_;
1879
1880         PVE::Ceph::Tools::check_ceph_inited();
1881
1882         my $rpcenv = PVE::RPCEnvironment::get();
1883         my $user = $rpcenv->get_user();
1884         $rpcenv->check($user, '/storage', ['Datastore.Allocate'])
1885             if $param->{remove_storages};
1886
1887         my $pool = $param->{name};
1888
1889         my $worker = sub {
1890             my $storages = $get_storages->($pool);
1891
1892             # if not forced, destroy ceph pool only when no
1893             # vm disks are on it anymore
1894             if (!$param->{force}) {
1895                 my $storagecfg = PVE::Storage::config();
1896                 foreach my $storeid (keys %$storages) {
1897                     my $storage = $storages->{$storeid};
1898
1899                     # check if any vm disks are on the pool
1900                     print "checking storage '$storeid' for RBD images..\n";
1901                     my $res = PVE::Storage::vdisk_list($storagecfg, $storeid);
1902                     die "ceph pool '$pool' still in use by storage '$storeid'\n"
1903                         if @{$res->{$storeid}} != 0;
1904                 }
1905             }
1906
1907             PVE::Ceph::Tools::destroy_pool($pool);
1908
1909             if ($param->{remove_storages}) {
1910                 my $err;
1911                 foreach my $storeid (keys %$storages) {
1912                     # skip external clusters, not managed by pveceph
1913                     next if $storages->{$storeid}->{monhost};
1914                     eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
1915                     if ($@) {
1916                         warn "failed to remove storage '$storeid': $@\n";
1917                         $err = 1;
1918                     }
1919                 }
1920                 die "failed to remove (some) storages - check log and remove manually!\n"
1921                     if $err;
1922             }
1923         };
1924         return $rpcenv->fork_worker('cephdestroypool', $pool,  $user, $worker);
1925     }});
1926
1927
1928 __PACKAGE__->register_method ({
1929     name => 'crush',
1930     path => 'crush',
1931     method => 'GET',
1932     description => "Get OSD crush map",
1933     proxyto => 'node',
1934     protected => 1,
1935     permissions => {
1936         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1937     },
1938     parameters => {
1939         additionalProperties => 0,
1940         properties => {
1941             node => get_standard_option('pve-node'),
1942         },
1943     },
1944     returns => { type => 'string' },
1945     code => sub {
1946         my ($param) = @_;
1947
1948         PVE::Ceph::Tools::check_ceph_inited();
1949
1950         # this produces JSON (difficult to read for the user)
1951         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
1952
1953         my $txt = '';
1954
1955         my $mapfile = "/var/tmp/ceph-crush.map.$$";
1956         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
1957
1958         my $rados = PVE::RADOS->new();
1959
1960         eval {
1961             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
1962             file_set_contents($mapfile, $bindata);
1963             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
1964             $txt = file_get_contents($mapdata);
1965         };
1966         my $err = $@;
1967
1968         unlink $mapfile;
1969         unlink $mapdata;
1970
1971         die $err if $err;
1972
1973         return $txt;
1974     }});
1975
1976 __PACKAGE__->register_method({
1977     name => 'log',
1978     path => 'log',
1979     method => 'GET',
1980     description => "Read ceph log",
1981     proxyto => 'node',
1982     permissions => {
1983         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
1984     },
1985     protected => 1,
1986     parameters => {
1987         additionalProperties => 0,
1988         properties => {
1989             node => get_standard_option('pve-node'),
1990             start => {
1991                 type => 'integer',
1992                 minimum => 0,
1993                 optional => 1,
1994             },
1995             limit => {
1996                 type => 'integer',
1997                 minimum => 0,
1998                 optional => 1,
1999             },
2000         },
2001     },
2002     returns => {
2003         type => 'array',
2004         items => {
2005             type => "object",
2006             properties => {
2007                 n => {
2008                   description=>  "Line number",
2009                   type=> 'integer',
2010                 },
2011                 t => {
2012                   description=>  "Line text",
2013                   type => 'string',
2014                 }
2015             }
2016         }
2017     },
2018     code => sub {
2019         my ($param) = @_;
2020
2021         PVE::Ceph::Tools::check_ceph_inited();
2022
2023         my $rpcenv = PVE::RPCEnvironment::get();
2024         my $user = $rpcenv->get_user();
2025         my $node = $param->{node};
2026
2027         my $logfile = "/var/log/ceph/ceph.log";
2028         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
2029
2030         $rpcenv->set_result_attrib('total', $count);
2031
2032         return $lines;
2033     }});
2034
2035 __PACKAGE__->register_method ({
2036     name => 'rules',
2037     path => 'rules',
2038     method => 'GET',
2039     description => "List ceph rules.",
2040     proxyto => 'node',
2041     protected => 1,
2042     permissions => {
2043         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
2044     },
2045     parameters => {
2046         additionalProperties => 0,
2047         properties => {
2048             node => get_standard_option('pve-node'),
2049         },
2050     },
2051     returns => {
2052         type => 'array',
2053         items => {
2054             type => "object",
2055             properties => {},
2056         },
2057         links => [ { rel => 'child', href => "{name}" } ],
2058     },
2059     code => sub {
2060         my ($param) = @_;
2061
2062         PVE::Ceph::Tools::check_ceph_inited();
2063
2064         my $rados = PVE::RADOS->new();
2065
2066         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
2067
2068         my $res = [];
2069
2070         foreach my $rule (@$rules) {
2071             push @$res, { name => $rule };
2072         }
2073
2074         return $res;
2075     }});