PVE/API2/Ceph.pm

   1 package PVE::API2::CephOSD;
   2
   3 use strict;
   4 use warnings;
   5 use Cwd qw(abs_path);
   6 use Net::IP;
   7
   8 use PVE::SafeSyslog;
   9 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
  10 use PVE::Exception qw(raise raise_param_exc);
  11 use PVE::INotify;
  12 use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
  13 use PVE::AccessControl;
  14 use PVE::Storage;
  15 use PVE::RESTHandler;
  16 use PVE::RPCEnvironment;
  17 use PVE::JSONSchema qw(get_standard_option);
  18 use PVE::RADOS;
  19 use PVE::CephTools;
  20 use PVE::Diskmanage;
  21
  22 use base qw(PVE::RESTHandler);
  23
  24 use Data::Dumper; # fixme: remove
  25
  26 my $get_osd_status = sub {
  27     my ($rados, $osdid) = @_;
  28
  29     my $stat = $rados->mon_command({ prefix => 'osd dump' });
  30
  31     my $osdlist = $stat->{osds} || [];
  32
  33     my $flags = $stat->{flags} || undef;
  34
  35     my $osdstat;
  36     foreach my $d (@$osdlist) {
  37         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  38     }
  39     if (defined($osdid)) {
  40         die "no such OSD '$osdid'\n" if !$osdstat->{$osdid};
  41         return $osdstat->{$osdid};
  42     }
  43
  44     return wantarray? ($osdstat, $flags):$osdstat;
  45 };
  46
  47 my $get_osd_usage = sub {
  48     my ($rados) = @_;
  49
  50     my $osdlist = $rados->mon_command({ prefix => 'pg dump',
  51                                         dumpcontents => [ 'osds' ]}) || [];
  52
  53     my $osdstat;
  54     foreach my $d (@$osdlist) {
  55         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  56     }
  57
  58     return $osdstat;
  59 };
  60
  61 __PACKAGE__->register_method ({
  62     name => 'index',
  63     path => '',
  64     method => 'GET',
  65     description => "Get Ceph osd list/tree.",
  66     proxyto => 'node',
  67     protected => 1,
  68     permissions => {
  69         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  70     },
  71     parameters => {
  72         additionalProperties => 0,
  73         properties => {
  74             node => get_standard_option('pve-node'),
  75         },
  76     },
  77     # fixme: return a list instead of extjs tree format ?
  78     returns => {
  79         type => "object",
  80     },
  81     code => sub {
  82         my ($param) = @_;
  83
  84         PVE::CephTools::check_ceph_inited();
  85
  86         my $rados = PVE::RADOS->new();
  87         my $res = $rados->mon_command({ prefix => 'osd tree' });
  88
  89         die "no tree nodes found\n" if !($res && $res->{nodes});
  90
  91         my ($osdhash, $flags) = &$get_osd_status($rados);
  92
  93         my $usagehash = &$get_osd_usage($rados);
  94
  95         my $osdmetadata_tmp = $rados->mon_command({ prefix => 'osd metadata' });
  96
  97         my $osdmetadata = {};
  98         foreach my $osd (@$osdmetadata_tmp) {
  99             $osdmetadata->{$osd->{id}} = $osd;
 100         }
 101
 102         my $nodes = {};
 103         my $newnodes = {};
 104         foreach my $e (@{$res->{nodes}}) {
 105             $nodes->{$e->{id}} = $e;
 106
 107             my $new = {
 108                 id => $e->{id},
 109                 name => $e->{name},
 110                 type => $e->{type}
 111             };
 112
 113             foreach my $opt (qw(status crush_weight reweight device_class)) {
 114                 $new->{$opt} = $e->{$opt} if defined($e->{$opt});
 115             }
 116
 117             if (my $stat = $osdhash->{$e->{id}}) {
 118                 $new->{in} = $stat->{in} if defined($stat->{in});
 119             }
 120
 121             if (my $stat = $usagehash->{$e->{id}}) {
 122                 $new->{total_space} = ($stat->{kb} || 1) * 1024;
 123                 $new->{bytes_used} = ($stat->{kb_used} || 0) * 1024;
 124                 $new->{percent_used} = ($new->{bytes_used}*100)/$new->{total_space};
 125                 if (my $d = $stat->{fs_perf_stat}) {
 126                     $new->{commit_latency_ms} = $d->{commit_latency_ms};
 127                     $new->{apply_latency_ms} = $d->{apply_latency_ms};
 128                 }
 129             }
 130
 131             my $osdmd = $osdmetadata->{$e->{id}};
 132             if ($e->{type} eq 'osd' && $osdmd) {
 133                 if ($osdmd->{bluefs}) {
 134                     $new->{osdtype} = 'bluestore';
 135                     $new->{blfsdev} = $osdmd->{bluestore_bdev_dev_node};
 136                     $new->{dbdev} = $osdmd->{bluefs_db_dev_node};
 137                     $new->{waldev} = $osdmd->{bluefs_wal_dev_node};
 138                 } else {
 139                     $new->{osdtype} = 'filestore';
 140                 }
 141             }
 142
 143             $newnodes->{$e->{id}} = $new;
 144         }
 145
 146         foreach my $e (@{$res->{nodes}}) {
 147             my $new = $newnodes->{$e->{id}};
 148             if ($e->{children} && scalar(@{$e->{children}})) {
 149                 $new->{children} = [];
 150                 $new->{leaf} = 0;
 151                 foreach my $cid (@{$e->{children}}) {
 152                     $nodes->{$cid}->{parent} = $e->{id};
 153                     if ($nodes->{$cid}->{type} eq 'osd' &&
 154                         $e->{type} eq 'host') {
 155                         $newnodes->{$cid}->{host} = $e->{name};
 156                     }
 157                     push @{$new->{children}}, $newnodes->{$cid};
 158                 }
 159             } else {
 160                 $new->{leaf} = ($e->{id} >= 0) ? 1 : 0;
 161             }
 162         }
 163
 164         my $roots = [];
 165         foreach my $e (@{$res->{nodes}}) {
 166             if (!$nodes->{$e->{id}}->{parent}) {
 167                 push @$roots, $newnodes->{$e->{id}};
 168             }
 169         }
 170
 171         die "no root node\n" if !@$roots;
 172
 173         my $data = { root => { leaf =>  0, children => $roots } };
 174
 175         # we want this for the noout flag
 176         $data->{flags} = $flags if $flags;
 177
 178         return $data;
 179     }});
 180
 181 __PACKAGE__->register_method ({
 182     name => 'createosd',
 183     path => '',
 184     method => 'POST',
 185     description => "Create OSD",
 186     proxyto => 'node',
 187     protected => 1,
 188     parameters => {
 189         additionalProperties => 0,
 190         properties => {
 191             node => get_standard_option('pve-node'),
 192             dev => {
 193                 description => "Block device name.",
 194                 type => 'string',
 195             },
 196             journal_dev => {
 197                 description => "Block device name for journal (filestore) or block.db (bluestore).",
 198                 optional => 1,
 199                 type => 'string',
 200             },
 201             wal_dev => {
 202                 description => "Block device name for block.wal (bluestore only).",
 203                 optional => 1,
 204                 type => 'string',
 205             },
 206             fstype => {
 207                 description => "File system type (filestore only).",
 208                 type => 'string',
 209                 enum => ['xfs', 'ext4', 'btrfs'],
 210                 default => 'xfs',
 211                 optional => 1,
 212             },
 213             bluestore => {
 214                 description => "Use bluestore instead of filestore.",
 215                 type => 'boolean',
 216                 default => 0,
 217                 optional => 1,
 218             },
 219         },
 220     },
 221     returns => { type => 'string' },
 222     code => sub {
 223         my ($param) = @_;
 224
 225         my $rpcenv = PVE::RPCEnvironment::get();
 226
 227         my $authuser = $rpcenv->get_user();
 228
 229         raise_param_exc({ 'bluestore' => "conflicts with parameter 'fstype'" })
 230             if (defined($param->{fstype}) && defined($param->{bluestore}) && $param->{bluestore});
 231
 232         PVE::CephTools::check_ceph_inited();
 233
 234         PVE::CephTools::setup_pve_symlinks();
 235
 236         my $bluestore = $param->{bluestore} // 0;
 237
 238         my $journal_dev;
 239         my $wal_dev;
 240
 241         if ($param->{journal_dev} && ($param->{journal_dev} ne $param->{dev})) {
 242             $journal_dev = PVE::Diskmanage::verify_blockdev_path($param->{journal_dev});
 243             # if only journal is given, also put the wal there
 244             $wal_dev = $journal_dev;
 245         }
 246
 247         if ($param->{wal_dev} &&
 248             ($param->{wal_dev} ne $param->{dev}) &&
 249             (!$param->{journal_dev} || $param->{wal_dev} ne $param->{journal_dev})) {
 250             raise_param_exc({ 'wal_dev' => "can only be set with paramater 'bluestore'"})
 251                 if !$bluestore;
 252             $wal_dev = PVE::Diskmanage::verify_blockdev_path($param->{wal_dev});
 253         }
 254
 255         $param->{dev} = PVE::Diskmanage::verify_blockdev_path($param->{dev});
 256
 257         my $devname = $param->{dev};
 258         $devname =~ s|/dev/||;
 259
 260         my $disklist = PVE::Diskmanage::get_disks($devname, 1);
 261
 262         my $diskinfo = $disklist->{$devname};
 263         die "unable to get device info for '$devname'\n"
 264             if !$diskinfo;
 265
 266         die "device '$param->{dev}' is in use\n"
 267             if $diskinfo->{used};
 268
 269         my $devpath = $diskinfo->{devpath};
 270         my $rados = PVE::RADOS->new();
 271         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 272         die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid};
 273
 274         my $fsid = $monstat->{monmap}->{fsid};
 275         $fsid = $1 if $fsid =~ m/^([0-9a-f\-]+)$/;
 276
 277         my $ceph_bootstrap_osd_keyring = PVE::CephTools::get_config('ceph_bootstrap_osd_keyring');
 278
 279         if (! -f $ceph_bootstrap_osd_keyring) {
 280             my $bindata = $rados->mon_command({ prefix => 'auth get', entity => 'client.bootstrap-osd', format => 'plain' });
 281             PVE::Tools::file_set_contents($ceph_bootstrap_osd_keyring, $bindata);
 282         };
 283
 284         my $worker = sub {
 285             my $upid = shift;
 286
 287             my $fstype = $param->{fstype} || 'xfs';
 288
 289
 290             my $ccname = PVE::CephTools::get_config('ccname');
 291
 292             my $cmd = ['ceph-disk', 'prepare', '--zap-disk',
 293                        '--cluster', $ccname, '--cluster-uuid', $fsid ];
 294
 295             if ($bluestore) {
 296                 print "create OSD on $devpath (bluestore)\n";
 297                 push @$cmd, '--bluestore';
 298
 299                 if ($journal_dev) {
 300                     print "using device '$journal_dev' for block.db\n";
 301                     push @$cmd, '--block.db', $journal_dev;
 302                 }
 303
 304                 if ($wal_dev) {
 305                     print "using device '$wal_dev' for block.wal\n";
 306                     push @$cmd, '--block.wal', $wal_dev;
 307                 }
 308
 309                 push @$cmd, $devpath;
 310             } else {
 311                 print "create OSD on $devpath ($fstype)\n";
 312                 push @$cmd, '--filestore', '--fs-type', $fstype;
 313                 if ($journal_dev) {
 314                     print "using device '$journal_dev' for journal\n";
 315                     push @$cmd, '--journal-dev', $devpath, $journal_dev;
 316                 } else {
 317                     push @$cmd, $devpath;
 318                 }
 319             }
 320
 321
 322             run_command($cmd);
 323         };
 324
 325         return $rpcenv->fork_worker('cephcreateosd', $devname,  $authuser, $worker);
 326     }});
 327
 328 __PACKAGE__->register_method ({
 329     name => 'destroyosd',
 330     path => '{osdid}',
 331     method => 'DELETE',
 332     description => "Destroy OSD",
 333     proxyto => 'node',
 334     protected => 1,
 335     parameters => {
 336         additionalProperties => 0,
 337         properties => {
 338             node => get_standard_option('pve-node'),
 339             osdid => {
 340                 description => 'OSD ID',
 341                 type => 'integer',
 342             },
 343             cleanup => {
 344                 description => "If set, we remove partition table entries.",
 345                 type => 'boolean',
 346                 optional => 1,
 347                 default => 0,
 348             },
 349         },
 350     },
 351     returns => { type => 'string' },
 352     code => sub {
 353         my ($param) = @_;
 354
 355         my $rpcenv = PVE::RPCEnvironment::get();
 356
 357         my $authuser = $rpcenv->get_user();
 358
 359         PVE::CephTools::check_ceph_inited();
 360
 361         my $osdid = $param->{osdid};
 362
 363         my $rados = PVE::RADOS->new();
 364         my $osdstat = &$get_osd_status($rados, $osdid);
 365
 366         die "osd is in use (in == 1)\n" if $osdstat->{in};
 367         #&$run_ceph_cmd(['osd', 'out', $osdid]);
 368
 369         die "osd is still runnung (up == 1)\n" if $osdstat->{up};
 370
 371         my $osdsection = "osd.$osdid";
 372
 373         my $worker = sub {
 374             my $upid = shift;
 375
 376             # reopen with longer timeout
 377             $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
 378
 379             print "destroy OSD $osdsection\n";
 380
 381             eval { PVE::CephTools::ceph_service_cmd('stop', $osdsection); };
 382             warn $@ if $@;
 383
 384             print "Remove $osdsection from the CRUSH map\n";
 385             $rados->mon_command({ prefix => "osd crush remove", name => $osdsection, format => 'plain' });
 386
 387             print "Remove the $osdsection authentication key.\n";
 388             $rados->mon_command({ prefix => "auth del", entity => $osdsection, format => 'plain' });
 389
 390             print "Remove OSD $osdsection\n";
 391             $rados->mon_command({ prefix => "osd rm", ids => [ $osdsection ], format => 'plain' });
 392
 393             # try to unmount from standard mount point
 394             my $mountpoint = "/var/lib/ceph/osd/ceph-$osdid";
 395
 396             my $remove_partition = sub {
 397                 my ($part) = @_;
 398
 399                 return if !$part || (! -b $part );
 400                 my $partnum = PVE::Diskmanage::get_partnum($part);
 401                 my $devpath = PVE::Diskmanage::get_blockdev($part);
 402
 403                 print "remove partition $part (disk '${devpath}', partnum $partnum)\n";
 404                 eval { run_command(['/sbin/sgdisk', '-d', $partnum, "${devpath}"]); };
 405                 warn $@ if $@;
 406             };
 407
 408             my $partitions_to_remove = [];
 409
 410             if ($param->{cleanup}) {
 411                 if (my $fd = IO::File->new("/proc/mounts", "r")) {
 412                     while (defined(my $line = <$fd>)) {
 413                         my ($dev, $path, $fstype) = split(/\s+/, $line);
 414                         next if !($dev && $path && $fstype);
 415                         next if $dev !~ m|^/dev/|;
 416                         if ($path eq $mountpoint) {
 417                             my $data_part = abs_path($dev);
 418                             push @$partitions_to_remove, $data_part;
 419                             last;
 420                         }
 421                     }
 422                     close($fd);
 423                 }
 424
 425                 foreach my $path (qw(journal block block.db block.wal)) {
 426                     my $part = abs_path("$mountpoint/$path");
 427                     if ($part) {
 428                         push @$partitions_to_remove, $part;
 429                     }
 430                 }
 431             }
 432
 433             print "Unmount OSD $osdsection from  $mountpoint\n";
 434             eval { run_command(['/bin/umount', $mountpoint]); };
 435             if (my $err = $@) {
 436                 warn $err;
 437             } elsif ($param->{cleanup}) {
 438                 #be aware of the ceph udev rules which can remount.
 439                 foreach my $part (@$partitions_to_remove) {
 440                     $remove_partition->($part);
 441                 }
 442             }
 443         };
 444
 445         return $rpcenv->fork_worker('cephdestroyosd', $osdsection,  $authuser, $worker);
 446     }});
 447
 448 __PACKAGE__->register_method ({
 449     name => 'in',
 450     path => '{osdid}/in',
 451     method => 'POST',
 452     description => "ceph osd in",
 453     proxyto => 'node',
 454     protected => 1,
 455     permissions => {
 456         check => ['perm', '/', [ 'Sys.Modify' ]],
 457     },
 458     parameters => {
 459         additionalProperties => 0,
 460         properties => {
 461             node => get_standard_option('pve-node'),
 462             osdid => {
 463                 description => 'OSD ID',
 464                 type => 'integer',
 465             },
 466         },
 467     },
 468     returns => { type => "null" },
 469     code => sub {
 470         my ($param) = @_;
 471
 472         PVE::CephTools::check_ceph_inited();
 473
 474         my $osdid = $param->{osdid};
 475
 476         my $rados = PVE::RADOS->new();
 477
 478         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 479
 480         my $osdsection = "osd.$osdid";
 481
 482         $rados->mon_command({ prefix => "osd in", ids => [ $osdsection ], format => 'plain' });
 483
 484         return undef;
 485     }});
 486
 487 __PACKAGE__->register_method ({
 488     name => 'out',
 489     path => '{osdid}/out',
 490     method => 'POST',
 491     description => "ceph osd out",
 492     proxyto => 'node',
 493     protected => 1,
 494     permissions => {
 495         check => ['perm', '/', [ 'Sys.Modify' ]],
 496     },
 497     parameters => {
 498         additionalProperties => 0,
 499         properties => {
 500             node => get_standard_option('pve-node'),
 501             osdid => {
 502                 description => 'OSD ID',
 503                 type => 'integer',
 504             },
 505         },
 506     },
 507     returns => { type => "null" },
 508     code => sub {
 509         my ($param) = @_;
 510
 511         PVE::CephTools::check_ceph_inited();
 512
 513         my $osdid = $param->{osdid};
 514
 515         my $rados = PVE::RADOS->new();
 516
 517         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 518
 519         my $osdsection = "osd.$osdid";
 520
 521         $rados->mon_command({ prefix => "osd out", ids => [ $osdsection ], format => 'plain' });
 522
 523         return undef;
 524     }});
 525
 526 package PVE::API2::Ceph;
 527
 528 use strict;
 529 use warnings;
 530 use File::Basename;
 531 use File::Path;
 532 use POSIX qw (LONG_MAX);
 533 use Cwd qw(abs_path);
 534 use IO::Dir;
 535 use UUID;
 536 use Net::IP;
 537
 538 use PVE::SafeSyslog;
 539 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
 540 use PVE::Exception qw(raise raise_param_exc);
 541 use PVE::INotify;
 542 use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
 543 use PVE::AccessControl;
 544 use PVE::Storage;
 545 use PVE::RESTHandler;
 546 use PVE::RPCEnvironment;
 547 use PVE::JSONSchema qw(get_standard_option);
 548 use JSON;
 549 use PVE::RADOS;
 550 use PVE::CephTools;
 551
 552 use base qw(PVE::RESTHandler);
 553
 554 use Data::Dumper; # fixme: remove
 555
 556 my $pve_osd_default_journal_size = 1024*5;
 557
 558 __PACKAGE__->register_method ({
 559     subclass => "PVE::API2::CephOSD",
 560     path => 'osd',
 561 });
 562
 563 __PACKAGE__->register_method ({
 564     name => 'index',
 565     path => '',
 566     method => 'GET',
 567     description => "Directory index.",
 568     permissions => { user => 'all' },
 569     permissions => {
 570         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 571     },
 572     parameters => {
 573         additionalProperties => 0,
 574         properties => {
 575             node => get_standard_option('pve-node'),
 576         },
 577     },
 578     returns => {
 579         type => 'array',
 580         items => {
 581             type => "object",
 582             properties => {},
 583         },
 584         links => [ { rel => 'child', href => "{name}" } ],
 585     },
 586     code => sub {
 587         my ($param) = @_;
 588
 589         my $result = [
 590             { name => 'init' },
 591             { name => 'mon' },
 592             { name => 'osd' },
 593             { name => 'pools' },
 594             { name => 'stop' },
 595             { name => 'start' },
 596             { name => 'status' },
 597             { name => 'crush' },
 598             { name => 'config' },
 599             { name => 'log' },
 600             { name => 'disks' },
 601             { name => 'flags' },
 602             { name => 'rules' },
 603         ];
 604
 605         return $result;
 606     }});
 607
 608 __PACKAGE__->register_method ({
 609     name => 'disks',
 610     path => 'disks',
 611     method => 'GET',
 612     description => "List local disks.",
 613     proxyto => 'node',
 614     protected => 1,
 615     permissions => {
 616         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 617     },
 618     parameters => {
 619         additionalProperties => 0,
 620         properties => {
 621             node => get_standard_option('pve-node'),
 622             type => {
 623                 description => "Only list specific types of disks.",
 624                 type => 'string',
 625                 enum => ['unused', 'journal_disks'],
 626                 optional => 1,
 627             },
 628         },
 629     },
 630     returns => {
 631         type => 'array',
 632         items => {
 633             type => "object",
 634             properties => {
 635                 dev => { type => 'string' },
 636                 used => { type => 'string', optional => 1 },
 637                 gpt => { type => 'boolean' },
 638                 size => { type => 'integer' },
 639                 osdid => { type => 'integer' },
 640                 vendor =>  { type => 'string', optional => 1 },
 641                 model =>  { type => 'string', optional => 1 },
 642                 serial =>  { type => 'string', optional => 1 },
 643             },
 644         },
 645         # links => [ { rel => 'child', href => "{}" } ],
 646     },
 647     code => sub {
 648         my ($param) = @_;
 649
 650         PVE::CephTools::check_ceph_inited();
 651
 652         my $disks = PVE::Diskmanage::get_disks(undef, 1);
 653
 654         my $res = [];
 655         foreach my $dev (keys %$disks) {
 656             my $d = $disks->{$dev};
 657             if ($param->{type}) {
 658                 if ($param->{type} eq 'journal_disks') {
 659                     next if $d->{osdid} >= 0;
 660                     next if !$d->{gpt};
 661                 } elsif ($param->{type} eq 'unused') {
 662                     next if $d->{used};
 663                 } else {
 664                     die "internal error"; # should not happen
 665                 }
 666             }
 667
 668             $d->{dev} = "/dev/$dev";
 669             push @$res, $d;
 670         }
 671
 672         return $res;
 673     }});
 674
 675 __PACKAGE__->register_method ({
 676     name => 'config',
 677     path => 'config',
 678     method => 'GET',
 679     permissions => {
 680         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 681     },
 682     description => "Get Ceph configuration.",
 683     parameters => {
 684         additionalProperties => 0,
 685         properties => {
 686             node => get_standard_option('pve-node'),
 687         },
 688     },
 689     returns => { type => 'string' },
 690     code => sub {
 691         my ($param) = @_;
 692
 693         PVE::CephTools::check_ceph_inited();
 694
 695         my $path = PVE::CephTools::get_config('pve_ceph_cfgpath');
 696         return PVE::Tools::file_get_contents($path);
 697
 698     }});
 699
 700 __PACKAGE__->register_method ({
 701     name => 'listmon',
 702     path => 'mon',
 703     method => 'GET',
 704     description => "Get Ceph monitor list.",
 705     proxyto => 'node',
 706     protected => 1,
 707     permissions => {
 708         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 709     },
 710     parameters => {
 711         additionalProperties => 0,
 712         properties => {
 713             node => get_standard_option('pve-node'),
 714         },
 715     },
 716     returns => {
 717         type => 'array',
 718         items => {
 719             type => "object",
 720             properties => {
 721                 name => { type => 'string' },
 722                 addr => { type => 'string' },
 723             },
 724         },
 725         links => [ { rel => 'child', href => "{name}" } ],
 726     },
 727     code => sub {
 728         my ($param) = @_;
 729
 730         PVE::CephTools::check_ceph_inited();
 731
 732         my $res = [];
 733
 734         my $cfg = PVE::CephTools::parse_ceph_config();
 735
 736         my $monhash = {};
 737         foreach my $section (keys %$cfg) {
 738             my $d = $cfg->{$section};
 739             if ($section =~ m/^mon\.(\S+)$/) {
 740                 my $monid = $1;
 741                 if ($d->{'mon addr'} && $d->{'host'}) {
 742                     $monhash->{$monid} = {
 743                         addr => $d->{'mon addr'},
 744                         host => $d->{'host'},
 745                         name => $monid,
 746                     }
 747                 }
 748             }
 749         }
 750
 751         eval {
 752             my $rados = PVE::RADOS->new();
 753             my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 754             my $mons = $monstat->{monmap}->{mons};
 755             foreach my $d (@$mons) {
 756                 next if !defined($d->{name});
 757                 $monhash->{$d->{name}}->{rank} = $d->{rank};
 758                 $monhash->{$d->{name}}->{addr} = $d->{addr};
 759                 if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
 760                     $monhash->{$d->{name}}->{quorum} = 1;
 761                 }
 762             }
 763         };
 764         warn $@ if $@;
 765
 766         return PVE::RESTHandler::hash_to_array($monhash, 'name');
 767     }});
 768
 769 __PACKAGE__->register_method ({
 770     name => 'init',
 771     path => 'init',
 772     method => 'POST',
 773     description => "Create initial ceph default configuration and setup symlinks.",
 774     proxyto => 'node',
 775     protected => 1,
 776     permissions => {
 777         check => ['perm', '/', [ 'Sys.Modify' ]],
 778     },
 779     parameters => {
 780         additionalProperties => 0,
 781         properties => {
 782             node => get_standard_option('pve-node'),
 783             network => {
 784                 description => "Use specific network for all ceph related traffic",
 785                 type => 'string', format => 'CIDR',
 786                 optional => 1,
 787                 maxLength => 128,
 788             },
 789             size => {
 790                 description => 'Targeted number of replicas per object',
 791                 type => 'integer',
 792                 default => 3,
 793                 optional => 1,
 794                 minimum => 1,
 795                 maximum => 7,
 796             },
 797             min_size => {
 798                 description => 'Minimum number of available replicas per object to allow I/O',
 799                 type => 'integer',
 800                 default => 2,
 801                 optional => 1,
 802                 minimum => 1,
 803                 maximum => 7,
 804             },
 805             pg_bits => {
 806                 description => "Placement group bits, used to specify the " .
 807                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 808                     "default pg num' does not work for default pools.",
 809                 type => 'integer',
 810                 default => 6,
 811                 optional => 1,
 812                 minimum => 6,
 813                 maximum => 14,
 814             },
 815             disable_cephx => {
 816                 description => "Disable cephx authentification.\n\n" .
 817                     "WARNING: cephx is a security feature protecting against " .
 818                     "man-in-the-middle attacks. Only consider disabling cephx ".
 819                     "if your network is private!",
 820                 type => 'boolean',
 821                 optional => 1,
 822                 default => 0,
 823             },
 824         },
 825     },
 826     returns => { type => 'null' },
 827     code => sub {
 828         my ($param) = @_;
 829
 830         PVE::CephTools::check_ceph_installed();
 831
 832         # simply load old config if it already exists
 833         my $cfg = PVE::CephTools::parse_ceph_config();
 834
 835         if (!$cfg->{global}) {
 836
 837             my $fsid;
 838             my $uuid;
 839
 840             UUID::generate($uuid);
 841             UUID::unparse($uuid, $fsid);
 842
 843             my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 844
 845             $cfg->{global} = {
 846                 'fsid' => $fsid,
 847                 'auth cluster required' => $auth,
 848                 'auth service required' => $auth,
 849                 'auth client required' => $auth,
 850                 'osd journal size' => $pve_osd_default_journal_size,
 851                 'osd pool default size' => $param->{size} // 3,
 852                 'osd pool default min size' => $param->{min_size} // 2,
 853                 'mon allow pool delete' => 'true',
 854             };
 855
 856             # this does not work for default pools
 857             #'osd pool default pg num' => $pg_num,
 858             #'osd pool default pgp num' => $pg_num,
 859         }
 860
 861         $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 862         $cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring';
 863
 864         if ($param->{pg_bits}) {
 865             $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 866             $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 867         }
 868
 869         if ($param->{network}) {
 870             $cfg->{global}->{'public network'} = $param->{network};
 871             $cfg->{global}->{'cluster network'} = $param->{network};
 872         }
 873
 874         PVE::CephTools::write_ceph_config($cfg);
 875
 876         PVE::CephTools::setup_pve_symlinks();
 877
 878         return undef;
 879     }});
 880
 881 my $find_node_ip = sub {
 882     my ($cidr) = @_;
 883
 884     my $net = Net::IP->new($cidr) || die Net::IP::Error() . "\n";
 885     my $id = $net->version == 6 ? 'address6' : 'address';
 886
 887     my $config = PVE::INotify::read_file('interfaces');
 888     my $ifaces = $config->{ifaces};
 889
 890     foreach my $iface (keys %$ifaces) {
 891         my $d = $ifaces->{$iface};
 892         next if !$d->{$id};
 893         my $a = Net::IP->new($d->{$id});
 894         next if !$a;
 895         return $d->{$id} if $net->overlaps($a);
 896     }
 897
 898     die "unable to find local address within network '$cidr'\n";
 899 };
 900
 901 my $create_mgr = sub {
 902     my ($rados, $id) = @_;
 903
 904     my $clustername = PVE::CephTools::get_config('ccname');
 905     my $mgrdir = "/var/lib/ceph/mgr/$clustername-$id";
 906     my $mgrkeyring = "$mgrdir/keyring";
 907     my $mgrname = "mgr.$id";
 908
 909     die "ceph manager directory '$mgrdir' already exists\n"
 910         if -d $mgrdir;
 911
 912     print "creating manager directory '$mgrdir'\n";
 913     mkdir $mgrdir;
 914     print "creating keys for '$mgrname'\n";
 915     my $output = $rados->mon_command({ prefix => 'auth get-or-create',
 916                                        entity => $mgrname,
 917                                        caps => [
 918                                            mon => 'allow profile mgr',
 919                                            osd => 'allow *',
 920                                            mds => 'allow *',
 921                                        ],
 922                                        format => 'plain'});
 923     PVE::Tools::file_set_contents($mgrkeyring, $output);
 924
 925     print "setting owner for directory\n";
 926     run_command(["chown", 'ceph:ceph', '-R', $mgrdir]);
 927
 928     print "enabling service 'ceph-mgr\@$id.service'\n";
 929     PVE::CephTools::ceph_service_cmd('enable', $mgrname);
 930     print "starting service 'ceph-mgr\@$id.service'\n";
 931     PVE::CephTools::ceph_service_cmd('start', $mgrname);
 932 };
 933
 934 my $destroy_mgr = sub {
 935     my ($mgrid) = @_;
 936
 937     my $clustername = PVE::CephTools::get_config('ccname');
 938     my $mgrname = "mgr.$mgrid";
 939     my $mgrdir = "/var/lib/ceph/mgr/$clustername-$mgrid";
 940
 941     die "ceph manager directory '$mgrdir' not found\n"
 942         if ! -d $mgrdir;
 943
 944     print "disabling service 'ceph-mgr\@$mgrid.service'\n";
 945     PVE::CephTools::ceph_service_cmd('disable', $mgrname);
 946     print "stopping service 'ceph-mgr\@$mgrid.service'\n";
 947     PVE::CephTools::ceph_service_cmd('stop', $mgrname);
 948
 949     print "removing manager directory '$mgrdir'\n";
 950     File::Path::remove_tree($mgrdir);
 951 };
 952
 953 __PACKAGE__->register_method ({
 954     name => 'createmon',
 955     path => 'mon',
 956     method => 'POST',
 957     description => "Create Ceph Monitor and Manager",
 958     proxyto => 'node',
 959     protected => 1,
 960     permissions => {
 961         check => ['perm', '/', [ 'Sys.Modify' ]],
 962     },
 963     parameters => {
 964         additionalProperties => 0,
 965         properties => {
 966             node => get_standard_option('pve-node'),
 967             id => {
 968                 type => 'string',
 969                 optional => 1,
 970                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
 971                 description => "The ID for the monitor, when omitted the same as the nodename",
 972             },
 973             'exclude-manager' => {
 974                 type => 'boolean',
 975                 optional => 1,
 976                 default => 0,
 977                 description => "When set, only a monitor will be created.",
 978             },
 979         },
 980     },
 981     returns => { type => 'string' },
 982     code => sub {
 983         my ($param) = @_;
 984
 985         PVE::CephTools::check_ceph_inited();
 986
 987         PVE::CephTools::setup_pve_symlinks();
 988
 989         my $rpcenv = PVE::RPCEnvironment::get();
 990
 991         my $authuser = $rpcenv->get_user();
 992
 993         my $cfg = PVE::CephTools::parse_ceph_config();
 994
 995         my $moncount = 0;
 996
 997         my $monaddrhash = {};
 998
 999         my $systemd_managed = PVE::CephTools::systemd_managed();
1000
1001         foreach my $section (keys %$cfg) {
1002             next if $section eq 'global';
1003             my $d = $cfg->{$section};
1004             if ($section =~ m/^mon\./) {
1005                 $moncount++;
1006                 if ($d->{'mon addr'}) {
1007                     $monaddrhash->{$d->{'mon addr'}} = $section;
1008                 }
1009             }
1010         }
1011
1012         my $monid = $param->{id} // $param->{node};
1013
1014         my $monsection = "mon.$monid";
1015         my $ip;
1016         if (my $pubnet = $cfg->{global}->{'public network'}) {
1017             $ip = &$find_node_ip($pubnet);
1018         } else {
1019             $ip = PVE::Cluster::remote_node_ip($param->{node});
1020         }
1021
1022         my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]:6789" : "$ip:6789";
1023         my $monname = $param->{node};
1024
1025         die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
1026         die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n"
1027             if $monaddrhash->{$monaddr};
1028
1029         my $worker = sub  {
1030             my $upid = shift;
1031
1032             my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1033
1034             if (! -f $pve_ckeyring_path) {
1035                 run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
1036                             "--gen-key -n client.admin");
1037             }
1038
1039             my $pve_mon_key_path = PVE::CephTools::get_config('pve_mon_key_path');
1040             if (! -f $pve_mon_key_path) {
1041                 run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
1042                 run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " .
1043                             "--cap mds 'allow' " .
1044                             "--cap osd 'allow *' " .
1045                             "--cap mgr 'allow *' " .
1046                             "--cap mon 'allow *'");
1047                 run_command("cp $pve_mon_key_path.tmp /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
1048                 run_command("chown ceph:ceph /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
1049                 run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
1050                 run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
1051             }
1052
1053             my $ccname = PVE::CephTools::get_config('ccname');
1054
1055             my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
1056             -d $mondir && die "monitor filesystem '$mondir' already exist\n";
1057
1058             my $monmap = "/tmp/monmap";
1059
1060             eval {
1061                 mkdir $mondir;
1062
1063                 run_command("chown ceph:ceph $mondir") if $systemd_managed;
1064
1065                 if ($moncount > 0) {
1066                     my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1067                     my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
1068                     PVE::Tools::file_set_contents($monmap, $mapdata);
1069                 } else {
1070                     run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
1071                 }
1072
1073                 run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
1074                 run_command("chown ceph:ceph -R $mondir") if $systemd_managed;
1075             };
1076             my $err = $@;
1077             unlink $monmap;
1078             if ($err) {
1079                 File::Path::remove_tree($mondir);
1080                 die $err;
1081             }
1082
1083             $cfg->{$monsection} = {
1084                 'host' => $monname,
1085                 'mon addr' => $monaddr,
1086             };
1087
1088             PVE::CephTools::write_ceph_config($cfg);
1089
1090             my $create_keys_pid = fork();
1091             if (!defined($create_keys_pid)) {
1092                 die "Could not spawn ceph-create-keys to create bootstrap keys\n";
1093             } elsif ($create_keys_pid == 0) {
1094                 exit PVE::Tools::run_command(['ceph-create-keys', '-i', $monid]);
1095             } else {
1096                 PVE::CephTools::ceph_service_cmd('start', $monsection);
1097
1098                 if ($systemd_managed) {
1099                     #to ensure we have the correct startup order.
1100                     eval { PVE::Tools::run_command(['/bin/systemctl', 'enable', "ceph-mon\@${monid}.service"]); };
1101                     warn "Enable ceph-mon\@${monid}.service manually"if $@;
1102                 }
1103                 waitpid($create_keys_pid, 0);
1104             }
1105
1106             # create manager
1107             if (!$param->{'exclude-manager'}) {
1108                 my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1109                 $create_mgr->($rados, $monid);
1110             }
1111         };
1112
1113         return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
1114     }});
1115
1116 __PACKAGE__->register_method ({
1117     name => 'destroymon',
1118     path => 'mon/{monid}',
1119     method => 'DELETE',
1120     description => "Destroy Ceph Monitor and Manager.",
1121     proxyto => 'node',
1122     protected => 1,
1123     permissions => {
1124         check => ['perm', '/', [ 'Sys.Modify' ]],
1125     },
1126     parameters => {
1127         additionalProperties => 0,
1128         properties => {
1129             node => get_standard_option('pve-node'),
1130             monid => {
1131                 description => 'Monitor ID',
1132                 type => 'string',
1133                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1134             },
1135             'exclude-manager' => {
1136                 type => 'boolean',
1137                 default => 0,
1138                 optional => 1,
1139                 description => "When set, removes only the monitor, not the manager"
1140             }
1141         },
1142     },
1143     returns => { type => 'string' },
1144     code => sub {
1145         my ($param) = @_;
1146
1147         my $rpcenv = PVE::RPCEnvironment::get();
1148
1149         my $authuser = $rpcenv->get_user();
1150
1151         PVE::CephTools::check_ceph_inited();
1152
1153         my $cfg = PVE::CephTools::parse_ceph_config();
1154
1155         my $monid = $param->{monid};
1156         my $monsection = "mon.$monid";
1157
1158         my $rados = PVE::RADOS->new();
1159         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
1160         my $monlist = $monstat->{monmap}->{mons};
1161
1162         die "no such monitor id '$monid'\n"
1163             if !defined($cfg->{$monsection});
1164
1165         my $ccname = PVE::CephTools::get_config('ccname');
1166
1167         my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
1168         -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n";
1169
1170         die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
1171
1172         my $worker = sub {
1173             my $upid = shift;
1174
1175             # reopen with longer timeout
1176             $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1177
1178             $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });
1179
1180             eval { PVE::CephTools::ceph_service_cmd('stop', $monsection); };
1181             warn $@ if $@;
1182
1183             delete $cfg->{$monsection};
1184             PVE::CephTools::write_ceph_config($cfg);
1185             File::Path::remove_tree($mondir);
1186
1187             # remove manager
1188             if (!$param->{'exclude-manager'}) {
1189                 eval { $destroy_mgr->($monid); };
1190                 warn $@ if $@;
1191             }
1192         };
1193
1194         return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
1195     }});
1196
1197 __PACKAGE__->register_method ({
1198     name => 'createmgr',
1199     path => 'mgr',
1200     method => 'POST',
1201     description => "Create Ceph Manager",
1202     proxyto => 'node',
1203     protected => 1,
1204     permissions => {
1205         check => ['perm', '/', [ 'Sys.Modify' ]],
1206     },
1207     parameters => {
1208         additionalProperties => 0,
1209         properties => {
1210             node => get_standard_option('pve-node'),
1211             id => {
1212                 type => 'string',
1213                 optional => 1,
1214                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1215                 description => "The ID for the manager, when omitted the same as the nodename",
1216             },
1217         },
1218     },
1219     returns => { type => 'string' },
1220     code => sub {
1221         my ($param) = @_;
1222
1223         PVE::CephTools::check_ceph_inited();
1224
1225         my $rpcenv = PVE::RPCEnvironment::get();
1226
1227         my $authuser = $rpcenv->get_user();
1228
1229         my $mgrid = $param->{id} // $param->{node};
1230
1231         my $worker = sub  {
1232             my $upid = shift;
1233
1234             my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1235
1236             $create_mgr->($rados, $mgrid);
1237         };
1238
1239         return $rpcenv->fork_worker('cephcreatemgr', "mgr.$mgrid", $authuser, $worker);
1240     }});
1241
1242 __PACKAGE__->register_method ({
1243     name => 'destroymgr',
1244     path => 'mgr/{id}',
1245     method => 'DELETE',
1246     description => "Destroy Ceph Manager.",
1247     proxyto => 'node',
1248     protected => 1,
1249     permissions => {
1250         check => ['perm', '/', [ 'Sys.Modify' ]],
1251     },
1252     parameters => {
1253         additionalProperties => 0,
1254         properties => {
1255             node => get_standard_option('pve-node'),
1256             id => {
1257                 description => 'The ID of the manager',
1258                 type => 'string',
1259                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1260             },
1261         },
1262     },
1263     returns => { type => 'string' },
1264     code => sub {
1265         my ($param) = @_;
1266
1267         my $rpcenv = PVE::RPCEnvironment::get();
1268
1269         my $authuser = $rpcenv->get_user();
1270
1271         PVE::CephTools::check_ceph_inited();
1272
1273         my $mgrid = $param->{id};
1274
1275         my $worker = sub {
1276             my $upid = shift;
1277
1278             $destroy_mgr->($mgrid);
1279         };
1280
1281         return $rpcenv->fork_worker('cephdestroymgr', "mgr.$mgrid",  $authuser, $worker);
1282     }});
1283
1284 __PACKAGE__->register_method ({
1285     name => 'stop',
1286     path => 'stop',
1287     method => 'POST',
1288     description => "Stop ceph services.",
1289     proxyto => 'node',
1290     protected => 1,
1291     permissions => {
1292         check => ['perm', '/', [ 'Sys.Modify' ]],
1293     },
1294     parameters => {
1295         additionalProperties => 0,
1296         properties => {
1297             node => get_standard_option('pve-node'),
1298             service => {
1299                 description => 'Ceph service name.',
1300                 type => 'string',
1301                 optional => 1,
1302                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1303             },
1304         },
1305     },
1306     returns => { type => 'string' },
1307     code => sub {
1308         my ($param) = @_;
1309
1310         my $rpcenv = PVE::RPCEnvironment::get();
1311
1312         my $authuser = $rpcenv->get_user();
1313
1314         PVE::CephTools::check_ceph_inited();
1315
1316         my $cfg = PVE::CephTools::parse_ceph_config();
1317         scalar(keys %$cfg) || die "no configuration\n";
1318
1319         my $worker = sub {
1320             my $upid = shift;
1321
1322             my $cmd = ['stop'];
1323             if ($param->{service}) {
1324                 push @$cmd, $param->{service};
1325             }
1326
1327             PVE::CephTools::ceph_service_cmd(@$cmd);
1328         };
1329
1330         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
1331                                     $authuser, $worker);
1332     }});
1333
1334 __PACKAGE__->register_method ({
1335     name => 'start',
1336     path => 'start',
1337     method => 'POST',
1338     description => "Start ceph services.",
1339     proxyto => 'node',
1340     protected => 1,
1341     permissions => {
1342         check => ['perm', '/', [ 'Sys.Modify' ]],
1343     },
1344     parameters => {
1345         additionalProperties => 0,
1346         properties => {
1347             node => get_standard_option('pve-node'),
1348             service => {
1349                 description => 'Ceph service name.',
1350                 type => 'string',
1351                 optional => 1,
1352                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1353             },
1354         },
1355     },
1356     returns => { type => 'string' },
1357     code => sub {
1358         my ($param) = @_;
1359
1360         my $rpcenv = PVE::RPCEnvironment::get();
1361
1362         my $authuser = $rpcenv->get_user();
1363
1364         PVE::CephTools::check_ceph_inited();
1365
1366         my $cfg = PVE::CephTools::parse_ceph_config();
1367         scalar(keys %$cfg) || die "no configuration\n";
1368
1369         my $worker = sub {
1370             my $upid = shift;
1371
1372             my $cmd = ['start'];
1373             if ($param->{service}) {
1374                 push @$cmd, $param->{service};
1375             }
1376
1377             PVE::CephTools::ceph_service_cmd(@$cmd);
1378         };
1379
1380         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
1381                                     $authuser, $worker);
1382     }});
1383
1384 __PACKAGE__->register_method ({
1385     name => 'status',
1386     path => 'status',
1387     method => 'GET',
1388     description => "Get ceph status.",
1389     proxyto => 'node',
1390     protected => 1,
1391     permissions => {
1392         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1393     },
1394     parameters => {
1395         additionalProperties => 0,
1396         properties => {
1397             node => get_standard_option('pve-node'),
1398         },
1399     },
1400     returns => { type => 'object' },
1401     code => sub {
1402         my ($param) = @_;
1403
1404         PVE::CephTools::check_ceph_enabled();
1405
1406         my $rados = PVE::RADOS->new();
1407         my $status = $rados->mon_command({ prefix => 'status' });
1408         $status->{health} = $rados->mon_command({ prefix => 'health', detail => 'detail' });
1409         return $status;
1410     }});
1411
1412 __PACKAGE__->register_method ({
1413     name => 'lspools',
1414     path => 'pools',
1415     method => 'GET',
1416     description => "List all pools.",
1417     proxyto => 'node',
1418     protected => 1,
1419     permissions => {
1420         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1421     },
1422     parameters => {
1423         additionalProperties => 0,
1424         properties => {
1425             node => get_standard_option('pve-node'),
1426         },
1427     },
1428     returns => {
1429         type => 'array',
1430         items => {
1431             type => "object",
1432             properties => {
1433                 pool => { type => 'integer' },
1434                 pool_name => { type => 'string' },
1435                 size => { type => 'integer' },
1436             },
1437         },
1438         links => [ { rel => 'child', href => "{pool_name}" } ],
1439     },
1440     code => sub {
1441         my ($param) = @_;
1442
1443         PVE::CephTools::check_ceph_inited();
1444
1445         my $rados = PVE::RADOS->new();
1446
1447         my $stats = {};
1448         my $res = $rados->mon_command({ prefix => 'df' });
1449         my $total = $res->{stats}->{total_avail_bytes} || 0;
1450
1451         foreach my $d (@{$res->{pools}}) {
1452             next if !$d->{stats};
1453             next if !defined($d->{id});
1454             $stats->{$d->{id}} = $d->{stats};
1455         }
1456
1457         $res = $rados->mon_command({ prefix => 'osd dump' });
1458         my $rulestmp = $rados->mon_command({ prefix => 'osd crush rule dump'});
1459
1460         my $rules = {};
1461         for my $rule (@$rulestmp) {
1462             $rules->{$rule->{rule_id}} = $rule->{rule_name};
1463         }
1464
1465         my $data = [];
1466         foreach my $e (@{$res->{pools}}) {
1467             my $d = {};
1468             foreach my $attr (qw(pool pool_name size min_size pg_num crush_rule)) {
1469                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
1470             }
1471
1472             if (defined($d->{crush_rule}) && defined($rules->{$d->{crush_rule}})) {
1473                 $d->{crush_rule_name} = $rules->{$d->{crush_rule}};
1474             }
1475
1476             if (my $s = $stats->{$d->{pool}}) {
1477                 $d->{bytes_used} = $s->{bytes_used};
1478                 $d->{percent_used} = ($s->{bytes_used} / $total)*100
1479                     if $s->{max_avail} && $total;
1480             }
1481             push @$data, $d;
1482         }
1483
1484
1485         return $data;
1486     }});
1487
1488 __PACKAGE__->register_method ({
1489     name => 'createpool',
1490     path => 'pools',
1491     method => 'POST',
1492     description => "Create POOL",
1493     proxyto => 'node',
1494     protected => 1,
1495     permissions => {
1496         check => ['perm', '/', [ 'Sys.Modify' ]],
1497     },
1498     parameters => {
1499         additionalProperties => 0,
1500         properties => {
1501             node => get_standard_option('pve-node'),
1502             name => {
1503                 description => "The name of the pool. It must be unique.",
1504                 type => 'string',
1505             },
1506             size => {
1507                 description => 'Number of replicas per object',
1508                 type => 'integer',
1509                 default => 2,
1510                 optional => 1,
1511                 minimum => 1,
1512                 maximum => 7,
1513             },
1514             min_size => {
1515                 description => 'Minimum number of replicas per object',
1516                 type => 'integer',
1517                 default => 1,
1518                 optional => 1,
1519                 minimum => 1,
1520                 maximum => 7,
1521             },
1522             pg_num => {
1523                 description => "Number of placement groups.",
1524                 type => 'integer',
1525                 default => 64,
1526                 optional => 1,
1527                 minimum => 8,
1528                 maximum => 32768,
1529             },
1530             crush_rule => {
1531                 description => "The rule to use for mapping object placement in the cluster.",
1532                 type => 'string',
1533                 optional => 1,
1534             },
1535         },
1536     },
1537     returns => { type => 'null' },
1538     code => sub {
1539         my ($param) = @_;
1540
1541         PVE::CephTools::check_ceph_inited();
1542
1543         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1544
1545         die "not fully configured - missing '$pve_ckeyring_path'\n"
1546             if ! -f $pve_ckeyring_path;
1547
1548         my $pg_num = $param->{pg_num} || 64;
1549         my $size = $param->{size} || 2;
1550         my $min_size = $param->{min_size} || 1;
1551         my $rados = PVE::RADOS->new();
1552
1553         $rados->mon_command({
1554             prefix => "osd pool create",
1555             pool => $param->{name},
1556             pg_num => int($pg_num),
1557             format => 'plain',
1558         });
1559
1560         $rados->mon_command({
1561             prefix => "osd pool set",
1562             pool => $param->{name},
1563             var => 'min_size',
1564             val => $min_size,
1565             format => 'plain',
1566         });
1567
1568         $rados->mon_command({
1569             prefix => "osd pool set",
1570             pool => $param->{name},
1571             var => 'size',
1572             val => $size,
1573             format => 'plain',
1574         });
1575
1576         if (defined($param->{crush_rule})) {
1577             $rados->mon_command({
1578                 prefix => "osd pool set",
1579                 pool => $param->{name},
1580                 var => 'crush_rule',
1581                 val => $param->{crush_rule},
1582                 format => 'plain',
1583             });
1584         }
1585
1586         return undef;
1587     }});
1588
1589 __PACKAGE__->register_method ({
1590     name => 'get_flags',
1591     path => 'flags',
1592     method => 'GET',
1593     description => "get all set ceph flags",
1594     proxyto => 'node',
1595     protected => 1,
1596     permissions => {
1597         check => ['perm', '/', [ 'Sys.Audit' ]],
1598     },
1599     parameters => {
1600         additionalProperties => 0,
1601         properties => {
1602             node => get_standard_option('pve-node'),
1603         },
1604     },
1605     returns => { type => 'string' },
1606     code => sub {
1607         my ($param) = @_;
1608
1609         PVE::CephTools::check_ceph_inited();
1610
1611         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1612
1613         die "not fully configured - missing '$pve_ckeyring_path'\n"
1614             if ! -f $pve_ckeyring_path;
1615
1616         my $rados = PVE::RADOS->new();
1617
1618         my $stat = $rados->mon_command({ prefix => 'osd dump' });
1619
1620         return $stat->{flags} // '';
1621     }});
1622
1623 __PACKAGE__->register_method ({
1624     name => 'set_flag',
1625     path => 'flags/{flag}',
1626     method => 'POST',
1627     description => "Set a ceph flag",
1628     proxyto => 'node',
1629     protected => 1,
1630     permissions => {
1631         check => ['perm', '/', [ 'Sys.Modify' ]],
1632     },
1633     parameters => {
1634         additionalProperties => 0,
1635         properties => {
1636             node => get_standard_option('pve-node'),
1637             flag => {
1638                 description => 'The ceph flag to set/unset',
1639                 type => 'string',
1640                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1641             },
1642         },
1643     },
1644     returns => { type => 'null' },
1645     code => sub {
1646         my ($param) = @_;
1647
1648         PVE::CephTools::check_ceph_inited();
1649
1650         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1651
1652         die "not fully configured - missing '$pve_ckeyring_path'\n"
1653             if ! -f $pve_ckeyring_path;
1654
1655         my $set = $param->{set} // !$param->{unset};
1656         my $rados = PVE::RADOS->new();
1657
1658         $rados->mon_command({
1659             prefix => "osd set",
1660             key => $param->{flag},
1661         });
1662
1663         return undef;
1664     }});
1665
1666 __PACKAGE__->register_method ({
1667     name => 'unset_flag',
1668     path => 'flags/{flag}',
1669     method => 'DELETE',
1670     description => "Unset a ceph flag",
1671     proxyto => 'node',
1672     protected => 1,
1673     permissions => {
1674         check => ['perm', '/', [ 'Sys.Modify' ]],
1675     },
1676     parameters => {
1677         additionalProperties => 0,
1678         properties => {
1679             node => get_standard_option('pve-node'),
1680             flag => {
1681                 description => 'The ceph flag to set/unset',
1682                 type => 'string',
1683                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1684             },
1685         },
1686     },
1687     returns => { type => 'null' },
1688     code => sub {
1689         my ($param) = @_;
1690
1691         PVE::CephTools::check_ceph_inited();
1692
1693         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1694
1695         die "not fully configured - missing '$pve_ckeyring_path'\n"
1696             if ! -f $pve_ckeyring_path;
1697
1698         my $set = $param->{set} // !$param->{unset};
1699         my $rados = PVE::RADOS->new();
1700
1701         $rados->mon_command({
1702             prefix => "osd unset",
1703             key => $param->{flag},
1704         });
1705
1706         return undef;
1707     }});
1708
1709 __PACKAGE__->register_method ({
1710     name => 'destroypool',
1711     path => 'pools/{name}',
1712     method => 'DELETE',
1713     description => "Destroy pool",
1714     proxyto => 'node',
1715     protected => 1,
1716     permissions => {
1717         check => ['perm', '/', [ 'Sys.Modify' ]],
1718     },
1719     parameters => {
1720         additionalProperties => 0,
1721         properties => {
1722             node => get_standard_option('pve-node'),
1723             name => {
1724                 description => "The name of the pool. It must be unique.",
1725                 type => 'string',
1726             },
1727             force => {
1728                 description => "If true, destroys pool even if in use",
1729                 type => 'boolean',
1730                 optional => 1,
1731                 default => 0,
1732             }
1733         },
1734     },
1735     returns => { type => 'null' },
1736     code => sub {
1737         my ($param) = @_;
1738
1739         PVE::CephTools::check_ceph_inited();
1740
1741         # if not forced, destroy ceph pool only when no
1742         # vm disks are on it anymore
1743         if (!$param->{force}) {
1744             my $storagecfg = PVE::Storage::config();
1745             foreach my $storageid (keys %{$storagecfg->{ids}}) {
1746                 my $storage = $storagecfg->{ids}->{$storageid};
1747                 next if $storage->{type} ne 'rbd';
1748                 next if $storage->{pool} ne $param->{name};
1749
1750                 # check if any vm disks are on the pool
1751                 my $res = PVE::Storage::vdisk_list($storagecfg, $storageid);
1752                 die "ceph pool '$param->{name}' still in use by storage '$storageid'\n"
1753                     if @{$res->{$storageid}} != 0;
1754             }
1755         }
1756
1757         my $rados = PVE::RADOS->new();
1758         # fixme: '--yes-i-really-really-mean-it'
1759         $rados->mon_command({
1760             prefix => "osd pool delete",
1761             pool => $param->{name},
1762             pool2 => $param->{name},
1763             sure => '--yes-i-really-really-mean-it',
1764             format => 'plain',
1765         });
1766
1767         return undef;
1768     }});
1769
1770
1771 __PACKAGE__->register_method ({
1772     name => 'crush',
1773     path => 'crush',
1774     method => 'GET',
1775     description => "Get OSD crush map",
1776     proxyto => 'node',
1777     protected => 1,
1778     permissions => {
1779         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1780     },
1781     parameters => {
1782         additionalProperties => 0,
1783         properties => {
1784             node => get_standard_option('pve-node'),
1785         },
1786     },
1787     returns => { type => 'string' },
1788     code => sub {
1789         my ($param) = @_;
1790
1791         PVE::CephTools::check_ceph_inited();
1792
1793         # this produces JSON (difficult to read for the user)
1794         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
1795
1796         my $txt = '';
1797
1798         my $mapfile = "/var/tmp/ceph-crush.map.$$";
1799         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
1800
1801         my $rados = PVE::RADOS->new();
1802
1803         eval {
1804             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
1805             PVE::Tools::file_set_contents($mapfile, $bindata);
1806             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
1807             $txt = PVE::Tools::file_get_contents($mapdata);
1808         };
1809         my $err = $@;
1810
1811         unlink $mapfile;
1812         unlink $mapdata;
1813
1814         die $err if $err;
1815
1816         return $txt;
1817     }});
1818
1819 __PACKAGE__->register_method({
1820     name => 'log',
1821     path => 'log',
1822     method => 'GET',
1823     description => "Read ceph log",
1824     proxyto => 'node',
1825     permissions => {
1826         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
1827     },
1828     protected => 1,
1829     parameters => {
1830         additionalProperties => 0,
1831         properties => {
1832             node => get_standard_option('pve-node'),
1833             start => {
1834                 type => 'integer',
1835                 minimum => 0,
1836                 optional => 1,
1837             },
1838             limit => {
1839                 type => 'integer',
1840                 minimum => 0,
1841                 optional => 1,
1842             },
1843         },
1844     },
1845     returns => {
1846         type => 'array',
1847         items => {
1848             type => "object",
1849             properties => {
1850                 n => {
1851                   description=>  "Line number",
1852                   type=> 'integer',
1853                 },
1854                 t => {
1855                   description=>  "Line text",
1856                   type => 'string',
1857                 }
1858             }
1859         }
1860     },
1861     code => sub {
1862         my ($param) = @_;
1863
1864         my $rpcenv = PVE::RPCEnvironment::get();
1865         my $user = $rpcenv->get_user();
1866         my $node = $param->{node};
1867
1868         my $logfile = "/var/log/ceph/ceph.log";
1869         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
1870
1871         $rpcenv->set_result_attrib('total', $count);
1872
1873         return $lines;
1874     }});
1875
1876 __PACKAGE__->register_method ({
1877     name => 'rules',
1878     path => 'rules',
1879     method => 'GET',
1880     description => "List ceph rules.",
1881     proxyto => 'node',
1882     protected => 1,
1883     permissions => {
1884         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1885     },
1886     parameters => {
1887         additionalProperties => 0,
1888         properties => {
1889             node => get_standard_option('pve-node'),
1890         },
1891     },
1892     returns => {
1893         type => 'array',
1894         items => {
1895             type => "object",
1896             properties => {},
1897         },
1898         links => [ { rel => 'child', href => "{name}" } ],
1899     },
1900     code => sub {
1901         my ($param) = @_;
1902
1903         PVE::CephTools::check_ceph_inited();
1904
1905         my $rados = PVE::RADOS->new();
1906
1907         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
1908
1909         my $res = [];
1910
1911         foreach my $rule (@$rules) {
1912             push @$res, { name => $rule };
1913         }
1914
1915         return $res;
1916     }});