PVE/API2/Ceph.pm

   1 package PVE::API2::CephOSD;
   2
   3 use strict;
   4 use warnings;
   5 use Cwd qw(abs_path);
   6 use Net::IP;
   7
   8 use PVE::SafeSyslog;
   9 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
  10 use PVE::Exception qw(raise raise_param_exc);
  11 use PVE::INotify;
  12 use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
  13 use PVE::AccessControl;
  14 use PVE::Storage;
  15 use PVE::RESTHandler;
  16 use PVE::RPCEnvironment;
  17 use PVE::JSONSchema qw(get_standard_option);
  18 use PVE::RADOS;
  19 use PVE::CephTools;
  20 use PVE::Diskmanage;
  21
  22 use base qw(PVE::RESTHandler);
  23
  24 use Data::Dumper; # fixme: remove
  25
  26 my $get_osd_status = sub {
  27     my ($rados, $osdid) = @_;
  28
  29     my $stat = $rados->mon_command({ prefix => 'osd dump' });
  30
  31     my $osdlist = $stat->{osds} || [];
  32
  33     my $flags = $stat->{flags} || undef;
  34
  35     my $osdstat;
  36     foreach my $d (@$osdlist) {
  37         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  38     }
  39     if (defined($osdid)) {
  40         die "no such OSD '$osdid'\n" if !$osdstat->{$osdid};
  41         return $osdstat->{$osdid};
  42     }
  43
  44     return wantarray? ($osdstat, $flags):$osdstat;
  45 };
  46
  47 my $get_osd_usage = sub {
  48     my ($rados) = @_;
  49
  50     my $osdlist = $rados->mon_command({ prefix => 'pg dump',
  51                                         dumpcontents => [ 'osds' ]}) || [];
  52
  53     my $osdstat;
  54     foreach my $d (@$osdlist) {
  55         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  56     }
  57
  58     return $osdstat;
  59 };
  60
  61 __PACKAGE__->register_method ({
  62     name => 'index',
  63     path => '',
  64     method => 'GET',
  65     description => "Get Ceph osd list/tree.",
  66     proxyto => 'node',
  67     protected => 1,
  68     permissions => {
  69         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  70     },
  71     parameters => {
  72         additionalProperties => 0,
  73         properties => {
  74             node => get_standard_option('pve-node'),
  75         },
  76     },
  77     # fixme: return a list instead of extjs tree format ?
  78     returns => {
  79         type => "object",
  80     },
  81     code => sub {
  82         my ($param) = @_;
  83
  84         PVE::CephTools::check_ceph_inited();
  85
  86         my $rados = PVE::RADOS->new();
  87         my $res = $rados->mon_command({ prefix => 'osd tree' });
  88
  89         die "no tree nodes found\n" if !($res && $res->{nodes});
  90
  91         my ($osdhash, $flags) = &$get_osd_status($rados);
  92
  93         my $usagehash = &$get_osd_usage($rados);
  94
  95         my $nodes = {};
  96         my $newnodes = {};
  97         foreach my $e (@{$res->{nodes}}) {
  98             $nodes->{$e->{id}} = $e;
  99
 100             my $new = {
 101                 id => $e->{id},
 102                 name => $e->{name},
 103                 type => $e->{type}
 104             };
 105
 106             foreach my $opt (qw(status crush_weight reweight)) {
 107                 $new->{$opt} = $e->{$opt} if defined($e->{$opt});
 108             }
 109
 110             if (my $stat = $osdhash->{$e->{id}}) {
 111                 $new->{in} = $stat->{in} if defined($stat->{in});
 112             }
 113
 114             if (my $stat = $usagehash->{$e->{id}}) {
 115                 $new->{total_space} = ($stat->{kb} || 1) * 1024;
 116                 $new->{bytes_used} = ($stat->{kb_used} || 0) * 1024;
 117                 $new->{percent_used} = ($new->{bytes_used}*100)/$new->{total_space};
 118                 if (my $d = $stat->{fs_perf_stat}) {
 119                     $new->{commit_latency_ms} = $d->{commit_latency_ms};
 120                     $new->{apply_latency_ms} = $d->{apply_latency_ms};
 121                 }
 122             }
 123
 124             $newnodes->{$e->{id}} = $new;
 125         }
 126
 127         foreach my $e (@{$res->{nodes}}) {
 128             my $new = $newnodes->{$e->{id}};
 129             if ($e->{children} && scalar(@{$e->{children}})) {
 130                 $new->{children} = [];
 131                 $new->{leaf} = 0;
 132                 foreach my $cid (@{$e->{children}}) {
 133                     $nodes->{$cid}->{parent} = $e->{id};
 134                     if ($nodes->{$cid}->{type} eq 'osd' &&
 135                         $e->{type} eq 'host') {
 136                         $newnodes->{$cid}->{host} = $e->{name};
 137                     }
 138                     push @{$new->{children}}, $newnodes->{$cid};
 139                 }
 140             } else {
 141                 $new->{leaf} = ($e->{id} >= 0) ? 1 : 0;
 142             }
 143         }
 144
 145         my $roots = [];
 146         foreach my $e (@{$res->{nodes}}) {
 147             if (!$nodes->{$e->{id}}->{parent}) {
 148                 push @$roots, $newnodes->{$e->{id}};
 149             }
 150         }
 151
 152         die "no root node\n" if !@$roots;
 153
 154         my $data = { root => { leaf =>  0, children => $roots } };
 155
 156         # we want this for the noout flag
 157         $data->{flags} = $flags if $flags;
 158
 159         return $data;
 160     }});
 161
 162 __PACKAGE__->register_method ({
 163     name => 'createosd',
 164     path => '',
 165     method => 'POST',
 166     description => "Create OSD",
 167     proxyto => 'node',
 168     protected => 1,
 169     parameters => {
 170         additionalProperties => 0,
 171         properties => {
 172             node => get_standard_option('pve-node'),
 173             dev => {
 174                 description => "Block device name.",
 175                 type => 'string',
 176             },
 177             journal_dev => {
 178                 description => "Block device name for journal.",
 179                 optional => 1,
 180                 type => 'string',
 181             },
 182             fstype => {
 183                 description => "File system type.",
 184                 type => 'string',
 185                 enum => ['xfs', 'ext4', 'btrfs'],
 186                 default => 'xfs',
 187                 optional => 1,
 188             },
 189         },
 190     },
 191     returns => { type => 'string' },
 192     code => sub {
 193         my ($param) = @_;
 194
 195         my $rpcenv = PVE::RPCEnvironment::get();
 196
 197         my $authuser = $rpcenv->get_user();
 198
 199         PVE::CephTools::check_ceph_inited();
 200
 201         PVE::CephTools::setup_pve_symlinks();
 202
 203         my $journal_dev;
 204
 205         if ($param->{journal_dev} && ($param->{journal_dev} ne $param->{dev})) {
 206             $journal_dev = PVE::Diskmanage::verify_blockdev_path($param->{journal_dev});
 207         }
 208
 209         $param->{dev} = PVE::Diskmanage::verify_blockdev_path($param->{dev});
 210
 211         my $devname = $param->{dev};
 212         $devname =~ s|/dev/||;
 213
 214         my $disklist = PVE::Diskmanage::get_disks($devname, 1);
 215
 216         my $diskinfo = $disklist->{$devname};
 217         die "unable to get device info for '$devname'\n"
 218             if !$diskinfo;
 219
 220         die "device '$param->{dev}' is in use\n"
 221             if $diskinfo->{used};
 222
 223         my $devpath = $diskinfo->{devpath};
 224         my $rados = PVE::RADOS->new();
 225         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 226         die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid};
 227
 228         my $fsid = $monstat->{monmap}->{fsid};
 229         $fsid = $1 if $fsid =~ m/^([0-9a-f\-]+)$/;
 230
 231         my $ceph_bootstrap_osd_keyring = PVE::CephTools::get_config('ceph_bootstrap_osd_keyring');
 232
 233         if (! -f $ceph_bootstrap_osd_keyring) {
 234             my $bindata = $rados->mon_command({ prefix => 'auth get', entity => 'client.bootstrap-osd', format => 'plain' });
 235             PVE::Tools::file_set_contents($ceph_bootstrap_osd_keyring, $bindata);
 236         };
 237
 238         my $worker = sub {
 239             my $upid = shift;
 240
 241             my $fstype = $param->{fstype} || 'xfs';
 242
 243             print "create OSD on $devpath ($fstype)\n";
 244
 245             my $ccname = PVE::CephTools::get_config('ccname');
 246
 247             my $cmd = ['ceph-disk', 'prepare', '--zap-disk', '--fs-type', $fstype,
 248                        '--cluster', $ccname, '--cluster-uuid', $fsid ];
 249
 250             if ($journal_dev) {
 251                 print "using device '$journal_dev' for journal\n";
 252                 push @$cmd, '--journal-dev', $devpath, $journal_dev;
 253             } else {
 254                 push @$cmd, $devpath;
 255             }
 256
 257             run_command($cmd);
 258         };
 259
 260         return $rpcenv->fork_worker('cephcreateosd', $devname,  $authuser, $worker);
 261     }});
 262
 263 __PACKAGE__->register_method ({
 264     name => 'destroyosd',
 265     path => '{osdid}',
 266     method => 'DELETE',
 267     description => "Destroy OSD",
 268     proxyto => 'node',
 269     protected => 1,
 270     parameters => {
 271         additionalProperties => 0,
 272         properties => {
 273             node => get_standard_option('pve-node'),
 274             osdid => {
 275                 description => 'OSD ID',
 276                 type => 'integer',
 277             },
 278             cleanup => {
 279                 description => "If set, we remove partition table entries.",
 280                 type => 'boolean',
 281                 optional => 1,
 282                 default => 0,
 283             },
 284         },
 285     },
 286     returns => { type => 'string' },
 287     code => sub {
 288         my ($param) = @_;
 289
 290         my $rpcenv = PVE::RPCEnvironment::get();
 291
 292         my $authuser = $rpcenv->get_user();
 293
 294         PVE::CephTools::check_ceph_inited();
 295
 296         my $osdid = $param->{osdid};
 297
 298         my $rados = PVE::RADOS->new();
 299         my $osdstat = &$get_osd_status($rados, $osdid);
 300
 301         die "osd is in use (in == 1)\n" if $osdstat->{in};
 302         #&$run_ceph_cmd(['osd', 'out', $osdid]);
 303
 304         die "osd is still runnung (up == 1)\n" if $osdstat->{up};
 305
 306         my $osdsection = "osd.$osdid";
 307
 308         my $worker = sub {
 309             my $upid = shift;
 310
 311             # reopen with longer timeout
 312             $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
 313
 314             print "destroy OSD $osdsection\n";
 315
 316             eval { PVE::CephTools::ceph_service_cmd('stop', $osdsection); };
 317             warn $@ if $@;
 318
 319             print "Remove $osdsection from the CRUSH map\n";
 320             $rados->mon_command({ prefix => "osd crush remove", name => $osdsection, format => 'plain' });
 321
 322             print "Remove the $osdsection authentication key.\n";
 323             $rados->mon_command({ prefix => "auth del", entity => $osdsection, format => 'plain' });
 324
 325             print "Remove OSD $osdsection\n";
 326             $rados->mon_command({ prefix => "osd rm", ids => [ $osdsection ], format => 'plain' });
 327
 328             # try to unmount from standard mount point
 329             my $mountpoint = "/var/lib/ceph/osd/ceph-$osdid";
 330
 331             my $remove_partition = sub {
 332                 my ($part) = @_;
 333
 334                 return if !$part || (! -b $part );
 335                 my $partnum = PVE::Diskmanage::get_partnum($part);
 336                 my $devpath = PVE::Diskmanage::get_blockdev($part);
 337
 338                 print "remove partition $part (disk '${devpath}', partnum $partnum)\n";
 339                 eval { run_command(['/sbin/sgdisk', '-d', $partnum, "${devpath}"]); };
 340                 warn $@ if $@;
 341             };
 342
 343             my $journal_part;
 344             my $data_part;
 345
 346             if ($param->{cleanup}) {
 347                 my $jpath = "$mountpoint/journal";
 348                 $journal_part = abs_path($jpath);
 349
 350                 if (my $fd = IO::File->new("/proc/mounts", "r")) {
 351                     while (defined(my $line = <$fd>)) {
 352                         my ($dev, $path, $fstype) = split(/\s+/, $line);
 353                         next if !($dev && $path && $fstype);
 354                         next if $dev !~ m|^/dev/|;
 355                         if ($path eq $mountpoint) {
 356                             $data_part = abs_path($dev);
 357                             last;
 358                         }
 359                     }
 360                     close($fd);
 361                 }
 362             }
 363
 364             print "Unmount OSD $osdsection from  $mountpoint\n";
 365             eval { run_command(['/bin/umount', $mountpoint]); };
 366             if (my $err = $@) {
 367                 warn $err;
 368             } elsif ($param->{cleanup}) {
 369                 #be aware of the ceph udev rules which can remount.
 370                 &$remove_partition($data_part);
 371                 &$remove_partition($journal_part);
 372             }
 373         };
 374
 375         return $rpcenv->fork_worker('cephdestroyosd', $osdsection,  $authuser, $worker);
 376     }});
 377
 378 __PACKAGE__->register_method ({
 379     name => 'in',
 380     path => '{osdid}/in',
 381     method => 'POST',
 382     description => "ceph osd in",
 383     proxyto => 'node',
 384     protected => 1,
 385     permissions => {
 386         check => ['perm', '/', [ 'Sys.Modify' ]],
 387     },
 388     parameters => {
 389         additionalProperties => 0,
 390         properties => {
 391             node => get_standard_option('pve-node'),
 392             osdid => {
 393                 description => 'OSD ID',
 394                 type => 'integer',
 395             },
 396         },
 397     },
 398     returns => { type => "null" },
 399     code => sub {
 400         my ($param) = @_;
 401
 402         PVE::CephTools::check_ceph_inited();
 403
 404         my $osdid = $param->{osdid};
 405
 406         my $rados = PVE::RADOS->new();
 407
 408         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 409
 410         my $osdsection = "osd.$osdid";
 411
 412         $rados->mon_command({ prefix => "osd in", ids => [ $osdsection ], format => 'plain' });
 413
 414         return undef;
 415     }});
 416
 417 __PACKAGE__->register_method ({
 418     name => 'out',
 419     path => '{osdid}/out',
 420     method => 'POST',
 421     description => "ceph osd out",
 422     proxyto => 'node',
 423     protected => 1,
 424     permissions => {
 425         check => ['perm', '/', [ 'Sys.Modify' ]],
 426     },
 427     parameters => {
 428         additionalProperties => 0,
 429         properties => {
 430             node => get_standard_option('pve-node'),
 431             osdid => {
 432                 description => 'OSD ID',
 433                 type => 'integer',
 434             },
 435         },
 436     },
 437     returns => { type => "null" },
 438     code => sub {
 439         my ($param) = @_;
 440
 441         PVE::CephTools::check_ceph_inited();
 442
 443         my $osdid = $param->{osdid};
 444
 445         my $rados = PVE::RADOS->new();
 446
 447         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 448
 449         my $osdsection = "osd.$osdid";
 450
 451         $rados->mon_command({ prefix => "osd out", ids => [ $osdsection ], format => 'plain' });
 452
 453         return undef;
 454     }});
 455
 456 package PVE::API2::Ceph;
 457
 458 use strict;
 459 use warnings;
 460 use File::Basename;
 461 use File::Path;
 462 use POSIX qw (LONG_MAX);
 463 use Cwd qw(abs_path);
 464 use IO::Dir;
 465 use UUID;
 466 use Net::IP;
 467
 468 use PVE::SafeSyslog;
 469 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
 470 use PVE::Exception qw(raise raise_param_exc);
 471 use PVE::INotify;
 472 use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
 473 use PVE::AccessControl;
 474 use PVE::Storage;
 475 use PVE::RESTHandler;
 476 use PVE::RPCEnvironment;
 477 use PVE::JSONSchema qw(get_standard_option);
 478 use JSON;
 479 use PVE::RADOS;
 480 use PVE::CephTools;
 481
 482 use base qw(PVE::RESTHandler);
 483
 484 use Data::Dumper; # fixme: remove
 485
 486 my $pve_osd_default_journal_size = 1024*5;
 487
 488 __PACKAGE__->register_method ({
 489     subclass => "PVE::API2::CephOSD",
 490     path => 'osd',
 491 });
 492
 493 __PACKAGE__->register_method ({
 494     name => 'index',
 495     path => '',
 496     method => 'GET',
 497     description => "Directory index.",
 498     permissions => { user => 'all' },
 499     permissions => {
 500         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 501     },
 502     parameters => {
 503         additionalProperties => 0,
 504         properties => {
 505             node => get_standard_option('pve-node'),
 506         },
 507     },
 508     returns => {
 509         type => 'array',
 510         items => {
 511             type => "object",
 512             properties => {},
 513         },
 514         links => [ { rel => 'child', href => "{name}" } ],
 515     },
 516     code => sub {
 517         my ($param) = @_;
 518
 519         my $result = [
 520             { name => 'init' },
 521             { name => 'mon' },
 522             { name => 'osd' },
 523             { name => 'pools' },
 524             { name => 'stop' },
 525             { name => 'start' },
 526             { name => 'status' },
 527             { name => 'crush' },
 528             { name => 'config' },
 529             { name => 'log' },
 530             { name => 'disks' },
 531             { name => 'flags' },
 532         ];
 533
 534         return $result;
 535     }});
 536
 537 __PACKAGE__->register_method ({
 538     name => 'disks',
 539     path => 'disks',
 540     method => 'GET',
 541     description => "List local disks.",
 542     proxyto => 'node',
 543     protected => 1,
 544     permissions => {
 545         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 546     },
 547     parameters => {
 548         additionalProperties => 0,
 549         properties => {
 550             node => get_standard_option('pve-node'),
 551             type => {
 552                 description => "Only list specific types of disks.",
 553                 type => 'string',
 554                 enum => ['unused', 'journal_disks'],
 555                 optional => 1,
 556             },
 557         },
 558     },
 559     returns => {
 560         type => 'array',
 561         items => {
 562             type => "object",
 563             properties => {
 564                 dev => { type => 'string' },
 565                 used => { type => 'string', optional => 1 },
 566                 gpt => { type => 'boolean' },
 567                 size => { type => 'integer' },
 568                 osdid => { type => 'integer' },
 569                 vendor =>  { type => 'string', optional => 1 },
 570                 model =>  { type => 'string', optional => 1 },
 571                 serial =>  { type => 'string', optional => 1 },
 572             },
 573         },
 574         # links => [ { rel => 'child', href => "{}" } ],
 575     },
 576     code => sub {
 577         my ($param) = @_;
 578
 579         PVE::CephTools::check_ceph_inited();
 580
 581         my $disks = PVE::Diskmanage::get_disks(undef, 1);
 582
 583         my $res = [];
 584         foreach my $dev (keys %$disks) {
 585             my $d = $disks->{$dev};
 586             if ($param->{type}) {
 587                 if ($param->{type} eq 'journal_disks') {
 588                     next if $d->{osdid} >= 0;
 589                     next if !$d->{gpt};
 590                 } elsif ($param->{type} eq 'unused') {
 591                     next if $d->{used};
 592                 } else {
 593                     die "internal error"; # should not happen
 594                 }
 595             }
 596
 597             $d->{dev} = "/dev/$dev";
 598             push @$res, $d;
 599         }
 600
 601         return $res;
 602     }});
 603
 604 __PACKAGE__->register_method ({
 605     name => 'config',
 606     path => 'config',
 607     method => 'GET',
 608     permissions => {
 609         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 610     },
 611     description => "Get Ceph configuration.",
 612     parameters => {
 613         additionalProperties => 0,
 614         properties => {
 615             node => get_standard_option('pve-node'),
 616         },
 617     },
 618     returns => { type => 'string' },
 619     code => sub {
 620         my ($param) = @_;
 621
 622         PVE::CephTools::check_ceph_inited();
 623
 624         my $path = PVE::CephTools::get_config('pve_ceph_cfgpath');
 625         return PVE::Tools::file_get_contents($path);
 626
 627     }});
 628
 629 __PACKAGE__->register_method ({
 630     name => 'listmon',
 631     path => 'mon',
 632     method => 'GET',
 633     description => "Get Ceph monitor list.",
 634     proxyto => 'node',
 635     protected => 1,
 636     permissions => {
 637         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 638     },
 639     parameters => {
 640         additionalProperties => 0,
 641         properties => {
 642             node => get_standard_option('pve-node'),
 643         },
 644     },
 645     returns => {
 646         type => 'array',
 647         items => {
 648             type => "object",
 649             properties => {
 650                 name => { type => 'string' },
 651                 addr => { type => 'string' },
 652             },
 653         },
 654         links => [ { rel => 'child', href => "{name}" } ],
 655     },
 656     code => sub {
 657         my ($param) = @_;
 658
 659         PVE::CephTools::check_ceph_inited();
 660
 661         my $res = [];
 662
 663         my $cfg = PVE::CephTools::parse_ceph_config();
 664
 665         my $monhash = {};
 666         foreach my $section (keys %$cfg) {
 667             my $d = $cfg->{$section};
 668             if ($section =~ m/^mon\.(\S+)$/) {
 669                 my $monid = $1;
 670                 if ($d->{'mon addr'} && $d->{'host'}) {
 671                     $monhash->{$monid} = {
 672                         addr => $d->{'mon addr'},
 673                         host => $d->{'host'},
 674                         name => $monid,
 675                     }
 676                 }
 677             }
 678         }
 679
 680         eval {
 681             my $rados = PVE::RADOS->new();
 682             my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 683             my $mons = $monstat->{monmap}->{mons};
 684             foreach my $d (@$mons) {
 685                 next if !defined($d->{name});
 686                 $monhash->{$d->{name}}->{rank} = $d->{rank};
 687                 $monhash->{$d->{name}}->{addr} = $d->{addr};
 688                 if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
 689                     $monhash->{$d->{name}}->{quorum} = 1;
 690                 }
 691             }
 692         };
 693         warn $@ if $@;
 694
 695         return PVE::RESTHandler::hash_to_array($monhash, 'name');
 696     }});
 697
 698 __PACKAGE__->register_method ({
 699     name => 'init',
 700     path => 'init',
 701     method => 'POST',
 702     description => "Create initial ceph default configuration and setup symlinks.",
 703     proxyto => 'node',
 704     protected => 1,
 705     permissions => {
 706         check => ['perm', '/', [ 'Sys.Modify' ]],
 707     },
 708     parameters => {
 709         additionalProperties => 0,
 710         properties => {
 711             node => get_standard_option('pve-node'),
 712             network => {
 713                 description => "Use specific network for all ceph related traffic",
 714                 type => 'string', format => 'CIDR',
 715                 optional => 1,
 716                 maxLength => 128,
 717             },
 718             size => {
 719                 description => 'Number of replicas per object',
 720                 type => 'integer',
 721                 default => 2,
 722                 optional => 1,
 723                 minimum => 1,
 724                 maximum => 7,
 725             },
 726             pg_bits => {
 727                 description => "Placement group bits, used to specify the " .
 728                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 729                     "default pg num' does not work for default pools.",
 730                 type => 'integer',
 731                 default => 6,
 732                 optional => 1,
 733                 minimum => 6,
 734                 maximum => 14,
 735             },
 736             disable_cephx => {
 737                 description => "Disable cephx authentification.\n\n" .
 738                     "WARNING: cephx is a security feature protecting against " .
 739                     "man-in-the-middle attacks. Only consider disabling cephx ".
 740                     "if your network is private!",
 741                 type => 'boolean',
 742                 optional => 1,
 743                 default => 0,
 744             },
 745         },
 746     },
 747     returns => { type => 'null' },
 748     code => sub {
 749         my ($param) = @_;
 750
 751         PVE::CephTools::check_ceph_installed();
 752
 753         # simply load old config if it already exists
 754         my $cfg = PVE::CephTools::parse_ceph_config();
 755
 756         if (!$cfg->{global}) {
 757
 758             my $fsid;
 759             my $uuid;
 760
 761             UUID::generate($uuid);
 762             UUID::unparse($uuid, $fsid);
 763
 764             my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 765
 766             $cfg->{global} = {
 767                 'fsid' => $fsid,
 768                 'auth cluster required' => $auth,
 769                 'auth service required' => $auth,
 770                 'auth client required' => $auth,
 771                 'osd journal size' => $pve_osd_default_journal_size,
 772                 'osd pool default min size' => 1,
 773             };
 774
 775             # this does not work for default pools
 776             #'osd pool default pg num' => $pg_num,
 777             #'osd pool default pgp num' => $pg_num,
 778         }
 779
 780         $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 781         $cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring';
 782
 783         $cfg->{global}->{'osd pool default size'} = $param->{size} if $param->{size};
 784
 785         if ($param->{pg_bits}) {
 786             $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 787             $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 788         }
 789
 790         if ($param->{network}) {
 791             $cfg->{global}->{'public network'} = $param->{network};
 792             $cfg->{global}->{'cluster network'} = $param->{network};
 793         }
 794
 795         PVE::CephTools::write_ceph_config($cfg);
 796
 797         PVE::CephTools::setup_pve_symlinks();
 798
 799         return undef;
 800     }});
 801
 802 my $find_node_ip = sub {
 803     my ($cidr) = @_;
 804
 805     my $net = Net::IP->new($cidr) || die Net::IP::Error() . "\n";
 806     my $id = $net->version == 6 ? 'address6' : 'address';
 807
 808     my $config = PVE::INotify::read_file('interfaces');
 809     my $ifaces = $config->{ifaces};
 810
 811     foreach my $iface (keys %$ifaces) {
 812         my $d = $ifaces->{$iface};
 813         next if !$d->{$id};
 814         my $a = Net::IP->new($d->{$id});
 815         next if !$a;
 816         return $d->{$id} if $net->overlaps($a);
 817     }
 818
 819     die "unable to find local address within network '$cidr'\n";
 820 };
 821
 822 __PACKAGE__->register_method ({
 823     name => 'createmon',
 824     path => 'mon',
 825     method => 'POST',
 826     description => "Create Ceph Monitor",
 827     proxyto => 'node',
 828     protected => 1,
 829     permissions => {
 830         check => ['perm', '/', [ 'Sys.Modify' ]],
 831     },
 832     parameters => {
 833         additionalProperties => 0,
 834         properties => {
 835             node => get_standard_option('pve-node'),
 836         },
 837     },
 838     returns => { type => 'string' },
 839     code => sub {
 840         my ($param) = @_;
 841
 842         PVE::CephTools::check_ceph_inited();
 843
 844         PVE::CephTools::setup_pve_symlinks();
 845
 846         my $rpcenv = PVE::RPCEnvironment::get();
 847
 848         my $authuser = $rpcenv->get_user();
 849
 850         my $cfg = PVE::CephTools::parse_ceph_config();
 851
 852         my $moncount = 0;
 853
 854         my $monaddrhash = {};
 855
 856         my $systemd_managed = PVE::CephTools::systemd_managed();
 857
 858         foreach my $section (keys %$cfg) {
 859             next if $section eq 'global';
 860             my $d = $cfg->{$section};
 861             if ($section =~ m/^mon\./) {
 862                 $moncount++;
 863                 if ($d->{'mon addr'}) {
 864                     $monaddrhash->{$d->{'mon addr'}} = $section;
 865                 }
 866             }
 867         }
 868
 869         my $monid;
 870         for (my $i = 0; $i < 7; $i++) {
 871             if (!$cfg->{"mon.$i"}) {
 872                 $monid = $i;
 873                 last;
 874             }
 875         }
 876         die "unable to find usable monitor id\n" if !defined($monid);
 877
 878         my $monsection = "mon.$monid";
 879         my $ip;
 880         if (my $pubnet = $cfg->{global}->{'public network'}) {
 881             $ip = &$find_node_ip($pubnet);
 882         } else {
 883             $ip = PVE::Cluster::remote_node_ip($param->{node});
 884         }
 885
 886         my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]:6789" : "$ip:6789";
 887         my $monname = $param->{node};
 888
 889         die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
 890         die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n"
 891             if $monaddrhash->{$monaddr};
 892
 893         my $worker = sub  {
 894             my $upid = shift;
 895
 896             my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
 897
 898             if (! -f $pve_ckeyring_path) {
 899                 run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
 900                             "--gen-key -n client.admin");
 901             }
 902
 903             my $pve_mon_key_path = PVE::CephTools::get_config('pve_mon_key_path');
 904             if (! -f $pve_mon_key_path) {
 905                 run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
 906                 run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " .
 907                             "--cap mds 'allow' " .
 908                             "--cap osd 'allow *' " .
 909                             "--cap mon 'allow *'");
 910                 run_command("cp $pve_mon_key_path.tmp /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
 911                 run_command("chown ceph:ceph /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
 912                 run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
 913                 run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
 914             }
 915
 916             my $ccname = PVE::CephTools::get_config('ccname');
 917
 918             my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 919             -d $mondir && die "monitor filesystem '$mondir' already exist\n";
 920
 921             my $monmap = "/tmp/monmap";
 922
 923             eval {
 924                 mkdir $mondir;
 925
 926                 run_command("chown ceph:ceph $mondir") if $systemd_managed;
 927
 928                 if ($moncount > 0) {
 929                     my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
 930                     my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
 931                     PVE::Tools::file_set_contents($monmap, $mapdata);
 932                 } else {
 933                     run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
 934                 }
 935
 936                 run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
 937                 run_command("chown ceph:ceph -R $mondir") if $systemd_managed;
 938             };
 939             my $err = $@;
 940             unlink $monmap;
 941             if ($err) {
 942                 File::Path::remove_tree($mondir);
 943                 die $err;
 944             }
 945
 946             $cfg->{$monsection} = {
 947                 'host' => $monname,
 948                 'mon addr' => $monaddr,
 949             };
 950
 951             PVE::CephTools::write_ceph_config($cfg);
 952
 953             my $create_keys_pid = fork();
 954             if (!defined($create_keys_pid)) {
 955                 die "Could not spawn ceph-create-keys to create bootstrap keys\n";
 956             } elsif ($create_keys_pid == 0) {
 957                 exit PVE::Tools::run_command(['ceph-create-keys', '-i', $monid]);
 958             } else {
 959                 PVE::CephTools::ceph_service_cmd('start', $monsection);
 960
 961                 if ($systemd_managed) {
 962                     #to ensure we have the correct startup order.
 963                     eval { PVE::Tools::run_command(['/bin/systemctl', 'enable', "ceph-mon\@${monid}.service"]); };
 964                     warn "Enable ceph-mon\@${monid}.service manually"if $@;
 965                 }
 966                 waitpid($create_keys_pid, 0);
 967             }
 968         };
 969
 970         return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
 971     }});
 972
 973 __PACKAGE__->register_method ({
 974     name => 'destroymon',
 975     path => 'mon/{monid}',
 976     method => 'DELETE',
 977     description => "Destroy Ceph monitor.",
 978     proxyto => 'node',
 979     protected => 1,
 980     permissions => {
 981         check => ['perm', '/', [ 'Sys.Modify' ]],
 982     },
 983     parameters => {
 984         additionalProperties => 0,
 985         properties => {
 986             node => get_standard_option('pve-node'),
 987             monid => {
 988                 description => 'Monitor ID',
 989                 type => 'integer',
 990             },
 991         },
 992     },
 993     returns => { type => 'string' },
 994     code => sub {
 995         my ($param) = @_;
 996
 997         my $rpcenv = PVE::RPCEnvironment::get();
 998
 999         my $authuser = $rpcenv->get_user();
1000
1001         PVE::CephTools::check_ceph_inited();
1002
1003         my $cfg = PVE::CephTools::parse_ceph_config();
1004
1005         my $monid = $param->{monid};
1006         my $monsection = "mon.$monid";
1007
1008         my $rados = PVE::RADOS->new();
1009         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
1010         my $monlist = $monstat->{monmap}->{mons};
1011
1012         die "no such monitor id '$monid'\n"
1013             if !defined($cfg->{$monsection});
1014
1015         my $ccname = PVE::CephTools::get_config('ccname');
1016
1017         my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
1018         -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n";
1019
1020         die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
1021
1022         my $worker = sub {
1023             my $upid = shift;
1024
1025             # reopen with longer timeout
1026             $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1027
1028             $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });
1029
1030             eval { PVE::CephTools::ceph_service_cmd('stop', $monsection); };
1031             warn $@ if $@;
1032
1033             delete $cfg->{$monsection};
1034             PVE::CephTools::write_ceph_config($cfg);
1035             File::Path::remove_tree($mondir);
1036         };
1037
1038         return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
1039     }});
1040
1041 __PACKAGE__->register_method ({
1042     name => 'stop',
1043     path => 'stop',
1044     method => 'POST',
1045     description => "Stop ceph services.",
1046     proxyto => 'node',
1047     protected => 1,
1048     permissions => {
1049         check => ['perm', '/', [ 'Sys.Modify' ]],
1050     },
1051     parameters => {
1052         additionalProperties => 0,
1053         properties => {
1054             node => get_standard_option('pve-node'),
1055             service => {
1056                 description => 'Ceph service name.',
1057                 type => 'string',
1058                 optional => 1,
1059                 pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}',
1060             },
1061         },
1062     },
1063     returns => { type => 'string' },
1064     code => sub {
1065         my ($param) = @_;
1066
1067         my $rpcenv = PVE::RPCEnvironment::get();
1068
1069         my $authuser = $rpcenv->get_user();
1070
1071         PVE::CephTools::check_ceph_inited();
1072
1073         my $cfg = PVE::CephTools::parse_ceph_config();
1074         scalar(keys %$cfg) || die "no configuration\n";
1075
1076         my $worker = sub {
1077             my $upid = shift;
1078
1079             my $cmd = ['stop'];
1080             if ($param->{service}) {
1081                 push @$cmd, $param->{service};
1082             }
1083
1084             PVE::CephTools::ceph_service_cmd(@$cmd);
1085         };
1086
1087         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
1088                                     $authuser, $worker);
1089     }});
1090
1091 __PACKAGE__->register_method ({
1092     name => 'start',
1093     path => 'start',
1094     method => 'POST',
1095     description => "Start ceph services.",
1096     proxyto => 'node',
1097     protected => 1,
1098     permissions => {
1099         check => ['perm', '/', [ 'Sys.Modify' ]],
1100     },
1101     parameters => {
1102         additionalProperties => 0,
1103         properties => {
1104             node => get_standard_option('pve-node'),
1105             service => {
1106                 description => 'Ceph service name.',
1107                 type => 'string',
1108                 optional => 1,
1109                 pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}',
1110             },
1111         },
1112     },
1113     returns => { type => 'string' },
1114     code => sub {
1115         my ($param) = @_;
1116
1117         my $rpcenv = PVE::RPCEnvironment::get();
1118
1119         my $authuser = $rpcenv->get_user();
1120
1121         PVE::CephTools::check_ceph_inited();
1122
1123         my $cfg = PVE::CephTools::parse_ceph_config();
1124         scalar(keys %$cfg) || die "no configuration\n";
1125
1126         my $worker = sub {
1127             my $upid = shift;
1128
1129             my $cmd = ['start'];
1130             if ($param->{service}) {
1131                 push @$cmd, $param->{service};
1132             }
1133
1134             PVE::CephTools::ceph_service_cmd(@$cmd);
1135         };
1136
1137         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
1138                                     $authuser, $worker);
1139     }});
1140
1141 __PACKAGE__->register_method ({
1142     name => 'status',
1143     path => 'status',
1144     method => 'GET',
1145     description => "Get ceph status.",
1146     proxyto => 'node',
1147     protected => 1,
1148     permissions => {
1149         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1150     },
1151     parameters => {
1152         additionalProperties => 0,
1153         properties => {
1154             node => get_standard_option('pve-node'),
1155         },
1156     },
1157     returns => { type => 'object' },
1158     code => sub {
1159         my ($param) = @_;
1160
1161         PVE::CephTools::check_ceph_enabled();
1162
1163         my $rados = PVE::RADOS->new();
1164         return $rados->mon_command({ prefix => 'status' });
1165     }});
1166
1167 __PACKAGE__->register_method ({
1168     name => 'lspools',
1169     path => 'pools',
1170     method => 'GET',
1171     description => "List all pools.",
1172     proxyto => 'node',
1173     protected => 1,
1174     permissions => {
1175         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1176     },
1177     parameters => {
1178         additionalProperties => 0,
1179         properties => {
1180             node => get_standard_option('pve-node'),
1181         },
1182     },
1183     returns => {
1184         type => 'array',
1185         items => {
1186             type => "object",
1187             properties => {
1188                 pool => { type => 'integer' },
1189                 pool_name => { type => 'string' },
1190                 size => { type => 'integer' },
1191             },
1192         },
1193         links => [ { rel => 'child', href => "{pool_name}" } ],
1194     },
1195     code => sub {
1196         my ($param) = @_;
1197
1198         PVE::CephTools::check_ceph_inited();
1199
1200         my $rados = PVE::RADOS->new();
1201
1202         my $stats = {};
1203         my $res = $rados->mon_command({ prefix => 'df' });
1204         my $total = $res->{stats}->{total_avail_bytes} || 0;
1205
1206         foreach my $d (@{$res->{pools}}) {
1207             next if !$d->{stats};
1208             next if !defined($d->{id});
1209             $stats->{$d->{id}} = $d->{stats};
1210         }
1211
1212         $res = $rados->mon_command({ prefix => 'osd dump' });
1213
1214         my $data = [];
1215         foreach my $e (@{$res->{pools}}) {
1216             my $d = {};
1217             foreach my $attr (qw(pool pool_name size min_size pg_num crush_ruleset)) {
1218                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
1219             }
1220             if (my $s = $stats->{$d->{pool}}) {
1221                 $d->{bytes_used} = $s->{bytes_used};
1222                 $d->{percent_used} = ($s->{bytes_used} / $total)*100
1223                     if $s->{max_avail} && $total;
1224             }
1225             push @$data, $d;
1226         }
1227
1228
1229         return $data;
1230     }});
1231
1232 __PACKAGE__->register_method ({
1233     name => 'createpool',
1234     path => 'pools',
1235     method => 'POST',
1236     description => "Create POOL",
1237     proxyto => 'node',
1238     protected => 1,
1239     permissions => {
1240         check => ['perm', '/', [ 'Sys.Modify' ]],
1241     },
1242     parameters => {
1243         additionalProperties => 0,
1244         properties => {
1245             node => get_standard_option('pve-node'),
1246             name => {
1247                 description => "The name of the pool. It must be unique.",
1248                 type => 'string',
1249             },
1250             size => {
1251                 description => 'Number of replicas per object',
1252                 type => 'integer',
1253                 default => 2,
1254                 optional => 1,
1255                 minimum => 1,
1256                 maximum => 7,
1257             },
1258             min_size => {
1259                 description => 'Minimum number of replicas per object',
1260                 type => 'integer',
1261                 default => 1,
1262                 optional => 1,
1263                 minimum => 1,
1264                 maximum => 7,
1265             },
1266             pg_num => {
1267                 description => "Number of placement groups.",
1268                 type => 'integer',
1269                 default => 64,
1270                 optional => 1,
1271                 minimum => 8,
1272                 maximum => 32768,
1273             },
1274             crush_ruleset => {
1275                 description => "The ruleset to use for mapping object placement in the cluster.",
1276                 type => 'integer',
1277                 minimum => 0,
1278                 maximum => 32768,
1279                 default => 0,
1280                 optional => 1,
1281             },
1282         },
1283     },
1284     returns => { type => 'null' },
1285     code => sub {
1286         my ($param) = @_;
1287
1288         PVE::CephTools::check_ceph_inited();
1289
1290         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1291
1292         die "not fully configured - missing '$pve_ckeyring_path'\n"
1293             if ! -f $pve_ckeyring_path;
1294
1295         my $pg_num = $param->{pg_num} || 64;
1296         my $size = $param->{size} || 2;
1297         my $min_size = $param->{min_size} || 1;
1298         my $ruleset = $param->{crush_ruleset} || 0;
1299         my $rados = PVE::RADOS->new();
1300
1301         $rados->mon_command({
1302             prefix => "osd pool create",
1303             pool => $param->{name},
1304             pg_num => int($pg_num),
1305 # this does not work for unknown reason
1306 #           properties => ["size=$size", "min_size=$min_size", "crush_ruleset=$ruleset"],
1307             format => 'plain',
1308         });
1309
1310         $rados->mon_command({
1311             prefix => "osd pool set",
1312             pool => $param->{name},
1313             var => 'min_size',
1314             val => $min_size,
1315             format => 'plain',
1316         });
1317
1318         $rados->mon_command({
1319             prefix => "osd pool set",
1320             pool => $param->{name},
1321             var => 'size',
1322             val => $size,
1323             format => 'plain',
1324         });
1325
1326         if (defined($param->{crush_ruleset})) {
1327             $rados->mon_command({
1328                 prefix => "osd pool set",
1329                 pool => $param->{name},
1330                 var => 'crush_ruleset',
1331                 val => $param->{crush_ruleset},
1332                 format => 'plain',
1333             });
1334         }
1335
1336         return undef;
1337     }});
1338
1339 __PACKAGE__->register_method ({
1340     name => 'get_flags',
1341     path => 'flags',
1342     method => 'GET',
1343     description => "get all set ceph flags",
1344     proxyto => 'node',
1345     protected => 1,
1346     permissions => {
1347         check => ['perm', '/', [ 'Sys.Audit' ]],
1348     },
1349     parameters => {
1350         additionalProperties => 0,
1351         properties => {
1352             node => get_standard_option('pve-node'),
1353         },
1354     },
1355     returns => { type => 'string' },
1356     code => sub {
1357         my ($param) = @_;
1358
1359         PVE::CephTools::check_ceph_inited();
1360
1361         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1362
1363         die "not fully configured - missing '$pve_ckeyring_path'\n"
1364             if ! -f $pve_ckeyring_path;
1365
1366         my $rados = PVE::RADOS->new();
1367
1368         my $stat = $rados->mon_command({ prefix => 'osd dump' });
1369
1370         return $stat->{flags} // '';
1371     }});
1372
1373 __PACKAGE__->register_method ({
1374     name => 'set_flag',
1375     path => 'flags/{flag}',
1376     method => 'POST',
1377     description => "Set a ceph flag",
1378     proxyto => 'node',
1379     protected => 1,
1380     permissions => {
1381         check => ['perm', '/', [ 'Sys.Modify' ]],
1382     },
1383     parameters => {
1384         additionalProperties => 0,
1385         properties => {
1386             node => get_standard_option('pve-node'),
1387             flag => {
1388                 description => 'The ceph flag to set/unset',
1389                 type => 'string',
1390                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1391             },
1392         },
1393     },
1394     returns => { type => 'null' },
1395     code => sub {
1396         my ($param) = @_;
1397
1398         PVE::CephTools::check_ceph_inited();
1399
1400         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1401
1402         die "not fully configured - missing '$pve_ckeyring_path'\n"
1403             if ! -f $pve_ckeyring_path;
1404
1405         my $set = $param->{set} // !$param->{unset};
1406         my $rados = PVE::RADOS->new();
1407
1408         $rados->mon_command({
1409             prefix => "osd set",
1410             key => $param->{flag},
1411         });
1412
1413         return undef;
1414     }});
1415
1416 __PACKAGE__->register_method ({
1417     name => 'unset_flag',
1418     path => 'flags/{flag}',
1419     method => 'DELETE',
1420     description => "Unset a ceph flag",
1421     proxyto => 'node',
1422     protected => 1,
1423     permissions => {
1424         check => ['perm', '/', [ 'Sys.Modify' ]],
1425     },
1426     parameters => {
1427         additionalProperties => 0,
1428         properties => {
1429             node => get_standard_option('pve-node'),
1430             flag => {
1431                 description => 'The ceph flag to set/unset',
1432                 type => 'string',
1433                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1434             },
1435         },
1436     },
1437     returns => { type => 'null' },
1438     code => sub {
1439         my ($param) = @_;
1440
1441         PVE::CephTools::check_ceph_inited();
1442
1443         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1444
1445         die "not fully configured - missing '$pve_ckeyring_path'\n"
1446             if ! -f $pve_ckeyring_path;
1447
1448         my $set = $param->{set} // !$param->{unset};
1449         my $rados = PVE::RADOS->new();
1450
1451         $rados->mon_command({
1452             prefix => "osd unset",
1453             key => $param->{flag},
1454         });
1455
1456         return undef;
1457     }});
1458
1459 __PACKAGE__->register_method ({
1460     name => 'destroypool',
1461     path => 'pools/{name}',
1462     method => 'DELETE',
1463     description => "Destroy pool",
1464     proxyto => 'node',
1465     protected => 1,
1466     permissions => {
1467         check => ['perm', '/', [ 'Sys.Modify' ]],
1468     },
1469     parameters => {
1470         additionalProperties => 0,
1471         properties => {
1472             node => get_standard_option('pve-node'),
1473             name => {
1474                 description => "The name of the pool. It must be unique.",
1475                 type => 'string',
1476             },
1477             force => {
1478                 description => "If true, destroys pool even if in use",
1479                 type => 'boolean',
1480                 optional => 1,
1481                 default => 0,
1482             }
1483         },
1484     },
1485     returns => { type => 'null' },
1486     code => sub {
1487         my ($param) = @_;
1488
1489         PVE::CephTools::check_ceph_inited();
1490
1491         # if not forced, destroy ceph pool only when no
1492         # vm disks are on it anymore
1493         if (!$param->{force}) {
1494             my $storagecfg = PVE::Storage::config();
1495             foreach my $storageid (keys %{$storagecfg->{ids}}) {
1496                 my $storage = $storagecfg->{ids}->{$storageid};
1497                 next if $storage->{type} ne 'rbd';
1498                 next if $storage->{pool} ne $param->{name};
1499
1500                 # check if any vm disks are on the pool
1501                 my $res = PVE::Storage::vdisk_list($storagecfg, $storageid);
1502                 die "ceph pool '$param->{name}' still in use by storage '$storageid'\n"
1503                     if @{$res->{$storageid}} != 0;
1504             }
1505         }
1506
1507         my $rados = PVE::RADOS->new();
1508         # fixme: '--yes-i-really-really-mean-it'
1509         $rados->mon_command({
1510             prefix => "osd pool delete",
1511             pool => $param->{name},
1512             pool2 => $param->{name},
1513             sure => '--yes-i-really-really-mean-it',
1514             format => 'plain',
1515         });
1516
1517         return undef;
1518     }});
1519
1520
1521 __PACKAGE__->register_method ({
1522     name => 'crush',
1523     path => 'crush',
1524     method => 'GET',
1525     description => "Get OSD crush map",
1526     proxyto => 'node',
1527     protected => 1,
1528     permissions => {
1529         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1530     },
1531     parameters => {
1532         additionalProperties => 0,
1533         properties => {
1534             node => get_standard_option('pve-node'),
1535         },
1536     },
1537     returns => { type => 'string' },
1538     code => sub {
1539         my ($param) = @_;
1540
1541         PVE::CephTools::check_ceph_inited();
1542
1543         # this produces JSON (difficult to read for the user)
1544         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
1545
1546         my $txt = '';
1547
1548         my $mapfile = "/var/tmp/ceph-crush.map.$$";
1549         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
1550
1551         my $rados = PVE::RADOS->new();
1552
1553         eval {
1554             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
1555             PVE::Tools::file_set_contents($mapfile, $bindata);
1556             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
1557             $txt = PVE::Tools::file_get_contents($mapdata);
1558         };
1559         my $err = $@;
1560
1561         unlink $mapfile;
1562         unlink $mapdata;
1563
1564         die $err if $err;
1565
1566         return $txt;
1567     }});
1568
1569 __PACKAGE__->register_method({
1570     name => 'log',
1571     path => 'log',
1572     method => 'GET',
1573     description => "Read ceph log",
1574     proxyto => 'node',
1575     permissions => {
1576         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
1577     },
1578     protected => 1,
1579     parameters => {
1580         additionalProperties => 0,
1581         properties => {
1582             node => get_standard_option('pve-node'),
1583             start => {
1584                 type => 'integer',
1585                 minimum => 0,
1586                 optional => 1,
1587             },
1588             limit => {
1589                 type => 'integer',
1590                 minimum => 0,
1591                 optional => 1,
1592             },
1593         },
1594     },
1595     returns => {
1596         type => 'array',
1597         items => {
1598             type => "object",
1599             properties => {
1600                 n => {
1601                   description=>  "Line number",
1602                   type=> 'integer',
1603                 },
1604                 t => {
1605                   description=>  "Line text",
1606                   type => 'string',
1607                 }
1608             }
1609         }
1610     },
1611     code => sub {
1612         my ($param) = @_;
1613
1614         my $rpcenv = PVE::RPCEnvironment::get();
1615         my $user = $rpcenv->get_user();
1616         my $node = $param->{node};
1617
1618         my $logfile = "/var/log/ceph/ceph.log";
1619         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
1620
1621         $rpcenv->set_result_attrib('total', $count);
1622
1623         return $lines;
1624     }});
1625
1626