PVE/API2/Ceph.pm

   1 package PVE::API2::CephOSD;
   2
   3 use strict;
   4 use warnings;
   5 use Cwd qw(abs_path);
   6 use Net::IP;
   7
   8 use PVE::SafeSyslog;
   9 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
  10 use PVE::Exception qw(raise raise_param_exc);
  11 use PVE::INotify;
  12 use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
  13 use PVE::AccessControl;
  14 use PVE::Storage;
  15 use PVE::RESTHandler;
  16 use PVE::RPCEnvironment;
  17 use PVE::JSONSchema qw(get_standard_option);
  18 use PVE::RADOS;
  19 use PVE::CephTools;
  20 use PVE::Diskmanage;
  21
  22 use base qw(PVE::RESTHandler);
  23
  24 use Data::Dumper; # fixme: remove
  25
  26 my $get_osd_status = sub {
  27     my ($rados, $osdid) = @_;
  28
  29     my $stat = $rados->mon_command({ prefix => 'osd dump' });
  30
  31     my $osdlist = $stat->{osds} || [];
  32
  33     my $flags = $stat->{flags} || undef;
  34
  35     my $osdstat;
  36     foreach my $d (@$osdlist) {
  37         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  38     }
  39     if (defined($osdid)) {
  40         die "no such OSD '$osdid'\n" if !$osdstat->{$osdid};
  41         return $osdstat->{$osdid};
  42     }
  43
  44     return wantarray? ($osdstat, $flags):$osdstat;
  45 };
  46
  47 my $get_osd_usage = sub {
  48     my ($rados) = @_;
  49
  50     my $osdlist = $rados->mon_command({ prefix => 'pg dump',
  51                                         dumpcontents => [ 'osds' ]}) || [];
  52
  53     my $osdstat;
  54     foreach my $d (@$osdlist) {
  55         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  56     }
  57
  58     return $osdstat;
  59 };
  60
  61 __PACKAGE__->register_method ({
  62     name => 'index',
  63     path => '',
  64     method => 'GET',
  65     description => "Get Ceph osd list/tree.",
  66     proxyto => 'node',
  67     protected => 1,
  68     permissions => {
  69         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  70     },
  71     parameters => {
  72         additionalProperties => 0,
  73         properties => {
  74             node => get_standard_option('pve-node'),
  75         },
  76     },
  77     # fixme: return a list instead of extjs tree format ?
  78     returns => {
  79         type => "object",
  80     },
  81     code => sub {
  82         my ($param) = @_;
  83
  84         PVE::CephTools::check_ceph_inited();
  85
  86         my $rados = PVE::RADOS->new();
  87         my $res = $rados->mon_command({ prefix => 'osd tree' });
  88
  89         die "no tree nodes found\n" if !($res && $res->{nodes});
  90
  91         my ($osdhash, $flags) = &$get_osd_status($rados);
  92
  93         my $usagehash = &$get_osd_usage($rados);
  94
  95         my $nodes = {};
  96         my $newnodes = {};
  97         foreach my $e (@{$res->{nodes}}) {
  98             $nodes->{$e->{id}} = $e;
  99
 100             my $new = {
 101                 id => $e->{id},
 102                 name => $e->{name},
 103                 type => $e->{type}
 104             };
 105
 106             foreach my $opt (qw(status crush_weight reweight)) {
 107                 $new->{$opt} = $e->{$opt} if defined($e->{$opt});
 108             }
 109
 110             if (my $stat = $osdhash->{$e->{id}}) {
 111                 $new->{in} = $stat->{in} if defined($stat->{in});
 112             }
 113
 114             if (my $stat = $usagehash->{$e->{id}}) {
 115                 $new->{total_space} = ($stat->{kb} || 1) * 1024;
 116                 $new->{bytes_used} = ($stat->{kb_used} || 0) * 1024;
 117                 $new->{percent_used} = ($new->{bytes_used}*100)/$new->{total_space};
 118                 if (my $d = $stat->{fs_perf_stat}) {
 119                     $new->{commit_latency_ms} = $d->{commit_latency_ms};
 120                     $new->{apply_latency_ms} = $d->{apply_latency_ms};
 121                 }
 122             }
 123
 124             $newnodes->{$e->{id}} = $new;
 125         }
 126
 127         foreach my $e (@{$res->{nodes}}) {
 128             my $new = $newnodes->{$e->{id}};
 129             if ($e->{children} && scalar(@{$e->{children}})) {
 130                 $new->{children} = [];
 131                 $new->{leaf} = 0;
 132                 foreach my $cid (@{$e->{children}}) {
 133                     $nodes->{$cid}->{parent} = $e->{id};
 134                     if ($nodes->{$cid}->{type} eq 'osd' &&
 135                         $e->{type} eq 'host') {
 136                         $newnodes->{$cid}->{host} = $e->{name};
 137                     }
 138                     push @{$new->{children}}, $newnodes->{$cid};
 139                 }
 140             } else {
 141                 $new->{leaf} = ($e->{id} >= 0) ? 1 : 0;
 142             }
 143         }
 144
 145         my $roots = [];
 146         foreach my $e (@{$res->{nodes}}) {
 147             if (!$nodes->{$e->{id}}->{parent}) {
 148                 push @$roots, $newnodes->{$e->{id}};
 149             }
 150         }
 151
 152         die "no root node\n" if !@$roots;
 153
 154         my $data = { root => { leaf =>  0, children => $roots } };
 155
 156         # we want this for the noout flag
 157         $data->{flags} = $flags if $flags;
 158
 159         return $data;
 160     }});
 161
 162 __PACKAGE__->register_method ({
 163     name => 'createosd',
 164     path => '',
 165     method => 'POST',
 166     description => "Create OSD",
 167     proxyto => 'node',
 168     protected => 1,
 169     parameters => {
 170         additionalProperties => 0,
 171         properties => {
 172             node => get_standard_option('pve-node'),
 173             dev => {
 174                 description => "Block device name.",
 175                 type => 'string',
 176             },
 177             journal_dev => {
 178                 description => "Block device name for journal.",
 179                 optional => 1,
 180                 type => 'string',
 181             },
 182             fstype => {
 183                 description => "File system type.",
 184                 type => 'string',
 185                 enum => ['xfs', 'ext4', 'btrfs'],
 186                 default => 'xfs',
 187                 optional => 1,
 188             },
 189         },
 190     },
 191     returns => { type => 'string' },
 192     code => sub {
 193         my ($param) = @_;
 194
 195         my $rpcenv = PVE::RPCEnvironment::get();
 196
 197         my $authuser = $rpcenv->get_user();
 198
 199         PVE::CephTools::check_ceph_inited();
 200
 201         PVE::CephTools::setup_pve_symlinks();
 202
 203         my $journal_dev;
 204
 205         if ($param->{journal_dev} && ($param->{journal_dev} ne $param->{dev})) {
 206             $journal_dev = PVE::Diskmanage::verify_blockdev_path($param->{journal_dev});
 207         }
 208
 209         $param->{dev} = PVE::Diskmanage::verify_blockdev_path($param->{dev});
 210
 211         my $devname = $param->{dev};
 212         $devname =~ s|/dev/||;
 213
 214         my $disklist = PVE::Diskmanage::get_disks($devname, 1);
 215
 216         my $diskinfo = $disklist->{$devname};
 217         die "unable to get device info for '$devname'\n"
 218             if !$diskinfo;
 219
 220         die "device '$param->{dev}' is in use\n"
 221             if $diskinfo->{used};
 222
 223         my $devpath = $diskinfo->{devpath};
 224         my $rados = PVE::RADOS->new();
 225         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 226         die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid};
 227
 228         my $fsid = $monstat->{monmap}->{fsid};
 229         $fsid = $1 if $fsid =~ m/^([0-9a-f\-]+)$/;
 230
 231         my $ceph_bootstrap_osd_keyring = PVE::CephTools::get_config('ceph_bootstrap_osd_keyring');
 232
 233         if (! -f $ceph_bootstrap_osd_keyring) {
 234             my $bindata = $rados->mon_command({ prefix => 'auth get', entity => 'client.bootstrap-osd', format => 'plain' });
 235             PVE::Tools::file_set_contents($ceph_bootstrap_osd_keyring, $bindata);
 236         };
 237
 238         my $worker = sub {
 239             my $upid = shift;
 240
 241             my $fstype = $param->{fstype} || 'xfs';
 242
 243             print "create OSD on $devpath ($fstype)\n";
 244
 245             my $ccname = PVE::CephTools::get_config('ccname');
 246
 247             my $cmd = ['ceph-disk', 'prepare', '--zap-disk', '--fs-type', $fstype,
 248                        '--cluster', $ccname, '--cluster-uuid', $fsid ];
 249
 250             if ($journal_dev) {
 251                 print "using device '$journal_dev' for journal\n";
 252                 push @$cmd, '--journal-dev', $devpath, $journal_dev;
 253             } else {
 254                 push @$cmd, $devpath;
 255             }
 256
 257             run_command($cmd);
 258         };
 259
 260         return $rpcenv->fork_worker('cephcreateosd', $devname,  $authuser, $worker);
 261     }});
 262
 263 __PACKAGE__->register_method ({
 264     name => 'destroyosd',
 265     path => '{osdid}',
 266     method => 'DELETE',
 267     description => "Destroy OSD",
 268     proxyto => 'node',
 269     protected => 1,
 270     parameters => {
 271         additionalProperties => 0,
 272         properties => {
 273             node => get_standard_option('pve-node'),
 274             osdid => {
 275                 description => 'OSD ID',
 276                 type => 'integer',
 277             },
 278             cleanup => {
 279                 description => "If set, we remove partition table entries.",
 280                 type => 'boolean',
 281                 optional => 1,
 282                 default => 0,
 283             },
 284         },
 285     },
 286     returns => { type => 'string' },
 287     code => sub {
 288         my ($param) = @_;
 289
 290         my $rpcenv = PVE::RPCEnvironment::get();
 291
 292         my $authuser = $rpcenv->get_user();
 293
 294         PVE::CephTools::check_ceph_inited();
 295
 296         my $osdid = $param->{osdid};
 297
 298         my $rados = PVE::RADOS->new();
 299         my $osdstat = &$get_osd_status($rados, $osdid);
 300
 301         die "osd is in use (in == 1)\n" if $osdstat->{in};
 302         #&$run_ceph_cmd(['osd', 'out', $osdid]);
 303
 304         die "osd is still runnung (up == 1)\n" if $osdstat->{up};
 305
 306         my $osdsection = "osd.$osdid";
 307
 308         my $worker = sub {
 309             my $upid = shift;
 310
 311             # reopen with longer timeout
 312             $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
 313
 314             print "destroy OSD $osdsection\n";
 315
 316             eval { PVE::CephTools::ceph_service_cmd('stop', $osdsection); };
 317             warn $@ if $@;
 318
 319             print "Remove $osdsection from the CRUSH map\n";
 320             $rados->mon_command({ prefix => "osd crush remove", name => $osdsection, format => 'plain' });
 321
 322             print "Remove the $osdsection authentication key.\n";
 323             $rados->mon_command({ prefix => "auth del", entity => $osdsection, format => 'plain' });
 324
 325             print "Remove OSD $osdsection\n";
 326             $rados->mon_command({ prefix => "osd rm", ids => [ $osdsection ], format => 'plain' });
 327
 328             # try to unmount from standard mount point
 329             my $mountpoint = "/var/lib/ceph/osd/ceph-$osdid";
 330
 331             my $remove_partition = sub {
 332                 my ($disklist, $part) = @_;
 333
 334                 return if !$part || (! -b $part );
 335
 336                 foreach my $real_dev (keys %$disklist) {
 337                     my $diskinfo = $disklist->{$real_dev};
 338                     my $devpath = $diskinfo->{devpath};
 339                     next if !$diskinfo->{gpt};
 340                     if ($part =~ m|^${devpath}(\d+)$|) {
 341                         my $partnum = $1;
 342                         print "remove partition $part (disk '${devpath}', partnum $partnum)\n";
 343                         eval { run_command(['/sbin/sgdisk', '-d', $partnum, "${devpath}"]); };
 344                         warn $@ if $@;
 345                         last;
 346                     }
 347                 }
 348             };
 349
 350             my $journal_part;
 351             my $data_part;
 352
 353             if ($param->{cleanup}) {
 354                 my $jpath = "$mountpoint/journal";
 355                 $journal_part = abs_path($jpath);
 356
 357                 if (my $fd = IO::File->new("/proc/mounts", "r")) {
 358                     while (defined(my $line = <$fd>)) {
 359                         my ($dev, $path, $fstype) = split(/\s+/, $line);
 360                         next if !($dev && $path && $fstype);
 361                         next if $dev !~ m|^/dev/|;
 362                         if ($path eq $mountpoint) {
 363                             $data_part = abs_path($dev);
 364                             last;
 365                         }
 366                     }
 367                     close($fd);
 368                 }
 369             }
 370
 371             print "Unmount OSD $osdsection from  $mountpoint\n";
 372             eval { run_command(['umount', $mountpoint]); };
 373             if (my $err = $@) {
 374                 warn $err;
 375             } elsif ($param->{cleanup}) {
 376                 my $disklist = PVE::Diskmanage::get_disks(undef, 1);
 377                 &$remove_partition($disklist, $journal_part);
 378                 &$remove_partition($disklist, $data_part);
 379             }
 380         };
 381
 382         return $rpcenv->fork_worker('cephdestroyosd', $osdsection,  $authuser, $worker);
 383     }});
 384
 385 __PACKAGE__->register_method ({
 386     name => 'in',
 387     path => '{osdid}/in',
 388     method => 'POST',
 389     description => "ceph osd in",
 390     proxyto => 'node',
 391     protected => 1,
 392     permissions => {
 393         check => ['perm', '/', [ 'Sys.Modify' ]],
 394     },
 395     parameters => {
 396         additionalProperties => 0,
 397         properties => {
 398             node => get_standard_option('pve-node'),
 399             osdid => {
 400                 description => 'OSD ID',
 401                 type => 'integer',
 402             },
 403         },
 404     },
 405     returns => { type => "null" },
 406     code => sub {
 407         my ($param) = @_;
 408
 409         PVE::CephTools::check_ceph_inited();
 410
 411         my $osdid = $param->{osdid};
 412
 413         my $rados = PVE::RADOS->new();
 414
 415         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 416
 417         my $osdsection = "osd.$osdid";
 418
 419         $rados->mon_command({ prefix => "osd in", ids => [ $osdsection ], format => 'plain' });
 420
 421         return undef;
 422     }});
 423
 424 __PACKAGE__->register_method ({
 425     name => 'out',
 426     path => '{osdid}/out',
 427     method => 'POST',
 428     description => "ceph osd out",
 429     proxyto => 'node',
 430     protected => 1,
 431     permissions => {
 432         check => ['perm', '/', [ 'Sys.Modify' ]],
 433     },
 434     parameters => {
 435         additionalProperties => 0,
 436         properties => {
 437             node => get_standard_option('pve-node'),
 438             osdid => {
 439                 description => 'OSD ID',
 440                 type => 'integer',
 441             },
 442         },
 443     },
 444     returns => { type => "null" },
 445     code => sub {
 446         my ($param) = @_;
 447
 448         PVE::CephTools::check_ceph_inited();
 449
 450         my $osdid = $param->{osdid};
 451
 452         my $rados = PVE::RADOS->new();
 453
 454         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 455
 456         my $osdsection = "osd.$osdid";
 457
 458         $rados->mon_command({ prefix => "osd out", ids => [ $osdsection ], format => 'plain' });
 459
 460         return undef;
 461     }});
 462
 463 package PVE::API2::Ceph;
 464
 465 use strict;
 466 use warnings;
 467 use File::Basename;
 468 use File::Path;
 469 use POSIX qw (LONG_MAX);
 470 use Cwd qw(abs_path);
 471 use IO::Dir;
 472 use UUID;
 473 use Net::IP;
 474
 475 use PVE::SafeSyslog;
 476 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
 477 use PVE::Exception qw(raise raise_param_exc);
 478 use PVE::INotify;
 479 use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
 480 use PVE::AccessControl;
 481 use PVE::Storage;
 482 use PVE::RESTHandler;
 483 use PVE::RPCEnvironment;
 484 use PVE::JSONSchema qw(get_standard_option);
 485 use JSON;
 486 use PVE::RADOS;
 487 use PVE::CephTools;
 488
 489 use base qw(PVE::RESTHandler);
 490
 491 use Data::Dumper; # fixme: remove
 492
 493 my $pve_osd_default_journal_size = 1024*5;
 494
 495 __PACKAGE__->register_method ({
 496     subclass => "PVE::API2::CephOSD",
 497     path => 'osd',
 498 });
 499
 500 __PACKAGE__->register_method ({
 501     name => 'index',
 502     path => '',
 503     method => 'GET',
 504     description => "Directory index.",
 505     permissions => { user => 'all' },
 506     permissions => {
 507         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 508     },
 509     parameters => {
 510         additionalProperties => 0,
 511         properties => {
 512             node => get_standard_option('pve-node'),
 513         },
 514     },
 515     returns => {
 516         type => 'array',
 517         items => {
 518             type => "object",
 519             properties => {},
 520         },
 521         links => [ { rel => 'child', href => "{name}" } ],
 522     },
 523     code => sub {
 524         my ($param) = @_;
 525
 526         my $result = [
 527             { name => 'init' },
 528             { name => 'mon' },
 529             { name => 'osd' },
 530             { name => 'pools' },
 531             { name => 'stop' },
 532             { name => 'start' },
 533             { name => 'status' },
 534             { name => 'crush' },
 535             { name => 'config' },
 536             { name => 'log' },
 537             { name => 'disks' },
 538             { name => 'flags' },
 539         ];
 540
 541         return $result;
 542     }});
 543
 544 __PACKAGE__->register_method ({
 545     name => 'disks',
 546     path => 'disks',
 547     method => 'GET',
 548     description => "List local disks.",
 549     proxyto => 'node',
 550     protected => 1,
 551     permissions => {
 552         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 553     },
 554     parameters => {
 555         additionalProperties => 0,
 556         properties => {
 557             node => get_standard_option('pve-node'),
 558             type => {
 559                 description => "Only list specific types of disks.",
 560                 type => 'string',
 561                 enum => ['unused', 'journal_disks'],
 562                 optional => 1,
 563             },
 564         },
 565     },
 566     returns => {
 567         type => 'array',
 568         items => {
 569             type => "object",
 570             properties => {
 571                 dev => { type => 'string' },
 572                 used => { type => 'string', optional => 1 },
 573                 gpt => { type => 'boolean' },
 574                 size => { type => 'integer' },
 575                 osdid => { type => 'integer' },
 576                 vendor =>  { type => 'string', optional => 1 },
 577                 model =>  { type => 'string', optional => 1 },
 578                 serial =>  { type => 'string', optional => 1 },
 579             },
 580         },
 581         # links => [ { rel => 'child', href => "{}" } ],
 582     },
 583     code => sub {
 584         my ($param) = @_;
 585
 586         PVE::CephTools::check_ceph_inited();
 587
 588         my $disks = PVE::Diskmanage::get_disks(undef, 1);
 589
 590         my $res = [];
 591         foreach my $dev (keys %$disks) {
 592             my $d = $disks->{$dev};
 593             if ($param->{type}) {
 594                 if ($param->{type} eq 'journal_disks') {
 595                     next if $d->{osdid} >= 0;
 596                     next if !$d->{gpt};
 597                 } elsif ($param->{type} eq 'unused') {
 598                     next if $d->{used};
 599                 } else {
 600                     die "internal error"; # should not happen
 601                 }
 602             }
 603
 604             $d->{dev} = "/dev/$dev";
 605             push @$res, $d;
 606         }
 607
 608         return $res;
 609     }});
 610
 611 __PACKAGE__->register_method ({
 612     name => 'config',
 613     path => 'config',
 614     method => 'GET',
 615     permissions => {
 616         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 617     },
 618     description => "Get Ceph configuration.",
 619     parameters => {
 620         additionalProperties => 0,
 621         properties => {
 622             node => get_standard_option('pve-node'),
 623         },
 624     },
 625     returns => { type => 'string' },
 626     code => sub {
 627         my ($param) = @_;
 628
 629         PVE::CephTools::check_ceph_inited();
 630
 631         my $path = PVE::CephTools::get_config('pve_ceph_cfgpath');
 632         return PVE::Tools::file_get_contents($path);
 633
 634     }});
 635
 636 __PACKAGE__->register_method ({
 637     name => 'listmon',
 638     path => 'mon',
 639     method => 'GET',
 640     description => "Get Ceph monitor list.",
 641     proxyto => 'node',
 642     protected => 1,
 643     permissions => {
 644         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 645     },
 646     parameters => {
 647         additionalProperties => 0,
 648         properties => {
 649             node => get_standard_option('pve-node'),
 650         },
 651     },
 652     returns => {
 653         type => 'array',
 654         items => {
 655             type => "object",
 656             properties => {
 657                 name => { type => 'string' },
 658                 addr => { type => 'string' },
 659             },
 660         },
 661         links => [ { rel => 'child', href => "{name}" } ],
 662     },
 663     code => sub {
 664         my ($param) = @_;
 665
 666         PVE::CephTools::check_ceph_inited();
 667
 668         my $res = [];
 669
 670         my $cfg = PVE::CephTools::parse_ceph_config();
 671
 672         my $monhash = {};
 673         foreach my $section (keys %$cfg) {
 674             my $d = $cfg->{$section};
 675             if ($section =~ m/^mon\.(\S+)$/) {
 676                 my $monid = $1;
 677                 if ($d->{'mon addr'} && $d->{'host'}) {
 678                     $monhash->{$monid} = {
 679                         addr => $d->{'mon addr'},
 680                         host => $d->{'host'},
 681                         name => $monid,
 682                     }
 683                 }
 684             }
 685         }
 686
 687         eval {
 688             my $rados = PVE::RADOS->new();
 689             my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 690             my $mons = $monstat->{monmap}->{mons};
 691             foreach my $d (@$mons) {
 692                 next if !defined($d->{name});
 693                 $monhash->{$d->{name}}->{rank} = $d->{rank};
 694                 $monhash->{$d->{name}}->{addr} = $d->{addr};
 695                 if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
 696                     $monhash->{$d->{name}}->{quorum} = 1;
 697                 }
 698             }
 699         };
 700         warn $@ if $@;
 701
 702         return PVE::RESTHandler::hash_to_array($monhash, 'name');
 703     }});
 704
 705 __PACKAGE__->register_method ({
 706     name => 'init',
 707     path => 'init',
 708     method => 'POST',
 709     description => "Create initial ceph default configuration and setup symlinks.",
 710     proxyto => 'node',
 711     protected => 1,
 712     permissions => {
 713         check => ['perm', '/', [ 'Sys.Modify' ]],
 714     },
 715     parameters => {
 716         additionalProperties => 0,
 717         properties => {
 718             node => get_standard_option('pve-node'),
 719             network => {
 720                 description => "Use specific network for all ceph related traffic",
 721                 type => 'string', format => 'CIDR',
 722                 optional => 1,
 723                 maxLength => 128,
 724             },
 725             size => {
 726                 description => 'Number of replicas per object',
 727                 type => 'integer',
 728                 default => 2,
 729                 optional => 1,
 730                 minimum => 1,
 731                 maximum => 3,
 732             },
 733             pg_bits => {
 734                 description => "Placement group bits, used to specify the " .
 735                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 736                     "default pg num' does not work for default pools.",
 737                 type => 'integer',
 738                 default => 6,
 739                 optional => 1,
 740                 minimum => 6,
 741                 maximum => 14,
 742             },
 743         },
 744     },
 745     returns => { type => 'null' },
 746     code => sub {
 747         my ($param) = @_;
 748
 749         PVE::CephTools::check_ceph_installed();
 750
 751         # simply load old config if it already exists
 752         my $cfg = PVE::CephTools::parse_ceph_config();
 753
 754         if (!$cfg->{global}) {
 755
 756             my $fsid;
 757             my $uuid;
 758
 759             UUID::generate($uuid);
 760             UUID::unparse($uuid, $fsid);
 761
 762             $cfg->{global} = {
 763                 'fsid' => $fsid,
 764                 'auth cluster required' => 'cephx',
 765                 'auth service required' => 'cephx',
 766                 'auth client required' => 'cephx',
 767                 'filestore xattr use omap' => 'true',
 768                 'osd journal size' => $pve_osd_default_journal_size,
 769                 'osd pool default min size' => 1,
 770             };
 771
 772             # this does not work for default pools
 773             #'osd pool default pg num' => $pg_num,
 774             #'osd pool default pgp num' => $pg_num,
 775         }
 776
 777         $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 778         $cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring';
 779
 780         $cfg->{global}->{'osd pool default size'} = $param->{size} if $param->{size};
 781
 782         if ($param->{pg_bits}) {
 783             $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 784             $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 785         }
 786
 787         if ($param->{network}) {
 788             $cfg->{global}->{'public network'} = $param->{network};
 789             $cfg->{global}->{'cluster network'} = $param->{network};
 790         }
 791
 792         PVE::CephTools::write_ceph_config($cfg);
 793
 794         PVE::CephTools::setup_pve_symlinks();
 795
 796         return undef;
 797     }});
 798
 799 my $find_node_ip = sub {
 800     my ($cidr) = @_;
 801
 802     my $net = Net::IP->new($cidr) || die Net::IP::Error() . "\n";
 803     my $id = $net->version == 6 ? 'address6' : 'address';
 804
 805     my $config = PVE::INotify::read_file('interfaces');
 806     my $ifaces = $config->{ifaces};
 807
 808     foreach my $iface (keys %$ifaces) {
 809         my $d = $ifaces->{$iface};
 810         next if !$d->{$id};
 811         my $a = Net::IP->new($d->{$id});
 812         next if !$a;
 813         return $d->{$id} if $net->overlaps($a);
 814     }
 815
 816     die "unable to find local address within network '$cidr'\n";
 817 };
 818
 819 __PACKAGE__->register_method ({
 820     name => 'createmon',
 821     path => 'mon',
 822     method => 'POST',
 823     description => "Create Ceph Monitor",
 824     proxyto => 'node',
 825     protected => 1,
 826     permissions => {
 827         check => ['perm', '/', [ 'Sys.Modify' ]],
 828     },
 829     parameters => {
 830         additionalProperties => 0,
 831         properties => {
 832             node => get_standard_option('pve-node'),
 833         },
 834     },
 835     returns => { type => 'string' },
 836     code => sub {
 837         my ($param) = @_;
 838
 839         PVE::CephTools::check_ceph_inited();
 840
 841         PVE::CephTools::setup_pve_symlinks();
 842
 843         my $rpcenv = PVE::RPCEnvironment::get();
 844
 845         my $authuser = $rpcenv->get_user();
 846
 847         my $cfg = PVE::CephTools::parse_ceph_config();
 848
 849         my $moncount = 0;
 850
 851         my $monaddrhash = {};
 852
 853         foreach my $section (keys %$cfg) {
 854             next if $section eq 'global';
 855             my $d = $cfg->{$section};
 856             if ($section =~ m/^mon\./) {
 857                 $moncount++;
 858                 if ($d->{'mon addr'}) {
 859                     $monaddrhash->{$d->{'mon addr'}} = $section;
 860                 }
 861             }
 862         }
 863
 864         my $monid;
 865         for (my $i = 0; $i < 7; $i++) {
 866             if (!$cfg->{"mon.$i"}) {
 867                 $monid = $i;
 868                 last;
 869             }
 870         }
 871         die "unable to find usable monitor id\n" if !defined($monid);
 872
 873         my $monsection = "mon.$monid";
 874         my $ip;
 875         if (my $pubnet = $cfg->{global}->{'public network'}) {
 876             $ip = &$find_node_ip($pubnet);
 877         } else {
 878             $ip = PVE::Cluster::remote_node_ip($param->{node});
 879         }
 880
 881         my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]:6789" : "$ip:6789";
 882         my $monname = $param->{node};
 883
 884         die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
 885         die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n"
 886             if $monaddrhash->{$monaddr};
 887
 888         my $worker = sub  {
 889             my $upid = shift;
 890
 891             my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
 892
 893             if (! -f $pve_ckeyring_path) {
 894                 run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
 895                             "--gen-key -n client.admin");
 896             }
 897
 898             my $pve_mon_key_path = PVE::CephTools::get_config('pve_mon_key_path');
 899             if (! -f $pve_mon_key_path) {
 900                 run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
 901                 run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " .
 902                             "--cap mds 'allow' " .
 903                             "--cap osd 'allow *' " .
 904                             "--cap mon 'allow *'");
 905                 run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
 906                 run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
 907             }
 908
 909             my $ccname = PVE::CephTools::get_config('ccname');
 910
 911             my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 912             -d $mondir && die "monitor filesystem '$mondir' already exist\n";
 913
 914             my $monmap = "/tmp/monmap";
 915
 916             eval {
 917                 mkdir $mondir;
 918
 919                 if ($moncount > 0) {
 920                     my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
 921                     my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
 922                     PVE::Tools::file_set_contents($monmap, $mapdata);
 923                 } else {
 924                     run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
 925                 }
 926
 927                 run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
 928             };
 929             my $err = $@;
 930             unlink $monmap;
 931             if ($err) {
 932                 File::Path::remove_tree($mondir);
 933                 die $err;
 934             }
 935
 936             $cfg->{$monsection} = {
 937                 'host' => $monname,
 938                 'mon addr' => $monaddr,
 939             };
 940
 941             PVE::CephTools::write_ceph_config($cfg);
 942
 943             PVE::CephTools::ceph_service_cmd('start', $monsection);
 944         };
 945
 946         return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
 947     }});
 948
 949 __PACKAGE__->register_method ({
 950     name => 'destroymon',
 951     path => 'mon/{monid}',
 952     method => 'DELETE',
 953     description => "Destroy Ceph monitor.",
 954     proxyto => 'node',
 955     protected => 1,
 956     permissions => {
 957         check => ['perm', '/', [ 'Sys.Modify' ]],
 958     },
 959     parameters => {
 960         additionalProperties => 0,
 961         properties => {
 962             node => get_standard_option('pve-node'),
 963             monid => {
 964                 description => 'Monitor ID',
 965                 type => 'integer',
 966             },
 967         },
 968     },
 969     returns => { type => 'string' },
 970     code => sub {
 971         my ($param) = @_;
 972
 973         my $rpcenv = PVE::RPCEnvironment::get();
 974
 975         my $authuser = $rpcenv->get_user();
 976
 977         PVE::CephTools::check_ceph_inited();
 978
 979         my $cfg = PVE::CephTools::parse_ceph_config();
 980
 981         my $monid = $param->{monid};
 982         my $monsection = "mon.$monid";
 983
 984         my $rados = PVE::RADOS->new();
 985         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 986         my $monlist = $monstat->{monmap}->{mons};
 987
 988         die "no such monitor id '$monid'\n"
 989             if !defined($cfg->{$monsection});
 990
 991         my $ccname = PVE::CephTools::get_config('ccname');
 992
 993         my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 994         -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n";
 995
 996         die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
 997
 998         my $worker = sub {
 999             my $upid = shift;
1000
1001             # reopen with longer timeout
1002             $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1003
1004             $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });
1005
1006             eval { PVE::CephTools::ceph_service_cmd('stop', $monsection); };
1007             warn $@ if $@;
1008
1009             delete $cfg->{$monsection};
1010             PVE::CephTools::write_ceph_config($cfg);
1011             File::Path::remove_tree($mondir);
1012         };
1013
1014         return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
1015     }});
1016
1017 __PACKAGE__->register_method ({
1018     name => 'stop',
1019     path => 'stop',
1020     method => 'POST',
1021     description => "Stop ceph services.",
1022     proxyto => 'node',
1023     protected => 1,
1024     permissions => {
1025         check => ['perm', '/', [ 'Sys.Modify' ]],
1026     },
1027     parameters => {
1028         additionalProperties => 0,
1029         properties => {
1030             node => get_standard_option('pve-node'),
1031             service => {
1032                 description => 'Ceph service name.',
1033                 type => 'string',
1034                 optional => 1,
1035                 pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}',
1036             },
1037         },
1038     },
1039     returns => { type => 'string' },
1040     code => sub {
1041         my ($param) = @_;
1042
1043         my $rpcenv = PVE::RPCEnvironment::get();
1044
1045         my $authuser = $rpcenv->get_user();
1046
1047         PVE::CephTools::check_ceph_inited();
1048
1049         my $cfg = PVE::CephTools::parse_ceph_config();
1050         scalar(keys %$cfg) || die "no configuration\n";
1051
1052         my $worker = sub {
1053             my $upid = shift;
1054
1055             my $cmd = ['stop'];
1056             if ($param->{service}) {
1057                 push @$cmd, $param->{service};
1058             }
1059
1060             PVE::CephTools::ceph_service_cmd(@$cmd);
1061         };
1062
1063         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
1064                                     $authuser, $worker);
1065     }});
1066
1067 __PACKAGE__->register_method ({
1068     name => 'start',
1069     path => 'start',
1070     method => 'POST',
1071     description => "Start ceph services.",
1072     proxyto => 'node',
1073     protected => 1,
1074     permissions => {
1075         check => ['perm', '/', [ 'Sys.Modify' ]],
1076     },
1077     parameters => {
1078         additionalProperties => 0,
1079         properties => {
1080             node => get_standard_option('pve-node'),
1081             service => {
1082                 description => 'Ceph service name.',
1083                 type => 'string',
1084                 optional => 1,
1085                 pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}',
1086             },
1087         },
1088     },
1089     returns => { type => 'string' },
1090     code => sub {
1091         my ($param) = @_;
1092
1093         my $rpcenv = PVE::RPCEnvironment::get();
1094
1095         my $authuser = $rpcenv->get_user();
1096
1097         PVE::CephTools::check_ceph_inited();
1098
1099         my $cfg = PVE::CephTools::parse_ceph_config();
1100         scalar(keys %$cfg) || die "no configuration\n";
1101
1102         my $worker = sub {
1103             my $upid = shift;
1104
1105             my $cmd = ['start'];
1106             if ($param->{service}) {
1107                 push @$cmd, $param->{service};
1108             }
1109
1110             PVE::CephTools::ceph_service_cmd(@$cmd);
1111         };
1112
1113         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
1114                                     $authuser, $worker);
1115     }});
1116
1117 __PACKAGE__->register_method ({
1118     name => 'status',
1119     path => 'status',
1120     method => 'GET',
1121     description => "Get ceph status.",
1122     proxyto => 'node',
1123     protected => 1,
1124     permissions => {
1125         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1126     },
1127     parameters => {
1128         additionalProperties => 0,
1129         properties => {
1130             node => get_standard_option('pve-node'),
1131         },
1132     },
1133     returns => { type => 'object' },
1134     code => sub {
1135         my ($param) = @_;
1136
1137         PVE::CephTools::check_ceph_enabled();
1138
1139         my $rados = PVE::RADOS->new();
1140         return $rados->mon_command({ prefix => 'status' });
1141     }});
1142
1143 __PACKAGE__->register_method ({
1144     name => 'lspools',
1145     path => 'pools',
1146     method => 'GET',
1147     description => "List all pools.",
1148     proxyto => 'node',
1149     protected => 1,
1150     permissions => {
1151         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1152     },
1153     parameters => {
1154         additionalProperties => 0,
1155         properties => {
1156             node => get_standard_option('pve-node'),
1157         },
1158     },
1159     returns => {
1160         type => 'array',
1161         items => {
1162             type => "object",
1163             properties => {
1164                 pool => { type => 'integer' },
1165                 pool_name => { type => 'string' },
1166                 size => { type => 'integer' },
1167             },
1168         },
1169         links => [ { rel => 'child', href => "{pool_name}" } ],
1170     },
1171     code => sub {
1172         my ($param) = @_;
1173
1174         PVE::CephTools::check_ceph_inited();
1175
1176         my $rados = PVE::RADOS->new();
1177
1178         my $stats = {};
1179         my $res = $rados->mon_command({ prefix => 'df' });
1180         my $total = $res->{stats}->{total_avail_bytes} || 0;
1181
1182         foreach my $d (@{$res->{pools}}) {
1183             next if !$d->{stats};
1184             next if !defined($d->{id});
1185             $stats->{$d->{id}} = $d->{stats};
1186         }
1187
1188         $res = $rados->mon_command({ prefix => 'osd dump' });
1189
1190         my $data = [];
1191         foreach my $e (@{$res->{pools}}) {
1192             my $d = {};
1193             foreach my $attr (qw(pool pool_name size min_size pg_num crush_ruleset)) {
1194                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
1195             }
1196             if (my $s = $stats->{$d->{pool}}) {
1197                 $d->{bytes_used} = $s->{bytes_used};
1198                 $d->{percent_used} = ($s->{bytes_used} / $total)*100
1199                     if $s->{max_avail} && $total;
1200             }
1201             push @$data, $d;
1202         }
1203
1204
1205         return $data;
1206     }});
1207
1208 __PACKAGE__->register_method ({
1209     name => 'createpool',
1210     path => 'pools',
1211     method => 'POST',
1212     description => "Create POOL",
1213     proxyto => 'node',
1214     protected => 1,
1215     permissions => {
1216         check => ['perm', '/', [ 'Sys.Modify' ]],
1217     },
1218     parameters => {
1219         additionalProperties => 0,
1220         properties => {
1221             node => get_standard_option('pve-node'),
1222             name => {
1223                 description => "The name of the pool. It must be unique.",
1224                 type => 'string',
1225             },
1226             size => {
1227                 description => 'Number of replicas per object',
1228                 type => 'integer',
1229                 default => 2,
1230                 optional => 1,
1231                 minimum => 1,
1232                 maximum => 3,
1233             },
1234             min_size => {
1235                 description => 'Minimum number of replicas per object',
1236                 type => 'integer',
1237                 default => 1,
1238                 optional => 1,
1239                 minimum => 1,
1240                 maximum => 3,
1241             },
1242             pg_num => {
1243                 description => "Number of placement groups.",
1244                 type => 'integer',
1245                 default => 64,
1246                 optional => 1,
1247                 minimum => 8,
1248                 maximum => 32768,
1249             },
1250             crush_ruleset => {
1251                 description => "The ruleset to use for mapping object placement in the cluster.",
1252                 type => 'integer',
1253                 minimum => 0,
1254                 maximum => 32768,
1255                 default => 0,
1256                 optional => 1,
1257             },
1258         },
1259     },
1260     returns => { type => 'null' },
1261     code => sub {
1262         my ($param) = @_;
1263
1264         PVE::CephTools::check_ceph_inited();
1265
1266         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1267
1268         die "not fully configured - missing '$pve_ckeyring_path'\n"
1269             if ! -f $pve_ckeyring_path;
1270
1271         my $pg_num = $param->{pg_num} || 64;
1272         my $size = $param->{size} || 2;
1273         my $min_size = $param->{min_size} || 1;
1274         my $ruleset = $param->{crush_ruleset} || 0;
1275         my $rados = PVE::RADOS->new();
1276
1277         $rados->mon_command({
1278             prefix => "osd pool create",
1279             pool => $param->{name},
1280             pg_num => int($pg_num),
1281 # this does not work for unknown reason
1282 #           properties => ["size=$size", "min_size=$min_size", "crush_ruleset=$ruleset"],
1283             format => 'plain',
1284         });
1285
1286         $rados->mon_command({
1287             prefix => "osd pool set",
1288             pool => $param->{name},
1289             var => 'min_size',
1290             val => $min_size,
1291             format => 'plain',
1292         });
1293
1294         $rados->mon_command({
1295             prefix => "osd pool set",
1296             pool => $param->{name},
1297             var => 'size',
1298             val => $size,
1299             format => 'plain',
1300         });
1301
1302         if (defined($param->{crush_ruleset})) {
1303             $rados->mon_command({
1304                 prefix => "osd pool set",
1305                 pool => $param->{name},
1306                 var => 'crush_ruleset',
1307                 val => $param->{crush_ruleset},
1308                 format => 'plain',
1309             });
1310         }
1311
1312         return undef;
1313     }});
1314
1315 __PACKAGE__->register_method ({
1316     name => 'get_flags',
1317     path => 'flags',
1318     method => 'GET',
1319     description => "get all set ceph flags",
1320     proxyto => 'node',
1321     protected => 1,
1322     permissions => {
1323         check => ['perm', '/', [ 'Sys.Audit' ]],
1324     },
1325     parameters => {
1326         additionalProperties => 0,
1327         properties => {
1328             node => get_standard_option('pve-node'),
1329         },
1330     },
1331     returns => { type => 'string' },
1332     code => sub {
1333         my ($param) = @_;
1334
1335         PVE::CephTools::check_ceph_inited();
1336
1337         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1338
1339         die "not fully configured - missing '$pve_ckeyring_path'\n"
1340             if ! -f $pve_ckeyring_path;
1341
1342         my $rados = PVE::RADOS->new();
1343
1344         my $stat = $rados->mon_command({ prefix => 'osd dump' });
1345
1346         return $stat->{flags} // '';
1347     }});
1348
1349 __PACKAGE__->register_method ({
1350     name => 'set_flag',
1351     path => 'flags/{flag}',
1352     method => 'POST',
1353     description => "Set a ceph flag",
1354     proxyto => 'node',
1355     protected => 1,
1356     permissions => {
1357         check => ['perm', '/', [ 'Sys.Modify' ]],
1358     },
1359     parameters => {
1360         additionalProperties => 0,
1361         properties => {
1362             node => get_standard_option('pve-node'),
1363             flag => {
1364                 description => 'The ceph flag to set/unset',
1365                 type => 'string',
1366                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1367             },
1368         },
1369     },
1370     returns => { type => 'null' },
1371     code => sub {
1372         my ($param) = @_;
1373
1374         PVE::CephTools::check_ceph_inited();
1375
1376         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1377
1378         die "not fully configured - missing '$pve_ckeyring_path'\n"
1379             if ! -f $pve_ckeyring_path;
1380
1381         my $set = $param->{set} // !$param->{unset};
1382         my $rados = PVE::RADOS->new();
1383
1384         $rados->mon_command({
1385             prefix => "osd set",
1386             key => $param->{flag},
1387         });
1388
1389         return undef;
1390     }});
1391
1392 __PACKAGE__->register_method ({
1393     name => 'unset_flag',
1394     path => 'flags/{flag}',
1395     method => 'DELETE',
1396     description => "Unset a ceph flag",
1397     proxyto => 'node',
1398     protected => 1,
1399     permissions => {
1400         check => ['perm', '/', [ 'Sys.Modify' ]],
1401     },
1402     parameters => {
1403         additionalProperties => 0,
1404         properties => {
1405             node => get_standard_option('pve-node'),
1406             flag => {
1407                 description => 'The ceph flag to set/unset',
1408                 type => 'string',
1409                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1410             },
1411         },
1412     },
1413     returns => { type => 'null' },
1414     code => sub {
1415         my ($param) = @_;
1416
1417         PVE::CephTools::check_ceph_inited();
1418
1419         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1420
1421         die "not fully configured - missing '$pve_ckeyring_path'\n"
1422             if ! -f $pve_ckeyring_path;
1423
1424         my $set = $param->{set} // !$param->{unset};
1425         my $rados = PVE::RADOS->new();
1426
1427         $rados->mon_command({
1428             prefix => "osd unset",
1429             key => $param->{flag},
1430         });
1431
1432         return undef;
1433     }});
1434
1435 __PACKAGE__->register_method ({
1436     name => 'destroypool',
1437     path => 'pools/{name}',
1438     method => 'DELETE',
1439     description => "Destroy pool",
1440     proxyto => 'node',
1441     protected => 1,
1442     permissions => {
1443         check => ['perm', '/', [ 'Sys.Modify' ]],
1444     },
1445     parameters => {
1446         additionalProperties => 0,
1447         properties => {
1448             node => get_standard_option('pve-node'),
1449             name => {
1450                 description => "The name of the pool. It must be unique.",
1451                 type => 'string',
1452             },
1453             force => {
1454                 description => "If true, destroys pool even if in use",
1455                 type => 'boolean',
1456                 optional => 1,
1457                 default => 0,
1458             }
1459         },
1460     },
1461     returns => { type => 'null' },
1462     code => sub {
1463         my ($param) = @_;
1464
1465         PVE::CephTools::check_ceph_inited();
1466
1467         # if not forced, destroy ceph pool only when no
1468         # vm disks are on it anymore
1469         if (!$param->{force}) {
1470             my $storagecfg = PVE::Storage::config();
1471             foreach my $storageid (keys %{$storagecfg->{ids}}) {
1472                 my $storage = $storagecfg->{ids}->{$storageid};
1473                 next if $storage->{type} ne 'rbd';
1474                 next if $storage->{pool} ne $param->{name};
1475
1476                 # check if any vm disks are on the pool
1477                 my $res = PVE::Storage::vdisk_list($storagecfg, $storageid);
1478                 die "ceph pool '$param->{name}' still in use by storage '$storageid'\n"
1479                     if @{$res->{$storageid}} != 0;
1480             }
1481         }
1482
1483         my $rados = PVE::RADOS->new();
1484         # fixme: '--yes-i-really-really-mean-it'
1485         $rados->mon_command({
1486             prefix => "osd pool delete",
1487             pool => $param->{name},
1488             pool2 => $param->{name},
1489             sure => '--yes-i-really-really-mean-it',
1490             format => 'plain',
1491         });
1492
1493         return undef;
1494     }});
1495
1496
1497 __PACKAGE__->register_method ({
1498     name => 'crush',
1499     path => 'crush',
1500     method => 'GET',
1501     description => "Get OSD crush map",
1502     proxyto => 'node',
1503     protected => 1,
1504     permissions => {
1505         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1506     },
1507     parameters => {
1508         additionalProperties => 0,
1509         properties => {
1510             node => get_standard_option('pve-node'),
1511         },
1512     },
1513     returns => { type => 'string' },
1514     code => sub {
1515         my ($param) = @_;
1516
1517         PVE::CephTools::check_ceph_inited();
1518
1519         # this produces JSON (difficult to read for the user)
1520         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
1521
1522         my $txt = '';
1523
1524         my $mapfile = "/var/tmp/ceph-crush.map.$$";
1525         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
1526
1527         my $rados = PVE::RADOS->new();
1528
1529         eval {
1530             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
1531             PVE::Tools::file_set_contents($mapfile, $bindata);
1532             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
1533             $txt = PVE::Tools::file_get_contents($mapdata);
1534         };
1535         my $err = $@;
1536
1537         unlink $mapfile;
1538         unlink $mapdata;
1539
1540         die $err if $err;
1541
1542         return $txt;
1543     }});
1544
1545 __PACKAGE__->register_method({
1546     name => 'log',
1547     path => 'log',
1548     method => 'GET',
1549     description => "Read ceph log",
1550     proxyto => 'node',
1551     permissions => {
1552         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
1553     },
1554     protected => 1,
1555     parameters => {
1556         additionalProperties => 0,
1557         properties => {
1558             node => get_standard_option('pve-node'),
1559             start => {
1560                 type => 'integer',
1561                 minimum => 0,
1562                 optional => 1,
1563             },
1564             limit => {
1565                 type => 'integer',
1566                 minimum => 0,
1567                 optional => 1,
1568             },
1569         },
1570     },
1571     returns => {
1572         type => 'array',
1573         items => {
1574             type => "object",
1575             properties => {
1576                 n => {
1577                   description=>  "Line number",
1578                   type=> 'integer',
1579                 },
1580                 t => {
1581                   description=>  "Line text",
1582                   type => 'string',
1583                 }
1584             }
1585         }
1586     },
1587     code => sub {
1588         my ($param) = @_;
1589
1590         my $rpcenv = PVE::RPCEnvironment::get();
1591         my $user = $rpcenv->get_user();
1592         my $node = $param->{node};
1593
1594         my $logfile = "/var/log/ceph/ceph.log";
1595         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
1596
1597         $rpcenv->set_result_attrib('total', $count);
1598
1599         return $lines;
1600     }});
1601
1602