PVE/API2/Ceph.pm

   1 package PVE::API2::CephOSD;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd qw(abs_path);
   7 use IO::File;
   8
   9 use PVE::CephTools;
  10 use PVE::Diskmanage;
  11 use PVE::Exception qw(raise_param_exc);
  12 use PVE::JSONSchema qw(get_standard_option);
  13 use PVE::RADOS;
  14 use PVE::RESTHandler;
  15 use PVE::RPCEnvironment;
  16 use PVE::Tools qw(run_command);
  17
  18 use base qw(PVE::RESTHandler);
  19
  20 my $get_osd_status = sub {
  21     my ($rados, $osdid) = @_;
  22
  23     my $stat = $rados->mon_command({ prefix => 'osd dump' });
  24
  25     my $osdlist = $stat->{osds} || [];
  26
  27     my $flags = $stat->{flags} || undef;
  28
  29     my $osdstat;
  30     foreach my $d (@$osdlist) {
  31         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  32     }
  33     if (defined($osdid)) {
  34         die "no such OSD '$osdid'\n" if !$osdstat->{$osdid};
  35         return $osdstat->{$osdid};
  36     }
  37
  38     return wantarray? ($osdstat, $flags):$osdstat;
  39 };
  40
  41 my $get_osd_usage = sub {
  42     my ($rados) = @_;
  43
  44     my $osdlist = $rados->mon_command({ prefix => 'pg dump',
  45                                         dumpcontents => [ 'osds' ]}) || [];
  46
  47     my $osdstat;
  48     foreach my $d (@$osdlist) {
  49         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  50     }
  51
  52     return $osdstat;
  53 };
  54
  55 __PACKAGE__->register_method ({
  56     name => 'index',
  57     path => '',
  58     method => 'GET',
  59     description => "Get Ceph osd list/tree.",
  60     proxyto => 'node',
  61     protected => 1,
  62     permissions => {
  63         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  64     },
  65     parameters => {
  66         additionalProperties => 0,
  67         properties => {
  68             node => get_standard_option('pve-node'),
  69         },
  70     },
  71     # fixme: return a list instead of extjs tree format ?
  72     returns => {
  73         type => "object",
  74     },
  75     code => sub {
  76         my ($param) = @_;
  77
  78         PVE::CephTools::check_ceph_inited();
  79
  80         my $rados = PVE::RADOS->new();
  81         my $res = $rados->mon_command({ prefix => 'osd tree' });
  82
  83         die "no tree nodes found\n" if !($res && $res->{nodes});
  84
  85         my ($osdhash, $flags) = &$get_osd_status($rados);
  86
  87         my $usagehash = &$get_osd_usage($rados);
  88
  89         my $osdmetadata_tmp = $rados->mon_command({ prefix => 'osd metadata' });
  90
  91         my $osdmetadata = {};
  92         foreach my $osd (@$osdmetadata_tmp) {
  93             $osdmetadata->{$osd->{id}} = $osd;
  94         }
  95
  96         my $nodes = {};
  97         my $newnodes = {};
  98         foreach my $e (@{$res->{nodes}}) {
  99             $nodes->{$e->{id}} = $e;
 100
 101             my $new = {
 102                 id => $e->{id},
 103                 name => $e->{name},
 104                 type => $e->{type}
 105             };
 106
 107             foreach my $opt (qw(status crush_weight reweight device_class)) {
 108                 $new->{$opt} = $e->{$opt} if defined($e->{$opt});
 109             }
 110
 111             if (my $stat = $osdhash->{$e->{id}}) {
 112                 $new->{in} = $stat->{in} if defined($stat->{in});
 113             }
 114
 115             if (my $stat = $usagehash->{$e->{id}}) {
 116                 $new->{total_space} = ($stat->{kb} || 1) * 1024;
 117                 $new->{bytes_used} = ($stat->{kb_used} || 0) * 1024;
 118                 $new->{percent_used} = ($new->{bytes_used}*100)/$new->{total_space};
 119                 if (my $d = $stat->{perf_stat}) {
 120                     $new->{commit_latency_ms} = $d->{commit_latency_ms};
 121                     $new->{apply_latency_ms} = $d->{apply_latency_ms};
 122                 }
 123             }
 124
 125             my $osdmd = $osdmetadata->{$e->{id}};
 126             if ($e->{type} eq 'osd' && $osdmd) {
 127                 if ($osdmd->{bluefs}) {
 128                     $new->{osdtype} = 'bluestore';
 129                     $new->{blfsdev} = $osdmd->{bluestore_bdev_dev_node};
 130                     $new->{dbdev} = $osdmd->{bluefs_db_dev_node};
 131                     $new->{waldev} = $osdmd->{bluefs_wal_dev_node};
 132                 } else {
 133                     $new->{osdtype} = 'filestore';
 134                 }
 135             }
 136
 137             $newnodes->{$e->{id}} = $new;
 138         }
 139
 140         foreach my $e (@{$res->{nodes}}) {
 141             my $new = $newnodes->{$e->{id}};
 142             if ($e->{children} && scalar(@{$e->{children}})) {
 143                 $new->{children} = [];
 144                 $new->{leaf} = 0;
 145                 foreach my $cid (@{$e->{children}}) {
 146                     $nodes->{$cid}->{parent} = $e->{id};
 147                     if ($nodes->{$cid}->{type} eq 'osd' &&
 148                         $e->{type} eq 'host') {
 149                         $newnodes->{$cid}->{host} = $e->{name};
 150                     }
 151                     push @{$new->{children}}, $newnodes->{$cid};
 152                 }
 153             } else {
 154                 $new->{leaf} = ($e->{id} >= 0) ? 1 : 0;
 155             }
 156         }
 157
 158         my $roots = [];
 159         foreach my $e (@{$res->{nodes}}) {
 160             if (!$nodes->{$e->{id}}->{parent}) {
 161                 push @$roots, $newnodes->{$e->{id}};
 162             }
 163         }
 164
 165         die "no root node\n" if !@$roots;
 166
 167         my $data = { root => { leaf =>  0, children => $roots } };
 168
 169         # we want this for the noout flag
 170         $data->{flags} = $flags if $flags;
 171
 172         return $data;
 173     }});
 174
 175 __PACKAGE__->register_method ({
 176     name => 'createosd',
 177     path => '',
 178     method => 'POST',
 179     description => "Create OSD",
 180     proxyto => 'node',
 181     protected => 1,
 182     parameters => {
 183         additionalProperties => 0,
 184         properties => {
 185             node => get_standard_option('pve-node'),
 186             dev => {
 187                 description => "Block device name.",
 188                 type => 'string',
 189             },
 190             journal_dev => {
 191                 description => "Block device name for journal (filestore) or block.db (bluestore).",
 192                 optional => 1,
 193                 type => 'string',
 194             },
 195             wal_dev => {
 196                 description => "Block device name for block.wal (bluestore only).",
 197                 optional => 1,
 198                 type => 'string',
 199             },
 200             fstype => {
 201                 description => "File system type (filestore only).",
 202                 type => 'string',
 203                 enum => ['xfs', 'ext4'],
 204                 default => 'xfs',
 205                 optional => 1,
 206             },
 207             bluestore => {
 208                 description => "Use bluestore instead of filestore. This is the default.",
 209                 type => 'boolean',
 210                 default => 1,
 211                 optional => 1,
 212             },
 213         },
 214     },
 215     returns => { type => 'string' },
 216     code => sub {
 217         my ($param) = @_;
 218
 219         my $rpcenv = PVE::RPCEnvironment::get();
 220
 221         my $authuser = $rpcenv->get_user();
 222
 223         raise_param_exc({ 'bluestore' => "conflicts with parameter 'fstype'" })
 224             if (defined($param->{fstype}) && defined($param->{bluestore}) && $param->{bluestore});
 225
 226         PVE::CephTools::check_ceph_inited();
 227
 228         PVE::CephTools::setup_pve_symlinks();
 229
 230         PVE::CephTools::check_ceph_installed('ceph_osd');
 231
 232         my $bluestore = $param->{bluestore} // 1;
 233
 234         my $journal_dev;
 235         my $wal_dev;
 236
 237         if ($param->{journal_dev} && ($param->{journal_dev} ne $param->{dev})) {
 238             $journal_dev = PVE::Diskmanage::verify_blockdev_path($param->{journal_dev});
 239         }
 240
 241         if ($param->{wal_dev} &&
 242             ($param->{wal_dev} ne $param->{dev}) &&
 243             (!$param->{journal_dev} || $param->{wal_dev} ne $param->{journal_dev})) {
 244             raise_param_exc({ 'wal_dev' => "can only be set with paramater 'bluestore'"})
 245                 if !$bluestore;
 246             $wal_dev = PVE::Diskmanage::verify_blockdev_path($param->{wal_dev});
 247         }
 248
 249         $param->{dev} = PVE::Diskmanage::verify_blockdev_path($param->{dev});
 250
 251         my $devname = $param->{dev};
 252         $devname =~ s|/dev/||;
 253
 254         my $disklist = PVE::Diskmanage::get_disks($devname, 1);
 255
 256         my $diskinfo = $disklist->{$devname};
 257         die "unable to get device info for '$devname'\n"
 258             if !$diskinfo;
 259
 260         die "device '$param->{dev}' is in use\n"
 261             if $diskinfo->{used};
 262
 263         my $devpath = $diskinfo->{devpath};
 264         my $rados = PVE::RADOS->new();
 265         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 266         die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid};
 267
 268         my $fsid = $monstat->{monmap}->{fsid};
 269         $fsid = $1 if $fsid =~ m/^([0-9a-f\-]+)$/;
 270
 271         my $ceph_bootstrap_osd_keyring = PVE::CephTools::get_config('ceph_bootstrap_osd_keyring');
 272
 273         if (! -f $ceph_bootstrap_osd_keyring) {
 274             my $bindata = $rados->mon_command({ prefix => 'auth get', entity => 'client.bootstrap-osd', format => 'plain' });
 275             file_set_contents($ceph_bootstrap_osd_keyring, $bindata);
 276         };
 277
 278         my $worker = sub {
 279             my $upid = shift;
 280
 281             my $fstype = $param->{fstype} || 'xfs';
 282
 283
 284             my $ccname = PVE::CephTools::get_config('ccname');
 285
 286             my $cmd = ['ceph-disk', 'prepare', '--zap-disk',
 287                        '--cluster', $ccname, '--cluster-uuid', $fsid ];
 288
 289             if ($bluestore) {
 290                 print "create OSD on $devpath (bluestore)\n";
 291                 push @$cmd, '--bluestore';
 292
 293                 if ($journal_dev) {
 294                     print "using device '$journal_dev' for block.db\n";
 295                     push @$cmd, '--block.db', $journal_dev;
 296                 }
 297
 298                 if ($wal_dev) {
 299                     print "using device '$wal_dev' for block.wal\n";
 300                     push @$cmd, '--block.wal', $wal_dev;
 301                 }
 302
 303                 push @$cmd, $devpath;
 304             } else {
 305                 print "create OSD on $devpath ($fstype)\n";
 306                 push @$cmd, '--filestore', '--fs-type', $fstype;
 307                 if ($journal_dev) {
 308                     print "using device '$journal_dev' for journal\n";
 309                     push @$cmd, '--journal-dev', $devpath, $journal_dev;
 310                 } else {
 311                     push @$cmd, $devpath;
 312                 }
 313             }
 314
 315
 316             run_command($cmd);
 317         };
 318
 319         return $rpcenv->fork_worker('cephcreateosd', $devname,  $authuser, $worker);
 320     }});
 321
 322 __PACKAGE__->register_method ({
 323     name => 'destroyosd',
 324     path => '{osdid}',
 325     method => 'DELETE',
 326     description => "Destroy OSD",
 327     proxyto => 'node',
 328     protected => 1,
 329     parameters => {
 330         additionalProperties => 0,
 331         properties => {
 332             node => get_standard_option('pve-node'),
 333             osdid => {
 334                 description => 'OSD ID',
 335                 type => 'integer',
 336             },
 337             cleanup => {
 338                 description => "If set, we remove partition table entries.",
 339                 type => 'boolean',
 340                 optional => 1,
 341                 default => 0,
 342             },
 343         },
 344     },
 345     returns => { type => 'string' },
 346     code => sub {
 347         my ($param) = @_;
 348
 349         my $rpcenv = PVE::RPCEnvironment::get();
 350
 351         my $authuser = $rpcenv->get_user();
 352
 353         PVE::CephTools::check_ceph_inited();
 354
 355         my $osdid = $param->{osdid};
 356
 357         my $rados = PVE::RADOS->new();
 358         my $osdstat = &$get_osd_status($rados, $osdid);
 359
 360         die "osd is in use (in == 1)\n" if $osdstat->{in};
 361         #&$run_ceph_cmd(['osd', 'out', $osdid]);
 362
 363         die "osd is still runnung (up == 1)\n" if $osdstat->{up};
 364
 365         my $osdsection = "osd.$osdid";
 366
 367         my $worker = sub {
 368             my $upid = shift;
 369
 370             # reopen with longer timeout
 371             $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
 372
 373             print "destroy OSD $osdsection\n";
 374
 375             eval {
 376                 PVE::CephTools::ceph_service_cmd('stop', $osdsection);
 377                 PVE::CephTools::ceph_service_cmd('disable', $osdsection);
 378             };
 379             warn $@ if $@;
 380
 381             print "Remove $osdsection from the CRUSH map\n";
 382             $rados->mon_command({ prefix => "osd crush remove", name => $osdsection, format => 'plain' });
 383
 384             print "Remove the $osdsection authentication key.\n";
 385             $rados->mon_command({ prefix => "auth del", entity => $osdsection, format => 'plain' });
 386
 387             print "Remove OSD $osdsection\n";
 388             $rados->mon_command({ prefix => "osd rm", ids => [ $osdsection ], format => 'plain' });
 389
 390             # try to unmount from standard mount point
 391             my $mountpoint = "/var/lib/ceph/osd/ceph-$osdid";
 392
 393             my $disks_to_wipe = {};
 394             my $remove_partition = sub {
 395                 my ($part) = @_;
 396
 397                 return if !$part || (! -b $part );
 398                 my $partnum = PVE::Diskmanage::get_partnum($part);
 399                 my $devpath = PVE::Diskmanage::get_blockdev($part);
 400
 401                 print "remove partition $part (disk '${devpath}', partnum $partnum)\n";
 402                 eval { run_command(['/sbin/sgdisk', '-d', $partnum, "${devpath}"]); };
 403                 warn $@ if $@;
 404
 405                 $disks_to_wipe->{$devpath} = 1;
 406             };
 407
 408             my $partitions_to_remove = [];
 409
 410             if ($param->{cleanup}) {
 411                 if (my $fd = IO::File->new("/proc/mounts", "r")) {
 412                     while (defined(my $line = <$fd>)) {
 413                         my ($dev, $path, $fstype) = split(/\s+/, $line);
 414                         next if !($dev && $path && $fstype);
 415                         next if $dev !~ m|^/dev/|;
 416                         if ($path eq $mountpoint) {
 417                             my $data_part = abs_path($dev);
 418                             push @$partitions_to_remove, $data_part;
 419                             last;
 420                         }
 421                     }
 422                     close($fd);
 423                 }
 424
 425                 foreach my $path (qw(journal block block.db block.wal)) {
 426                     my $part = abs_path("$mountpoint/$path");
 427                     if ($part) {
 428                         push @$partitions_to_remove, $part;
 429                     }
 430                 }
 431             }
 432
 433             print "Unmount OSD $osdsection from  $mountpoint\n";
 434             eval { run_command(['/bin/umount', $mountpoint]); };
 435             if (my $err = $@) {
 436                 warn $err;
 437             } elsif ($param->{cleanup}) {
 438                 #be aware of the ceph udev rules which can remount.
 439                 foreach my $part (@$partitions_to_remove) {
 440                     $remove_partition->($part);
 441                 }
 442                 my @wipe_cmd = qw(/bin/dd if=/dev/zero bs=1M count=200 conv=fdatasync);
 443                 foreach my $devpath (keys %$disks_to_wipe) {
 444                     print "wipe disk: $devpath\n";
 445                     eval { run_command([@wipe_cmd, "of=${devpath}"]) };
 446                     warn $@ if $@;
 447                 }
 448             }
 449         };
 450
 451         return $rpcenv->fork_worker('cephdestroyosd', $osdsection,  $authuser, $worker);
 452     }});
 453
 454 __PACKAGE__->register_method ({
 455     name => 'in',
 456     path => '{osdid}/in',
 457     method => 'POST',
 458     description => "ceph osd in",
 459     proxyto => 'node',
 460     protected => 1,
 461     permissions => {
 462         check => ['perm', '/', [ 'Sys.Modify' ]],
 463     },
 464     parameters => {
 465         additionalProperties => 0,
 466         properties => {
 467             node => get_standard_option('pve-node'),
 468             osdid => {
 469                 description => 'OSD ID',
 470                 type => 'integer',
 471             },
 472         },
 473     },
 474     returns => { type => "null" },
 475     code => sub {
 476         my ($param) = @_;
 477
 478         PVE::CephTools::check_ceph_inited();
 479
 480         my $osdid = $param->{osdid};
 481
 482         my $rados = PVE::RADOS->new();
 483
 484         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 485
 486         my $osdsection = "osd.$osdid";
 487
 488         $rados->mon_command({ prefix => "osd in", ids => [ $osdsection ], format => 'plain' });
 489
 490         return undef;
 491     }});
 492
 493 __PACKAGE__->register_method ({
 494     name => 'out',
 495     path => '{osdid}/out',
 496     method => 'POST',
 497     description => "ceph osd out",
 498     proxyto => 'node',
 499     protected => 1,
 500     permissions => {
 501         check => ['perm', '/', [ 'Sys.Modify' ]],
 502     },
 503     parameters => {
 504         additionalProperties => 0,
 505         properties => {
 506             node => get_standard_option('pve-node'),
 507             osdid => {
 508                 description => 'OSD ID',
 509                 type => 'integer',
 510             },
 511         },
 512     },
 513     returns => { type => "null" },
 514     code => sub {
 515         my ($param) = @_;
 516
 517         PVE::CephTools::check_ceph_inited();
 518
 519         my $osdid = $param->{osdid};
 520
 521         my $rados = PVE::RADOS->new();
 522
 523         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 524
 525         my $osdsection = "osd.$osdid";
 526
 527         $rados->mon_command({ prefix => "osd out", ids => [ $osdsection ], format => 'plain' });
 528
 529         return undef;
 530     }});
 531
 532 package PVE::API2::Ceph;
 533
 534 use strict;
 535 use warnings;
 536
 537 use File::Path;
 538 use Net::IP;
 539 use UUID;
 540
 541 use PVE::CephTools;
 542 use PVE::Cluster;
 543 use PVE::JSONSchema qw(get_standard_option);
 544 use PVE::Network;
 545 use PVE::RADOS;
 546 use PVE::RESTHandler;
 547 use PVE::RPCEnvironment;
 548 use PVE::Storage;
 549 use PVE::Tools qw(run_command file_get_contents file_set_contents);
 550
 551 use PVE::API2::Ceph::FS;
 552 use PVE::API2::Ceph::MDS;
 553 use PVE::API2::Storage::Config;
 554
 555 use base qw(PVE::RESTHandler);
 556
 557 my $pve_osd_default_journal_size = 1024*5;
 558
 559 __PACKAGE__->register_method ({
 560     subclass => "PVE::API2::CephOSD",
 561     path => 'osd',
 562 });
 563
 564 __PACKAGE__->register_method ({
 565     subclass => "PVE::API2::Ceph::MDS",
 566     path => 'mds',
 567 });
 568
 569 __PACKAGE__->register_method ({
 570     subclass => "PVE::API2::Ceph::FS",
 571     path => 'fs',
 572 });
 573
 574 __PACKAGE__->register_method ({
 575     name => 'index',
 576     path => '',
 577     method => 'GET',
 578     description => "Directory index.",
 579     permissions => { user => 'all' },
 580     permissions => {
 581         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 582     },
 583     parameters => {
 584         additionalProperties => 0,
 585         properties => {
 586             node => get_standard_option('pve-node'),
 587         },
 588     },
 589     returns => {
 590         type => 'array',
 591         items => {
 592             type => "object",
 593             properties => {},
 594         },
 595         links => [ { rel => 'child', href => "{name}" } ],
 596     },
 597     code => sub {
 598         my ($param) = @_;
 599
 600         my $result = [
 601             { name => 'init' },
 602             { name => 'mon' },
 603             { name => 'osd' },
 604             { name => 'pools' },
 605             { name => 'fs' },
 606             { name => 'mds' },
 607             { name => 'stop' },
 608             { name => 'start' },
 609             { name => 'status' },
 610             { name => 'crush' },
 611             { name => 'config' },
 612             { name => 'log' },
 613             { name => 'disks' },
 614             { name => 'flags' },
 615             { name => 'rules' },
 616         ];
 617
 618         return $result;
 619     }});
 620
 621 __PACKAGE__->register_method ({
 622     name => 'disks',
 623     path => 'disks',
 624     method => 'GET',
 625     description => "List local disks.",
 626     proxyto => 'node',
 627     protected => 1,
 628     permissions => {
 629         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 630     },
 631     parameters => {
 632         additionalProperties => 0,
 633         properties => {
 634             node => get_standard_option('pve-node'),
 635             type => {
 636                 description => "Only list specific types of disks.",
 637                 type => 'string',
 638                 enum => ['unused', 'journal_disks'],
 639                 optional => 1,
 640             },
 641         },
 642     },
 643     returns => {
 644         type => 'array',
 645         items => {
 646             type => "object",
 647             properties => {
 648                 dev => { type => 'string' },
 649                 used => { type => 'string', optional => 1 },
 650                 gpt => { type => 'boolean' },
 651                 size => { type => 'integer' },
 652                 osdid => { type => 'integer' },
 653                 vendor =>  { type => 'string', optional => 1 },
 654                 model =>  { type => 'string', optional => 1 },
 655                 serial =>  { type => 'string', optional => 1 },
 656             },
 657         },
 658         # links => [ { rel => 'child', href => "{}" } ],
 659     },
 660     code => sub {
 661         my ($param) = @_;
 662
 663         PVE::CephTools::check_ceph_inited();
 664
 665         my $disks = PVE::Diskmanage::get_disks(undef, 1);
 666
 667         my $res = [];
 668         foreach my $dev (keys %$disks) {
 669             my $d = $disks->{$dev};
 670             if ($param->{type}) {
 671                 if ($param->{type} eq 'journal_disks') {
 672                     next if $d->{osdid} >= 0;
 673                     next if !$d->{gpt};
 674                 } elsif ($param->{type} eq 'unused') {
 675                     next if $d->{used};
 676                 } else {
 677                     die "internal error"; # should not happen
 678                 }
 679             }
 680
 681             $d->{dev} = "/dev/$dev";
 682             push @$res, $d;
 683         }
 684
 685         return $res;
 686     }});
 687
 688 __PACKAGE__->register_method ({
 689     name => 'config',
 690     path => 'config',
 691     method => 'GET',
 692     permissions => {
 693         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 694     },
 695     description => "Get Ceph configuration.",
 696     parameters => {
 697         additionalProperties => 0,
 698         properties => {
 699             node => get_standard_option('pve-node'),
 700         },
 701     },
 702     returns => { type => 'string' },
 703     code => sub {
 704         my ($param) = @_;
 705
 706         PVE::CephTools::check_ceph_inited();
 707
 708         my $path = PVE::CephTools::get_config('pve_ceph_cfgpath');
 709         return file_get_contents($path);
 710
 711     }});
 712
 713 my $add_storage = sub {
 714     my ($pool, $storeid) = @_;
 715
 716     my $storage_params = {
 717         type => 'rbd',
 718         pool => $pool,
 719         storage => $storeid,
 720         krbd => 0,
 721         content => 'rootdir,images',
 722     };
 723
 724     PVE::API2::Storage::Config->create($storage_params);
 725 };
 726
 727 my $get_storages = sub {
 728     my ($pool) = @_;
 729
 730     my $cfg = PVE::Storage::config();
 731
 732     my $storages = $cfg->{ids};
 733     my $res = {};
 734     foreach my $storeid (keys %$storages) {
 735         my $curr = $storages->{$storeid};
 736         $res->{$storeid} = $storages->{$storeid}
 737             if $curr->{type} eq 'rbd' && $pool eq $curr->{pool};
 738     }
 739
 740     return $res;
 741 };
 742
 743 __PACKAGE__->register_method ({
 744     name => 'listmon',
 745     path => 'mon',
 746     method => 'GET',
 747     description => "Get Ceph monitor list.",
 748     proxyto => 'node',
 749     protected => 1,
 750     permissions => {
 751         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 752     },
 753     parameters => {
 754         additionalProperties => 0,
 755         properties => {
 756             node => get_standard_option('pve-node'),
 757         },
 758     },
 759     returns => {
 760         type => 'array',
 761         items => {
 762             type => "object",
 763             properties => {
 764                 name => { type => 'string' },
 765                 addr => { type => 'string' },
 766             },
 767         },
 768         links => [ { rel => 'child', href => "{name}" } ],
 769     },
 770     code => sub {
 771         my ($param) = @_;
 772
 773         PVE::CephTools::check_ceph_inited();
 774
 775         my $res = [];
 776
 777         my $cfg = PVE::CephTools::parse_ceph_config();
 778
 779         my $monhash = {};
 780         foreach my $section (keys %$cfg) {
 781             my $d = $cfg->{$section};
 782             if ($section =~ m/^mon\.(\S+)$/) {
 783                 my $monid = $1;
 784                 if ($d->{'mon addr'} && $d->{'host'}) {
 785                     $monhash->{$monid} = {
 786                         addr => $d->{'mon addr'},
 787                         host => $d->{'host'},
 788                         name => $monid,
 789                     }
 790                 }
 791             }
 792         }
 793
 794         eval {
 795             my $rados = PVE::RADOS->new();
 796             my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 797             my $mons = $monstat->{monmap}->{mons};
 798             foreach my $d (@$mons) {
 799                 next if !defined($d->{name});
 800                 $monhash->{$d->{name}}->{rank} = $d->{rank};
 801                 $monhash->{$d->{name}}->{addr} = $d->{addr};
 802                 if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
 803                     $monhash->{$d->{name}}->{quorum} = 1;
 804                 }
 805             }
 806         };
 807         warn $@ if $@;
 808
 809         return PVE::RESTHandler::hash_to_array($monhash, 'name');
 810     }});
 811
 812 __PACKAGE__->register_method ({
 813     name => 'init',
 814     path => 'init',
 815     method => 'POST',
 816     description => "Create initial ceph default configuration and setup symlinks.",
 817     proxyto => 'node',
 818     protected => 1,
 819     permissions => {
 820         check => ['perm', '/', [ 'Sys.Modify' ]],
 821     },
 822     parameters => {
 823         additionalProperties => 0,
 824         properties => {
 825             node => get_standard_option('pve-node'),
 826             network => {
 827                 description => "Use specific network for all ceph related traffic",
 828                 type => 'string', format => 'CIDR',
 829                 optional => 1,
 830                 maxLength => 128,
 831             },
 832             size => {
 833                 description => 'Targeted number of replicas per object',
 834                 type => 'integer',
 835                 default => 3,
 836                 optional => 1,
 837                 minimum => 1,
 838                 maximum => 7,
 839             },
 840             min_size => {
 841                 description => 'Minimum number of available replicas per object to allow I/O',
 842                 type => 'integer',
 843                 default => 2,
 844                 optional => 1,
 845                 minimum => 1,
 846                 maximum => 7,
 847             },
 848             pg_bits => {
 849                 description => "Placement group bits, used to specify the " .
 850                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 851                     "default pg num' does not work for default pools.",
 852                 type => 'integer',
 853                 default => 6,
 854                 optional => 1,
 855                 minimum => 6,
 856                 maximum => 14,
 857             },
 858             disable_cephx => {
 859                 description => "Disable cephx authentification.\n\n" .
 860                     "WARNING: cephx is a security feature protecting against " .
 861                     "man-in-the-middle attacks. Only consider disabling cephx ".
 862                     "if your network is private!",
 863                 type => 'boolean',
 864                 optional => 1,
 865                 default => 0,
 866             },
 867         },
 868     },
 869     returns => { type => 'null' },
 870     code => sub {
 871         my ($param) = @_;
 872
 873         my $version = PVE::CephTools::get_local_version(1);
 874
 875         if (!$version || $version < 12) {
 876             die "Ceph Luminous required - please run 'pveceph install'\n";
 877         } else {
 878             PVE::CephTools::check_ceph_installed('ceph_bin');
 879         }
 880
 881         # simply load old config if it already exists
 882         my $cfg = PVE::CephTools::parse_ceph_config();
 883
 884         if (!$cfg->{global}) {
 885
 886             my $fsid;
 887             my $uuid;
 888
 889             UUID::generate($uuid);
 890             UUID::unparse($uuid, $fsid);
 891
 892             my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 893
 894             $cfg->{global} = {
 895                 'fsid' => $fsid,
 896                 'auth cluster required' => $auth,
 897                 'auth service required' => $auth,
 898                 'auth client required' => $auth,
 899                 'osd journal size' => $pve_osd_default_journal_size,
 900                 'osd pool default size' => $param->{size} // 3,
 901                 'osd pool default min size' => $param->{min_size} // 2,
 902                 'mon allow pool delete' => 'true',
 903             };
 904
 905             # this does not work for default pools
 906             #'osd pool default pg num' => $pg_num,
 907             #'osd pool default pgp num' => $pg_num,
 908         }
 909
 910         $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 911         $cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring';
 912
 913         if ($param->{pg_bits}) {
 914             $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 915             $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 916         }
 917
 918         if ($param->{network}) {
 919             $cfg->{global}->{'public network'} = $param->{network};
 920             $cfg->{global}->{'cluster network'} = $param->{network};
 921         }
 922
 923         PVE::CephTools::write_ceph_config($cfg);
 924
 925         PVE::CephTools::setup_pve_symlinks();
 926
 927         return undef;
 928     }});
 929
 930 my $find_mon_ip = sub {
 931     my ($pubnet, $node, $overwrite_ip) = @_;
 932
 933     if (!$pubnet) {
 934         return $overwrite_ip // PVE::Cluster::remote_node_ip($node);
 935     }
 936
 937     my $allowed_ips = PVE::Network::get_local_ip_from_cidr($pubnet);
 938     die "No IP configured and up from ceph public network '$pubnet'\n"
 939         if scalar(@$allowed_ips) < 1;
 940
 941     if (!$overwrite_ip) {
 942         if (scalar(@$allowed_ips) == 1) {
 943             return $allowed_ips->[0];
 944         }
 945         die "Multiple IPs for ceph public network '$pubnet' detected on $node:\n".
 946             join("\n", @$allowed_ips) ."\nuse 'mon-address' to specify one of them.\n";
 947     } else {
 948         if (grep { $_ eq $overwrite_ip } @$allowed_ips) {
 949             return $overwrite_ip;
 950         }
 951         die "Monitor IP '$overwrite_ip' not in ceph public network '$pubnet'\n"
 952             if !PVE::Network::is_ip_in_cidr($overwrite_ip, $pubnet);
 953
 954         die "Specified monitor IP '$overwrite_ip' not configured or up on $node!\n";
 955     }
 956 };
 957
 958 my $create_mgr = sub {
 959     my ($rados, $id) = @_;
 960
 961     my $clustername = PVE::CephTools::get_config('ccname');
 962     my $mgrdir = "/var/lib/ceph/mgr/$clustername-$id";
 963     my $mgrkeyring = "$mgrdir/keyring";
 964     my $mgrname = "mgr.$id";
 965
 966     die "ceph manager directory '$mgrdir' already exists\n"
 967         if -d $mgrdir;
 968
 969     print "creating manager directory '$mgrdir'\n";
 970     mkdir $mgrdir;
 971     print "creating keys for '$mgrname'\n";
 972     my $output = $rados->mon_command({ prefix => 'auth get-or-create',
 973                                        entity => $mgrname,
 974                                        caps => [
 975                                            mon => 'allow profile mgr',
 976                                            osd => 'allow *',
 977                                            mds => 'allow *',
 978                                        ],
 979                                        format => 'plain'});
 980     file_set_contents($mgrkeyring, $output);
 981
 982     print "setting owner for directory\n";
 983     run_command(["chown", 'ceph:ceph', '-R', $mgrdir]);
 984
 985     print "enabling service 'ceph-mgr\@$id.service'\n";
 986     PVE::CephTools::ceph_service_cmd('enable', $mgrname);
 987     print "starting service 'ceph-mgr\@$id.service'\n";
 988     PVE::CephTools::ceph_service_cmd('start', $mgrname);
 989 };
 990
 991 my $destroy_mgr = sub {
 992     my ($mgrid) = @_;
 993
 994     my $clustername = PVE::CephTools::get_config('ccname');
 995     my $mgrname = "mgr.$mgrid";
 996     my $mgrdir = "/var/lib/ceph/mgr/$clustername-$mgrid";
 997
 998     die "ceph manager directory '$mgrdir' not found\n"
 999         if ! -d $mgrdir;
1000
1001     print "disabling service 'ceph-mgr\@$mgrid.service'\n";
1002     PVE::CephTools::ceph_service_cmd('disable', $mgrname);
1003     print "stopping service 'ceph-mgr\@$mgrid.service'\n";
1004     PVE::CephTools::ceph_service_cmd('stop', $mgrname);
1005
1006     print "removing manager directory '$mgrdir'\n";
1007     File::Path::remove_tree($mgrdir);
1008 };
1009
1010 __PACKAGE__->register_method ({
1011     name => 'createmon',
1012     path => 'mon',
1013     method => 'POST',
1014     description => "Create Ceph Monitor and Manager",
1015     proxyto => 'node',
1016     protected => 1,
1017     permissions => {
1018         check => ['perm', '/', [ 'Sys.Modify' ]],
1019     },
1020     parameters => {
1021         additionalProperties => 0,
1022         properties => {
1023             node => get_standard_option('pve-node'),
1024             id => {
1025                 type => 'string',
1026                 optional => 1,
1027                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1028                 description => "The ID for the monitor, when omitted the same as the nodename",
1029             },
1030             'exclude-manager' => {
1031                 type => 'boolean',
1032                 optional => 1,
1033                 default => 0,
1034                 description => "When set, only a monitor will be created.",
1035             },
1036             'mon-address' => {
1037                 description => 'Overwrites autodetected monitor IP address. ' .
1038                                'Must be in the public network of ceph.',
1039                 type => 'string', format => 'ip',
1040                 optional => 1,
1041             },
1042         },
1043     },
1044     returns => { type => 'string' },
1045     code => sub {
1046         my ($param) = @_;
1047
1048         PVE::CephTools::check_ceph_installed('ceph_mon');
1049
1050         PVE::CephTools::check_ceph_installed('ceph_mgr')
1051             if (!$param->{'exclude-manager'});
1052
1053         PVE::CephTools::check_ceph_inited();
1054
1055         PVE::CephTools::setup_pve_symlinks();
1056
1057         my $rpcenv = PVE::RPCEnvironment::get();
1058
1059         my $authuser = $rpcenv->get_user();
1060
1061         my $cfg = PVE::CephTools::parse_ceph_config();
1062
1063         my $moncount = 0;
1064
1065         my $monaddrhash = {};
1066
1067         my $systemd_managed = PVE::CephTools::systemd_managed();
1068
1069         foreach my $section (keys %$cfg) {
1070             next if $section eq 'global';
1071             my $d = $cfg->{$section};
1072             if ($section =~ m/^mon\./) {
1073                 $moncount++;
1074                 if ($d->{'mon addr'}) {
1075                     $monaddrhash->{$d->{'mon addr'}} = $section;
1076                 }
1077             }
1078         }
1079
1080         my $monid = $param->{id} // $param->{node};
1081
1082         my $monsection = "mon.$monid";
1083         my $pubnet = $cfg->{global}->{'public network'};
1084         my $ip = $find_mon_ip->($pubnet, $param->{node}, $param->{'mon-address'});
1085
1086         my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]:6789" : "$ip:6789";
1087         my $monname = $param->{node};
1088
1089         die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
1090         die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n"
1091             if $monaddrhash->{$monaddr};
1092
1093         my $worker = sub  {
1094             my $upid = shift;
1095
1096             my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1097
1098             if (! -f $pve_ckeyring_path) {
1099                 run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
1100                             "--gen-key -n client.admin");
1101             }
1102
1103             my $pve_mon_key_path = PVE::CephTools::get_config('pve_mon_key_path');
1104             if (! -f $pve_mon_key_path) {
1105                 run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
1106                 run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " .
1107                             "--cap mds 'allow' " .
1108                             "--cap osd 'allow *' " .
1109                             "--cap mgr 'allow *' " .
1110                             "--cap mon 'allow *'");
1111                 run_command("cp $pve_mon_key_path.tmp /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
1112                 run_command("chown ceph:ceph /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
1113                 run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
1114                 run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
1115             }
1116
1117             my $ccname = PVE::CephTools::get_config('ccname');
1118
1119             my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
1120             -d $mondir && die "monitor filesystem '$mondir' already exist\n";
1121
1122             my $monmap = "/tmp/monmap";
1123
1124             eval {
1125                 mkdir $mondir;
1126
1127                 run_command("chown ceph:ceph $mondir") if $systemd_managed;
1128
1129                 if ($moncount > 0) {
1130                     my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1131                     my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
1132                     file_set_contents($monmap, $mapdata);
1133                 } else {
1134                     run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
1135                 }
1136
1137                 run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
1138                 run_command("chown ceph:ceph -R $mondir") if $systemd_managed;
1139             };
1140             my $err = $@;
1141             unlink $monmap;
1142             if ($err) {
1143                 File::Path::remove_tree($mondir);
1144                 die $err;
1145             }
1146
1147             $cfg->{$monsection} = {
1148                 'host' => $monname,
1149                 'mon addr' => $monaddr,
1150             };
1151
1152             PVE::CephTools::write_ceph_config($cfg);
1153
1154             my $create_keys_pid = fork();
1155             if (!defined($create_keys_pid)) {
1156                 die "Could not spawn ceph-create-keys to create bootstrap keys\n";
1157             } elsif ($create_keys_pid == 0) {
1158                 exit PVE::Tools::run_command(['ceph-create-keys', '-i', $monid]);
1159             } else {
1160                 PVE::CephTools::ceph_service_cmd('start', $monsection);
1161
1162                 if ($systemd_managed) {
1163                     #to ensure we have the correct startup order.
1164                     eval { PVE::Tools::run_command(['/bin/systemctl', 'enable', "ceph-mon\@${monid}.service"]); };
1165                     warn "Enable ceph-mon\@${monid}.service manually"if $@;
1166                 }
1167                 waitpid($create_keys_pid, 0);
1168             }
1169
1170             # create manager
1171             if (!$param->{'exclude-manager'}) {
1172                 my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1173                 $create_mgr->($rados, $monid);
1174             }
1175         };
1176
1177         return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
1178     }});
1179
1180 __PACKAGE__->register_method ({
1181     name => 'destroymon',
1182     path => 'mon/{monid}',
1183     method => 'DELETE',
1184     description => "Destroy Ceph Monitor and Manager.",
1185     proxyto => 'node',
1186     protected => 1,
1187     permissions => {
1188         check => ['perm', '/', [ 'Sys.Modify' ]],
1189     },
1190     parameters => {
1191         additionalProperties => 0,
1192         properties => {
1193             node => get_standard_option('pve-node'),
1194             monid => {
1195                 description => 'Monitor ID',
1196                 type => 'string',
1197                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1198             },
1199             'exclude-manager' => {
1200                 type => 'boolean',
1201                 default => 0,
1202                 optional => 1,
1203                 description => "When set, removes only the monitor, not the manager"
1204             }
1205         },
1206     },
1207     returns => { type => 'string' },
1208     code => sub {
1209         my ($param) = @_;
1210
1211         my $rpcenv = PVE::RPCEnvironment::get();
1212
1213         my $authuser = $rpcenv->get_user();
1214
1215         PVE::CephTools::check_ceph_inited();
1216
1217         my $cfg = PVE::CephTools::parse_ceph_config();
1218
1219         my $monid = $param->{monid};
1220         my $monsection = "mon.$monid";
1221
1222         my $rados = PVE::RADOS->new();
1223         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
1224         my $monlist = $monstat->{monmap}->{mons};
1225
1226         die "no such monitor id '$monid'\n"
1227             if !defined($cfg->{$monsection});
1228
1229         my $ccname = PVE::CephTools::get_config('ccname');
1230
1231         my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
1232         -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n";
1233
1234         die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
1235
1236         my $worker = sub {
1237             my $upid = shift;
1238
1239             # reopen with longer timeout
1240             $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1241
1242             $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });
1243
1244             eval { PVE::CephTools::ceph_service_cmd('stop', $monsection); };
1245             warn $@ if $@;
1246
1247             delete $cfg->{$monsection};
1248             PVE::CephTools::write_ceph_config($cfg);
1249             File::Path::remove_tree($mondir);
1250
1251             # remove manager
1252             if (!$param->{'exclude-manager'}) {
1253                 eval { $destroy_mgr->($monid); };
1254                 warn $@ if $@;
1255             }
1256         };
1257
1258         return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
1259     }});
1260
1261 __PACKAGE__->register_method ({
1262     name => 'createmgr',
1263     path => 'mgr',
1264     method => 'POST',
1265     description => "Create Ceph Manager",
1266     proxyto => 'node',
1267     protected => 1,
1268     permissions => {
1269         check => ['perm', '/', [ 'Sys.Modify' ]],
1270     },
1271     parameters => {
1272         additionalProperties => 0,
1273         properties => {
1274             node => get_standard_option('pve-node'),
1275             id => {
1276                 type => 'string',
1277                 optional => 1,
1278                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1279                 description => "The ID for the manager, when omitted the same as the nodename",
1280             },
1281         },
1282     },
1283     returns => { type => 'string' },
1284     code => sub {
1285         my ($param) = @_;
1286
1287         PVE::CephTools::check_ceph_installed('ceph_mgr');
1288
1289         PVE::CephTools::check_ceph_inited();
1290
1291         my $rpcenv = PVE::RPCEnvironment::get();
1292
1293         my $authuser = $rpcenv->get_user();
1294
1295         my $mgrid = $param->{id} // $param->{node};
1296
1297         my $worker = sub  {
1298             my $upid = shift;
1299
1300             my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1301
1302             $create_mgr->($rados, $mgrid);
1303         };
1304
1305         return $rpcenv->fork_worker('cephcreatemgr', "mgr.$mgrid", $authuser, $worker);
1306     }});
1307
1308 __PACKAGE__->register_method ({
1309     name => 'destroymgr',
1310     path => 'mgr/{id}',
1311     method => 'DELETE',
1312     description => "Destroy Ceph Manager.",
1313     proxyto => 'node',
1314     protected => 1,
1315     permissions => {
1316         check => ['perm', '/', [ 'Sys.Modify' ]],
1317     },
1318     parameters => {
1319         additionalProperties => 0,
1320         properties => {
1321             node => get_standard_option('pve-node'),
1322             id => {
1323                 description => 'The ID of the manager',
1324                 type => 'string',
1325                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1326             },
1327         },
1328     },
1329     returns => { type => 'string' },
1330     code => sub {
1331         my ($param) = @_;
1332
1333         my $rpcenv = PVE::RPCEnvironment::get();
1334
1335         my $authuser = $rpcenv->get_user();
1336
1337         PVE::CephTools::check_ceph_inited();
1338
1339         my $mgrid = $param->{id};
1340
1341         my $worker = sub {
1342             my $upid = shift;
1343
1344             $destroy_mgr->($mgrid);
1345         };
1346
1347         return $rpcenv->fork_worker('cephdestroymgr', "mgr.$mgrid",  $authuser, $worker);
1348     }});
1349
1350 __PACKAGE__->register_method ({
1351     name => 'stop',
1352     path => 'stop',
1353     method => 'POST',
1354     description => "Stop ceph services.",
1355     proxyto => 'node',
1356     protected => 1,
1357     permissions => {
1358         check => ['perm', '/', [ 'Sys.Modify' ]],
1359     },
1360     parameters => {
1361         additionalProperties => 0,
1362         properties => {
1363             node => get_standard_option('pve-node'),
1364             service => {
1365                 description => 'Ceph service name.',
1366                 type => 'string',
1367                 optional => 1,
1368                 default => 'ceph.target',
1369                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1370             },
1371         },
1372     },
1373     returns => { type => 'string' },
1374     code => sub {
1375         my ($param) = @_;
1376
1377         my $rpcenv = PVE::RPCEnvironment::get();
1378
1379         my $authuser = $rpcenv->get_user();
1380
1381         PVE::CephTools::check_ceph_inited();
1382
1383         my $cfg = PVE::CephTools::parse_ceph_config();
1384         scalar(keys %$cfg) || die "no configuration\n";
1385
1386         my $worker = sub {
1387             my $upid = shift;
1388
1389             my $cmd = ['stop'];
1390             if ($param->{service}) {
1391                 push @$cmd, $param->{service};
1392             }
1393
1394             PVE::CephTools::ceph_service_cmd(@$cmd);
1395         };
1396
1397         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
1398                                     $authuser, $worker);
1399     }});
1400
1401 __PACKAGE__->register_method ({
1402     name => 'start',
1403     path => 'start',
1404     method => 'POST',
1405     description => "Start ceph services.",
1406     proxyto => 'node',
1407     protected => 1,
1408     permissions => {
1409         check => ['perm', '/', [ 'Sys.Modify' ]],
1410     },
1411     parameters => {
1412         additionalProperties => 0,
1413         properties => {
1414             node => get_standard_option('pve-node'),
1415             service => {
1416                 description => 'Ceph service name.',
1417                 type => 'string',
1418                 optional => 1,
1419                 default => 'ceph.target',
1420                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1421             },
1422         },
1423     },
1424     returns => { type => 'string' },
1425     code => sub {
1426         my ($param) = @_;
1427
1428         my $rpcenv = PVE::RPCEnvironment::get();
1429
1430         my $authuser = $rpcenv->get_user();
1431
1432         PVE::CephTools::check_ceph_inited();
1433
1434         my $cfg = PVE::CephTools::parse_ceph_config();
1435         scalar(keys %$cfg) || die "no configuration\n";
1436
1437         my $worker = sub {
1438             my $upid = shift;
1439
1440             my $cmd = ['start'];
1441             if ($param->{service}) {
1442                 push @$cmd, $param->{service};
1443             }
1444
1445             PVE::CephTools::ceph_service_cmd(@$cmd);
1446         };
1447
1448         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
1449                                     $authuser, $worker);
1450     }});
1451
1452 __PACKAGE__->register_method ({
1453     name => 'restart',
1454     path => 'restart',
1455     method => 'POST',
1456     description => "Restart ceph services.",
1457     proxyto => 'node',
1458     protected => 1,
1459     permissions => {
1460         check => ['perm', '/', [ 'Sys.Modify' ]],
1461     },
1462     parameters => {
1463         additionalProperties => 0,
1464         properties => {
1465             node => get_standard_option('pve-node'),
1466             service => {
1467                 description => 'Ceph service name.',
1468                 type => 'string',
1469                 optional => 1,
1470                 default => 'ceph.target',
1471                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1472             },
1473         },
1474     },
1475     returns => { type => 'string' },
1476     code => sub {
1477         my ($param) = @_;
1478
1479         my $rpcenv = PVE::RPCEnvironment::get();
1480
1481         my $authuser = $rpcenv->get_user();
1482
1483         PVE::CephTools::check_ceph_inited();
1484
1485         my $cfg = PVE::CephTools::parse_ceph_config();
1486         scalar(keys %$cfg) || die "no configuration\n";
1487
1488         my $worker = sub {
1489             my $upid = shift;
1490
1491             my $cmd = ['restart'];
1492             if ($param->{service}) {
1493                 push @$cmd, $param->{service};
1494             }
1495
1496             PVE::CephTools::ceph_service_cmd(@$cmd);
1497         };
1498
1499         return $rpcenv->fork_worker('srvrestart', $param->{service} || 'ceph',
1500                                     $authuser, $worker);
1501     }});
1502
1503 __PACKAGE__->register_method ({
1504     name => 'status',
1505     path => 'status',
1506     method => 'GET',
1507     description => "Get ceph status.",
1508     proxyto => 'node',
1509     protected => 1,
1510     permissions => {
1511         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1512     },
1513     parameters => {
1514         additionalProperties => 0,
1515         properties => {
1516             node => get_standard_option('pve-node'),
1517         },
1518     },
1519     returns => { type => 'object' },
1520     code => sub {
1521         my ($param) = @_;
1522
1523         PVE::CephTools::check_ceph_enabled();
1524
1525         my $rados = PVE::RADOS->new();
1526         my $status = $rados->mon_command({ prefix => 'status' });
1527         $status->{health} = $rados->mon_command({ prefix => 'health', detail => 'detail' });
1528         return $status;
1529     }});
1530
1531 __PACKAGE__->register_method ({
1532     name => 'lspools',
1533     path => 'pools',
1534     method => 'GET',
1535     description => "List all pools.",
1536     proxyto => 'node',
1537     protected => 1,
1538     permissions => {
1539         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1540     },
1541     parameters => {
1542         additionalProperties => 0,
1543         properties => {
1544             node => get_standard_option('pve-node'),
1545         },
1546     },
1547     returns => {
1548         type => 'array',
1549         items => {
1550             type => "object",
1551             properties => {
1552                 pool => { type => 'integer' },
1553                 pool_name => { type => 'string' },
1554                 size => { type => 'integer' },
1555             },
1556         },
1557         links => [ { rel => 'child', href => "{pool_name}" } ],
1558     },
1559     code => sub {
1560         my ($param) = @_;
1561
1562         PVE::CephTools::check_ceph_inited();
1563
1564         my $rados = PVE::RADOS->new();
1565
1566         my $stats = {};
1567         my $res = $rados->mon_command({ prefix => 'df' });
1568
1569         foreach my $d (@{$res->{pools}}) {
1570             next if !$d->{stats};
1571             next if !defined($d->{id});
1572             $stats->{$d->{id}} = $d->{stats};
1573         }
1574
1575         $res = $rados->mon_command({ prefix => 'osd dump' });
1576         my $rulestmp = $rados->mon_command({ prefix => 'osd crush rule dump'});
1577
1578         my $rules = {};
1579         for my $rule (@$rulestmp) {
1580             $rules->{$rule->{rule_id}} = $rule->{rule_name};
1581         }
1582
1583         my $data = [];
1584         foreach my $e (@{$res->{pools}}) {
1585             my $d = {};
1586             foreach my $attr (qw(pool pool_name size min_size pg_num crush_rule)) {
1587                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
1588             }
1589
1590             if (defined($d->{crush_rule}) && defined($rules->{$d->{crush_rule}})) {
1591                 $d->{crush_rule_name} = $rules->{$d->{crush_rule}};
1592             }
1593
1594             if (my $s = $stats->{$d->{pool}}) {
1595                 $d->{bytes_used} = $s->{bytes_used};
1596                 $d->{percent_used} = $s->{percent_used};
1597             }
1598             push @$data, $d;
1599         }
1600
1601
1602         return $data;
1603     }});
1604
1605 __PACKAGE__->register_method ({
1606     name => 'createpool',
1607     path => 'pools',
1608     method => 'POST',
1609     description => "Create POOL",
1610     proxyto => 'node',
1611     protected => 1,
1612     permissions => {
1613         check => ['perm', '/', [ 'Sys.Modify' ]],
1614     },
1615     parameters => {
1616         additionalProperties => 0,
1617         properties => {
1618             node => get_standard_option('pve-node'),
1619             name => {
1620                 description => "The name of the pool. It must be unique.",
1621                 type => 'string',
1622             },
1623             size => {
1624                 description => 'Number of replicas per object',
1625                 type => 'integer',
1626                 default => 3,
1627                 optional => 1,
1628                 minimum => 1,
1629                 maximum => 7,
1630             },
1631             min_size => {
1632                 description => 'Minimum number of replicas per object',
1633                 type => 'integer',
1634                 default => 2,
1635                 optional => 1,
1636                 minimum => 1,
1637                 maximum => 7,
1638             },
1639             pg_num => {
1640                 description => "Number of placement groups.",
1641                 type => 'integer',
1642                 default => 64,
1643                 optional => 1,
1644                 minimum => 8,
1645                 maximum => 32768,
1646             },
1647             crush_rule => {
1648                 description => "The rule to use for mapping object placement in the cluster.",
1649                 type => 'string',
1650                 optional => 1,
1651             },
1652             application => {
1653                 description => "The application of the pool, 'rbd' by default.",
1654                 type => 'string',
1655                 enum => ['rbd', 'cephfs', 'rgw'],
1656                 optional => 1,
1657             },
1658             add_storages => {
1659                 description => "Configure VM and CT storage using the new pool.",
1660                 type => 'boolean',
1661                 optional => 1,
1662             },
1663         },
1664     },
1665     returns => { type => 'string' },
1666     code => sub {
1667         my ($param) = @_;
1668
1669         PVE::Cluster::check_cfs_quorum();
1670         PVE::CephTools::check_ceph_inited();
1671
1672         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1673
1674         die "not fully configured - missing '$pve_ckeyring_path'\n"
1675             if ! -f $pve_ckeyring_path;
1676
1677         my $pool = $param->{name};
1678         my $rpcenv = PVE::RPCEnvironment::get();
1679         my $user = $rpcenv->get_user();
1680
1681         if ($param->{add_storages}) {
1682             $rpcenv->check($user, '/storage', ['Datastore.Allocate']);
1683             die "pool name contains characters which are illegal for storage naming\n"
1684                 if !PVE::JSONSchema::parse_storage_id($pool);
1685         }
1686
1687         my $pg_num = $param->{pg_num} || 64;
1688         my $size = $param->{size} || 3;
1689         my $min_size = $param->{min_size} || 2;
1690         my $application = $param->{application} // 'rbd';
1691
1692         my $worker = sub {
1693
1694             PVE::CephTools::create_pool($pool, $param);
1695
1696             if ($param->{add_storages}) {
1697                 my $err;
1698                 eval { $add_storage->($pool, "${pool}"); };
1699                 if ($@) {
1700                     warn "failed to add storage: $@";
1701                     $err = 1;
1702                 }
1703                 die "adding storage for pool '$pool' failed, check log and add manually!\n"
1704                     if $err;
1705             }
1706         };
1707
1708         return $rpcenv->fork_worker('cephcreatepool', $pool,  $user, $worker);
1709     }});
1710
1711 __PACKAGE__->register_method ({
1712     name => 'get_flags',
1713     path => 'flags',
1714     method => 'GET',
1715     description => "get all set ceph flags",
1716     proxyto => 'node',
1717     protected => 1,
1718     permissions => {
1719         check => ['perm', '/', [ 'Sys.Audit' ]],
1720     },
1721     parameters => {
1722         additionalProperties => 0,
1723         properties => {
1724             node => get_standard_option('pve-node'),
1725         },
1726     },
1727     returns => { type => 'string' },
1728     code => sub {
1729         my ($param) = @_;
1730
1731         PVE::CephTools::check_ceph_inited();
1732
1733         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1734
1735         die "not fully configured - missing '$pve_ckeyring_path'\n"
1736             if ! -f $pve_ckeyring_path;
1737
1738         my $rados = PVE::RADOS->new();
1739
1740         my $stat = $rados->mon_command({ prefix => 'osd dump' });
1741
1742         return $stat->{flags} // '';
1743     }});
1744
1745 __PACKAGE__->register_method ({
1746     name => 'set_flag',
1747     path => 'flags/{flag}',
1748     method => 'POST',
1749     description => "Set a ceph flag",
1750     proxyto => 'node',
1751     protected => 1,
1752     permissions => {
1753         check => ['perm', '/', [ 'Sys.Modify' ]],
1754     },
1755     parameters => {
1756         additionalProperties => 0,
1757         properties => {
1758             node => get_standard_option('pve-node'),
1759             flag => {
1760                 description => 'The ceph flag to set/unset',
1761                 type => 'string',
1762                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1763             },
1764         },
1765     },
1766     returns => { type => 'null' },
1767     code => sub {
1768         my ($param) = @_;
1769
1770         PVE::CephTools::check_ceph_inited();
1771
1772         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1773
1774         die "not fully configured - missing '$pve_ckeyring_path'\n"
1775             if ! -f $pve_ckeyring_path;
1776
1777         my $set = $param->{set} // !$param->{unset};
1778         my $rados = PVE::RADOS->new();
1779
1780         $rados->mon_command({
1781             prefix => "osd set",
1782             key => $param->{flag},
1783         });
1784
1785         return undef;
1786     }});
1787
1788 __PACKAGE__->register_method ({
1789     name => 'unset_flag',
1790     path => 'flags/{flag}',
1791     method => 'DELETE',
1792     description => "Unset a ceph flag",
1793     proxyto => 'node',
1794     protected => 1,
1795     permissions => {
1796         check => ['perm', '/', [ 'Sys.Modify' ]],
1797     },
1798     parameters => {
1799         additionalProperties => 0,
1800         properties => {
1801             node => get_standard_option('pve-node'),
1802             flag => {
1803                 description => 'The ceph flag to set/unset',
1804                 type => 'string',
1805                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1806             },
1807         },
1808     },
1809     returns => { type => 'null' },
1810     code => sub {
1811         my ($param) = @_;
1812
1813         PVE::CephTools::check_ceph_inited();
1814
1815         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1816
1817         die "not fully configured - missing '$pve_ckeyring_path'\n"
1818             if ! -f $pve_ckeyring_path;
1819
1820         my $set = $param->{set} // !$param->{unset};
1821         my $rados = PVE::RADOS->new();
1822
1823         $rados->mon_command({
1824             prefix => "osd unset",
1825             key => $param->{flag},
1826         });
1827
1828         return undef;
1829     }});
1830
1831 __PACKAGE__->register_method ({
1832     name => 'destroypool',
1833     path => 'pools/{name}',
1834     method => 'DELETE',
1835     description => "Destroy pool",
1836     proxyto => 'node',
1837     protected => 1,
1838     permissions => {
1839         check => ['perm', '/', [ 'Sys.Modify' ]],
1840     },
1841     parameters => {
1842         additionalProperties => 0,
1843         properties => {
1844             node => get_standard_option('pve-node'),
1845             name => {
1846                 description => "The name of the pool. It must be unique.",
1847                 type => 'string',
1848             },
1849             force => {
1850                 description => "If true, destroys pool even if in use",
1851                 type => 'boolean',
1852                 optional => 1,
1853                 default => 0,
1854             },
1855             remove_storages => {
1856                 description => "Remove all pveceph-managed storages configured for this pool",
1857                 type => 'boolean',
1858                 optional => 1,
1859                 default => 0,
1860             },
1861         },
1862     },
1863     returns => { type => 'string' },
1864     code => sub {
1865         my ($param) = @_;
1866
1867         PVE::CephTools::check_ceph_inited();
1868
1869         my $rpcenv = PVE::RPCEnvironment::get();
1870         my $user = $rpcenv->get_user();
1871         $rpcenv->check($user, '/storage', ['Datastore.Allocate'])
1872             if $param->{remove_storages};
1873
1874         my $pool = $param->{name};
1875
1876         my $worker = sub {
1877             my $storages = $get_storages->($pool);
1878
1879             # if not forced, destroy ceph pool only when no
1880             # vm disks are on it anymore
1881             if (!$param->{force}) {
1882                 my $storagecfg = PVE::Storage::config();
1883                 foreach my $storeid (keys %$storages) {
1884                     my $storage = $storages->{$storeid};
1885
1886                     # check if any vm disks are on the pool
1887                     print "checking storage '$storeid' for RBD images..\n";
1888                     my $res = PVE::Storage::vdisk_list($storagecfg, $storeid);
1889                     die "ceph pool '$pool' still in use by storage '$storeid'\n"
1890                         if @{$res->{$storeid}} != 0;
1891                 }
1892             }
1893
1894             PVE::CephTools::destroy_pool($pool);
1895
1896             if ($param->{remove_storages}) {
1897                 my $err;
1898                 foreach my $storeid (keys %$storages) {
1899                     # skip external clusters, not managed by pveceph
1900                     next if $storages->{$storeid}->{monhost};
1901                     eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
1902                     if ($@) {
1903                         warn "failed to remove storage '$storeid': $@\n";
1904                         $err = 1;
1905                     }
1906                 }
1907                 die "failed to remove (some) storages - check log and remove manually!\n"
1908                     if $err;
1909             }
1910         };
1911         return $rpcenv->fork_worker('cephdestroypool', $pool,  $user, $worker);
1912     }});
1913
1914
1915 __PACKAGE__->register_method ({
1916     name => 'crush',
1917     path => 'crush',
1918     method => 'GET',
1919     description => "Get OSD crush map",
1920     proxyto => 'node',
1921     protected => 1,
1922     permissions => {
1923         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1924     },
1925     parameters => {
1926         additionalProperties => 0,
1927         properties => {
1928             node => get_standard_option('pve-node'),
1929         },
1930     },
1931     returns => { type => 'string' },
1932     code => sub {
1933         my ($param) = @_;
1934
1935         PVE::CephTools::check_ceph_inited();
1936
1937         # this produces JSON (difficult to read for the user)
1938         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
1939
1940         my $txt = '';
1941
1942         my $mapfile = "/var/tmp/ceph-crush.map.$$";
1943         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
1944
1945         my $rados = PVE::RADOS->new();
1946
1947         eval {
1948             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
1949             file_set_contents($mapfile, $bindata);
1950             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
1951             $txt = file_get_contents($mapdata);
1952         };
1953         my $err = $@;
1954
1955         unlink $mapfile;
1956         unlink $mapdata;
1957
1958         die $err if $err;
1959
1960         return $txt;
1961     }});
1962
1963 __PACKAGE__->register_method({
1964     name => 'log',
1965     path => 'log',
1966     method => 'GET',
1967     description => "Read ceph log",
1968     proxyto => 'node',
1969     permissions => {
1970         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
1971     },
1972     protected => 1,
1973     parameters => {
1974         additionalProperties => 0,
1975         properties => {
1976             node => get_standard_option('pve-node'),
1977             start => {
1978                 type => 'integer',
1979                 minimum => 0,
1980                 optional => 1,
1981             },
1982             limit => {
1983                 type => 'integer',
1984                 minimum => 0,
1985                 optional => 1,
1986             },
1987         },
1988     },
1989     returns => {
1990         type => 'array',
1991         items => {
1992             type => "object",
1993             properties => {
1994                 n => {
1995                   description=>  "Line number",
1996                   type=> 'integer',
1997                 },
1998                 t => {
1999                   description=>  "Line text",
2000                   type => 'string',
2001                 }
2002             }
2003         }
2004     },
2005     code => sub {
2006         my ($param) = @_;
2007
2008         my $rpcenv = PVE::RPCEnvironment::get();
2009         my $user = $rpcenv->get_user();
2010         my $node = $param->{node};
2011
2012         my $logfile = "/var/log/ceph/ceph.log";
2013         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
2014
2015         $rpcenv->set_result_attrib('total', $count);
2016
2017         return $lines;
2018     }});
2019
2020 __PACKAGE__->register_method ({
2021     name => 'rules',
2022     path => 'rules',
2023     method => 'GET',
2024     description => "List ceph rules.",
2025     proxyto => 'node',
2026     protected => 1,
2027     permissions => {
2028         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
2029     },
2030     parameters => {
2031         additionalProperties => 0,
2032         properties => {
2033             node => get_standard_option('pve-node'),
2034         },
2035     },
2036     returns => {
2037         type => 'array',
2038         items => {
2039             type => "object",
2040             properties => {},
2041         },
2042         links => [ { rel => 'child', href => "{name}" } ],
2043     },
2044     code => sub {
2045         my ($param) = @_;
2046
2047         PVE::CephTools::check_ceph_inited();
2048
2049         my $rados = PVE::RADOS->new();
2050
2051         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
2052
2053         my $res = [];
2054
2055         foreach my $rule (@$rules) {
2056             push @$res, { name => $rule };
2057         }
2058
2059         return $res;
2060     }});