PVE/API2/Ceph.pm

   1 package PVE::API2::CephOSD;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd qw(abs_path);
   7 use IO::File;
   8
   9 use PVE::Ceph::Tools;
  10 use PVE::CephConfig;
  11 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  12 use PVE::Diskmanage;
  13 use PVE::Exception qw(raise_param_exc);
  14 use PVE::JSONSchema qw(get_standard_option);
  15 use PVE::RADOS;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::Tools qw(run_command file_set_contents);
  19
  20 use base qw(PVE::RESTHandler);
  21
  22 my $get_osd_status = sub {
  23     my ($rados, $osdid) = @_;
  24
  25     my $stat = $rados->mon_command({ prefix => 'osd dump' });
  26
  27     my $osdlist = $stat->{osds} || [];
  28
  29     my $flags = $stat->{flags} || undef;
  30
  31     my $osdstat;
  32     foreach my $d (@$osdlist) {
  33         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  34     }
  35     if (defined($osdid)) {
  36         die "no such OSD '$osdid'\n" if !$osdstat->{$osdid};
  37         return $osdstat->{$osdid};
  38     }
  39
  40     return wantarray? ($osdstat, $flags):$osdstat;
  41 };
  42
  43 my $get_osd_usage = sub {
  44     my ($rados) = @_;
  45
  46     my $osdlist = $rados->mon_command({ prefix => 'pg dump',
  47                                         dumpcontents => [ 'osds' ]}) || [];
  48
  49     my $osdstat;
  50     foreach my $d (@$osdlist) {
  51         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  52     }
  53
  54     return $osdstat;
  55 };
  56
  57 __PACKAGE__->register_method ({
  58     name => 'index',
  59     path => '',
  60     method => 'GET',
  61     description => "Get Ceph osd list/tree.",
  62     proxyto => 'node',
  63     protected => 1,
  64     permissions => {
  65         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  66     },
  67     parameters => {
  68         additionalProperties => 0,
  69         properties => {
  70             node => get_standard_option('pve-node'),
  71         },
  72     },
  73     # fixme: return a list instead of extjs tree format ?
  74     returns => {
  75         type => "object",
  76     },
  77     code => sub {
  78         my ($param) = @_;
  79
  80         PVE::Ceph::Tools::check_ceph_inited();
  81
  82         my $rados = PVE::RADOS->new();
  83         my $res = $rados->mon_command({ prefix => 'osd tree' });
  84
  85         die "no tree nodes found\n" if !($res && $res->{nodes});
  86
  87         my ($osdhash, $flags) = &$get_osd_status($rados);
  88
  89         my $usagehash = &$get_osd_usage($rados);
  90
  91         my $osdmetadata_tmp = $rados->mon_command({ prefix => 'osd metadata' });
  92
  93         my $osdmetadata = {};
  94         foreach my $osd (@$osdmetadata_tmp) {
  95             $osdmetadata->{$osd->{id}} = $osd;
  96         }
  97
  98         my $nodes = {};
  99         my $newnodes = {};
 100         foreach my $e (@{$res->{nodes}}) {
 101             $nodes->{$e->{id}} = $e;
 102
 103             my $new = {
 104                 id => $e->{id},
 105                 name => $e->{name},
 106                 type => $e->{type}
 107             };
 108
 109             foreach my $opt (qw(status crush_weight reweight device_class)) {
 110                 $new->{$opt} = $e->{$opt} if defined($e->{$opt});
 111             }
 112
 113             if (my $stat = $osdhash->{$e->{id}}) {
 114                 $new->{in} = $stat->{in} if defined($stat->{in});
 115             }
 116
 117             if (my $stat = $usagehash->{$e->{id}}) {
 118                 $new->{total_space} = ($stat->{kb} || 1) * 1024;
 119                 $new->{bytes_used} = ($stat->{kb_used} || 0) * 1024;
 120                 $new->{percent_used} = ($new->{bytes_used}*100)/$new->{total_space};
 121                 if (my $d = $stat->{perf_stat}) {
 122                     $new->{commit_latency_ms} = $d->{commit_latency_ms};
 123                     $new->{apply_latency_ms} = $d->{apply_latency_ms};
 124                 }
 125             }
 126
 127             my $osdmd = $osdmetadata->{$e->{id}};
 128             if ($e->{type} eq 'osd' && $osdmd) {
 129                 if ($osdmd->{bluefs}) {
 130                     $new->{osdtype} = 'bluestore';
 131                     $new->{blfsdev} = $osdmd->{bluestore_bdev_dev_node};
 132                     $new->{dbdev} = $osdmd->{bluefs_db_dev_node};
 133                     $new->{waldev} = $osdmd->{bluefs_wal_dev_node};
 134                 } else {
 135                     $new->{osdtype} = 'filestore';
 136                 }
 137             }
 138
 139             $newnodes->{$e->{id}} = $new;
 140         }
 141
 142         foreach my $e (@{$res->{nodes}}) {
 143             my $new = $newnodes->{$e->{id}};
 144             if ($e->{children} && scalar(@{$e->{children}})) {
 145                 $new->{children} = [];
 146                 $new->{leaf} = 0;
 147                 foreach my $cid (@{$e->{children}}) {
 148                     $nodes->{$cid}->{parent} = $e->{id};
 149                     if ($nodes->{$cid}->{type} eq 'osd' &&
 150                         $e->{type} eq 'host') {
 151                         $newnodes->{$cid}->{host} = $e->{name};
 152                     }
 153                     push @{$new->{children}}, $newnodes->{$cid};
 154                 }
 155             } else {
 156                 $new->{leaf} = ($e->{id} >= 0) ? 1 : 0;
 157             }
 158         }
 159
 160         my $roots = [];
 161         foreach my $e (@{$res->{nodes}}) {
 162             if (!$nodes->{$e->{id}}->{parent}) {
 163                 push @$roots, $newnodes->{$e->{id}};
 164             }
 165         }
 166
 167         die "no root node\n" if !@$roots;
 168
 169         my $data = { root => { leaf =>  0, children => $roots } };
 170
 171         # we want this for the noout flag
 172         $data->{flags} = $flags if $flags;
 173
 174         return $data;
 175     }});
 176
 177 __PACKAGE__->register_method ({
 178     name => 'createosd',
 179     path => '',
 180     method => 'POST',
 181     description => "Create OSD",
 182     proxyto => 'node',
 183     protected => 1,
 184     parameters => {
 185         additionalProperties => 0,
 186         properties => {
 187             node => get_standard_option('pve-node'),
 188             dev => {
 189                 description => "Block device name.",
 190                 type => 'string',
 191             },
 192             journal_dev => {
 193                 description => "Block device name for journal (filestore) or block.db (bluestore).",
 194                 optional => 1,
 195                 type => 'string',
 196             },
 197             wal_dev => {
 198                 description => "Block device name for block.wal (bluestore only).",
 199                 optional => 1,
 200                 type => 'string',
 201             },
 202             fstype => {
 203                 description => "File system type (filestore only).",
 204                 type => 'string',
 205                 enum => ['xfs', 'ext4'],
 206                 default => 'xfs',
 207                 optional => 1,
 208             },
 209             bluestore => {
 210                 description => "Use bluestore instead of filestore. This is the default.",
 211                 type => 'boolean',
 212                 default => 1,
 213                 optional => 1,
 214             },
 215         },
 216     },
 217     returns => { type => 'string' },
 218     code => sub {
 219         my ($param) = @_;
 220
 221         my $rpcenv = PVE::RPCEnvironment::get();
 222
 223         my $authuser = $rpcenv->get_user();
 224
 225         raise_param_exc({ 'bluestore' => "conflicts with parameter 'fstype'" })
 226             if (defined($param->{fstype}) && defined($param->{bluestore}) && $param->{bluestore});
 227
 228         PVE::Ceph::Tools::check_ceph_inited();
 229
 230         PVE::Ceph::Tools::setup_pve_symlinks();
 231
 232         PVE::Ceph::Tools::check_ceph_installed('ceph_osd');
 233
 234         my $bluestore = $param->{bluestore} // 1;
 235
 236         my $journal_dev;
 237         my $wal_dev;
 238
 239         if ($param->{journal_dev} && ($param->{journal_dev} ne $param->{dev})) {
 240             $journal_dev = PVE::Diskmanage::verify_blockdev_path($param->{journal_dev});
 241         }
 242
 243         if ($param->{wal_dev} &&
 244             ($param->{wal_dev} ne $param->{dev}) &&
 245             (!$param->{journal_dev} || $param->{wal_dev} ne $param->{journal_dev})) {
 246             raise_param_exc({ 'wal_dev' => "can only be set with paramater 'bluestore'"})
 247                 if !$bluestore;
 248             $wal_dev = PVE::Diskmanage::verify_blockdev_path($param->{wal_dev});
 249         }
 250
 251         $param->{dev} = PVE::Diskmanage::verify_blockdev_path($param->{dev});
 252
 253         my $devname = $param->{dev};
 254         $devname =~ s|/dev/||;
 255
 256         my $disklist = PVE::Diskmanage::get_disks($devname, 1);
 257
 258         my $diskinfo = $disklist->{$devname};
 259         die "unable to get device info for '$devname'\n"
 260             if !$diskinfo;
 261
 262         die "device '$param->{dev}' is in use\n"
 263             if $diskinfo->{used};
 264
 265         my $devpath = $diskinfo->{devpath};
 266         my $rados = PVE::RADOS->new();
 267         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 268         die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid};
 269
 270         my $fsid = $monstat->{monmap}->{fsid};
 271         $fsid = $1 if $fsid =~ m/^([0-9a-f\-]+)$/;
 272
 273         my $ceph_bootstrap_osd_keyring = PVE::Ceph::Tools::get_config('ceph_bootstrap_osd_keyring');
 274
 275         if (! -f $ceph_bootstrap_osd_keyring) {
 276             my $bindata = $rados->mon_command({ prefix => 'auth get', entity => 'client.bootstrap-osd', format => 'plain' });
 277             file_set_contents($ceph_bootstrap_osd_keyring, $bindata);
 278         };
 279
 280         my $worker = sub {
 281             my $upid = shift;
 282
 283             my $fstype = $param->{fstype} || 'xfs';
 284
 285
 286             my $ccname = PVE::Ceph::Tools::get_config('ccname');
 287
 288             my $cmd = ['ceph-disk', 'prepare', '--zap-disk',
 289                        '--cluster', $ccname, '--cluster-uuid', $fsid ];
 290
 291             if ($bluestore) {
 292                 print "create OSD on $devpath (bluestore)\n";
 293                 push @$cmd, '--bluestore';
 294
 295                 if ($journal_dev) {
 296                     print "using device '$journal_dev' for block.db\n";
 297                     push @$cmd, '--block.db', $journal_dev;
 298                 }
 299
 300                 if ($wal_dev) {
 301                     print "using device '$wal_dev' for block.wal\n";
 302                     push @$cmd, '--block.wal', $wal_dev;
 303                 }
 304
 305                 push @$cmd, $devpath;
 306             } else {
 307                 print "create OSD on $devpath ($fstype)\n";
 308                 push @$cmd, '--filestore', '--fs-type', $fstype;
 309                 if ($journal_dev) {
 310                     print "using device '$journal_dev' for journal\n";
 311                     push @$cmd, '--journal-dev', $devpath, $journal_dev;
 312                 } else {
 313                     push @$cmd, $devpath;
 314                 }
 315             }
 316
 317             PVE::Ceph::Tools::wipe_disks($devpath);
 318
 319             run_command($cmd);
 320         };
 321
 322         return $rpcenv->fork_worker('cephcreateosd', $devname,  $authuser, $worker);
 323     }});
 324
 325 __PACKAGE__->register_method ({
 326     name => 'destroyosd',
 327     path => '{osdid}',
 328     method => 'DELETE',
 329     description => "Destroy OSD",
 330     proxyto => 'node',
 331     protected => 1,
 332     parameters => {
 333         additionalProperties => 0,
 334         properties => {
 335             node => get_standard_option('pve-node'),
 336             osdid => {
 337                 description => 'OSD ID',
 338                 type => 'integer',
 339             },
 340             cleanup => {
 341                 description => "If set, we remove partition table entries.",
 342                 type => 'boolean',
 343                 optional => 1,
 344                 default => 0,
 345             },
 346         },
 347     },
 348     returns => { type => 'string' },
 349     code => sub {
 350         my ($param) = @_;
 351
 352         my $rpcenv = PVE::RPCEnvironment::get();
 353
 354         my $authuser = $rpcenv->get_user();
 355
 356         PVE::Ceph::Tools::check_ceph_inited();
 357
 358         my $osdid = $param->{osdid};
 359
 360         my $rados = PVE::RADOS->new();
 361         my $osdstat = &$get_osd_status($rados, $osdid);
 362
 363         die "osd is in use (in == 1)\n" if $osdstat->{in};
 364         #&$run_ceph_cmd(['osd', 'out', $osdid]);
 365
 366         die "osd is still runnung (up == 1)\n" if $osdstat->{up};
 367
 368         my $osdsection = "osd.$osdid";
 369
 370         my $worker = sub {
 371             my $upid = shift;
 372
 373             # reopen with longer timeout
 374             $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
 375
 376             print "destroy OSD $osdsection\n";
 377
 378             eval {
 379                 PVE::Ceph::Tools::ceph_service_cmd('stop', $osdsection);
 380                 PVE::Ceph::Tools::ceph_service_cmd('disable', $osdsection);
 381             };
 382             warn $@ if $@;
 383
 384             print "Remove $osdsection from the CRUSH map\n";
 385             $rados->mon_command({ prefix => "osd crush remove", name => $osdsection, format => 'plain' });
 386
 387             print "Remove the $osdsection authentication key.\n";
 388             $rados->mon_command({ prefix => "auth del", entity => $osdsection, format => 'plain' });
 389
 390             print "Remove OSD $osdsection\n";
 391             $rados->mon_command({ prefix => "osd rm", ids => [ $osdsection ], format => 'plain' });
 392
 393             # try to unmount from standard mount point
 394             my $mountpoint = "/var/lib/ceph/osd/ceph-$osdid";
 395
 396             my $disks_to_wipe = {};
 397             my $remove_partition = sub {
 398                 my ($part) = @_;
 399
 400                 return if !$part || (! -b $part );
 401                 my $partnum = PVE::Diskmanage::get_partnum($part);
 402                 my $devpath = PVE::Diskmanage::get_blockdev($part);
 403
 404                 print "remove partition $part (disk '${devpath}', partnum $partnum)\n";
 405                 eval { run_command(['/sbin/sgdisk', '-d', $partnum, "${devpath}"]); };
 406                 warn $@ if $@;
 407
 408                 $disks_to_wipe->{$devpath} = 1;
 409             };
 410
 411             my $partitions_to_remove = [];
 412
 413             if ($param->{cleanup}) {
 414                 if (my $fd = IO::File->new("/proc/mounts", "r")) {
 415                     while (defined(my $line = <$fd>)) {
 416                         my ($dev, $path, $fstype) = split(/\s+/, $line);
 417                         next if !($dev && $path && $fstype);
 418                         next if $dev !~ m|^/dev/|;
 419                         if ($path eq $mountpoint) {
 420                             my $data_part = abs_path($dev);
 421                             push @$partitions_to_remove, $data_part;
 422                             last;
 423                         }
 424                     }
 425                     close($fd);
 426                 }
 427
 428                 foreach my $path (qw(journal block block.db block.wal)) {
 429                     my $part = abs_path("$mountpoint/$path");
 430                     if ($part) {
 431                         push @$partitions_to_remove, $part;
 432                     }
 433                 }
 434             }
 435
 436             print "Unmount OSD $osdsection from  $mountpoint\n";
 437             eval { run_command(['/bin/umount', $mountpoint]); };
 438             if (my $err = $@) {
 439                 warn $err;
 440             } elsif ($param->{cleanup}) {
 441                 #be aware of the ceph udev rules which can remount.
 442                 foreach my $part (@$partitions_to_remove) {
 443                     $remove_partition->($part);
 444                 }
 445
 446                 PVE::Ceph::Tools::wipe_disks(keys %$disks_to_wipe);
 447             }
 448         };
 449
 450         return $rpcenv->fork_worker('cephdestroyosd', $osdsection,  $authuser, $worker);
 451     }});
 452
 453 __PACKAGE__->register_method ({
 454     name => 'in',
 455     path => '{osdid}/in',
 456     method => 'POST',
 457     description => "ceph osd in",
 458     proxyto => 'node',
 459     protected => 1,
 460     permissions => {
 461         check => ['perm', '/', [ 'Sys.Modify' ]],
 462     },
 463     parameters => {
 464         additionalProperties => 0,
 465         properties => {
 466             node => get_standard_option('pve-node'),
 467             osdid => {
 468                 description => 'OSD ID',
 469                 type => 'integer',
 470             },
 471         },
 472     },
 473     returns => { type => "null" },
 474     code => sub {
 475         my ($param) = @_;
 476
 477         PVE::Ceph::Tools::check_ceph_inited();
 478
 479         my $osdid = $param->{osdid};
 480
 481         my $rados = PVE::RADOS->new();
 482
 483         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 484
 485         my $osdsection = "osd.$osdid";
 486
 487         $rados->mon_command({ prefix => "osd in", ids => [ $osdsection ], format => 'plain' });
 488
 489         return undef;
 490     }});
 491
 492 __PACKAGE__->register_method ({
 493     name => 'out',
 494     path => '{osdid}/out',
 495     method => 'POST',
 496     description => "ceph osd out",
 497     proxyto => 'node',
 498     protected => 1,
 499     permissions => {
 500         check => ['perm', '/', [ 'Sys.Modify' ]],
 501     },
 502     parameters => {
 503         additionalProperties => 0,
 504         properties => {
 505             node => get_standard_option('pve-node'),
 506             osdid => {
 507                 description => 'OSD ID',
 508                 type => 'integer',
 509             },
 510         },
 511     },
 512     returns => { type => "null" },
 513     code => sub {
 514         my ($param) = @_;
 515
 516         PVE::Ceph::Tools::check_ceph_inited();
 517
 518         my $osdid = $param->{osdid};
 519
 520         my $rados = PVE::RADOS->new();
 521
 522         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 523
 524         my $osdsection = "osd.$osdid";
 525
 526         $rados->mon_command({ prefix => "osd out", ids => [ $osdsection ], format => 'plain' });
 527
 528         return undef;
 529     }});
 530
 531 package PVE::API2::Ceph;
 532
 533 use strict;
 534 use warnings;
 535
 536 use File::Path;
 537 use Net::IP;
 538 use UUID;
 539
 540 use PVE::Ceph::Tools;
 541 use PVE::Cluster qw(cfs_read_file cfs_write_file);
 542 use PVE::JSONSchema qw(get_standard_option);
 543 use PVE::Network;
 544 use PVE::RADOS;
 545 use PVE::RESTHandler;
 546 use PVE::RPCEnvironment;
 547 use PVE::Storage;
 548 use PVE::Tools qw(run_command file_get_contents file_set_contents);
 549
 550 use PVE::API2::Ceph::FS;
 551 use PVE::API2::Ceph::MDS;
 552 use PVE::API2::Storage::Config;
 553
 554 use base qw(PVE::RESTHandler);
 555
 556 my $pve_osd_default_journal_size = 1024*5;
 557
 558 __PACKAGE__->register_method ({
 559     subclass => "PVE::API2::CephOSD",
 560     path => 'osd',
 561 });
 562
 563 __PACKAGE__->register_method ({
 564     subclass => "PVE::API2::Ceph::MDS",
 565     path => 'mds',
 566 });
 567
 568 __PACKAGE__->register_method ({
 569     subclass => "PVE::API2::Ceph::FS",
 570     path => 'fs',
 571 });
 572
 573 __PACKAGE__->register_method ({
 574     name => 'index',
 575     path => '',
 576     method => 'GET',
 577     description => "Directory index.",
 578     permissions => { user => 'all' },
 579     permissions => {
 580         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 581     },
 582     parameters => {
 583         additionalProperties => 0,
 584         properties => {
 585             node => get_standard_option('pve-node'),
 586         },
 587     },
 588     returns => {
 589         type => 'array',
 590         items => {
 591             type => "object",
 592             properties => {},
 593         },
 594         links => [ { rel => 'child', href => "{name}" } ],
 595     },
 596     code => sub {
 597         my ($param) = @_;
 598
 599         my $result = [
 600             { name => 'init' },
 601             { name => 'mon' },
 602             { name => 'osd' },
 603             { name => 'pools' },
 604             { name => 'fs' },
 605             { name => 'mds' },
 606             { name => 'stop' },
 607             { name => 'start' },
 608             { name => 'status' },
 609             { name => 'crush' },
 610             { name => 'config' },
 611             { name => 'log' },
 612             { name => 'disks' },
 613             { name => 'flags' },
 614             { name => 'rules' },
 615         ];
 616
 617         return $result;
 618     }});
 619
 620 __PACKAGE__->register_method ({
 621     name => 'disks',
 622     path => 'disks',
 623     method => 'GET',
 624     description => "List local disks.",
 625     proxyto => 'node',
 626     protected => 1,
 627     permissions => {
 628         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 629     },
 630     parameters => {
 631         additionalProperties => 0,
 632         properties => {
 633             node => get_standard_option('pve-node'),
 634             type => {
 635                 description => "Only list specific types of disks.",
 636                 type => 'string',
 637                 enum => ['unused', 'journal_disks'],
 638                 optional => 1,
 639             },
 640         },
 641     },
 642     returns => {
 643         type => 'array',
 644         items => {
 645             type => "object",
 646             properties => {
 647                 dev => { type => 'string' },
 648                 used => { type => 'string', optional => 1 },
 649                 gpt => { type => 'boolean' },
 650                 size => { type => 'integer' },
 651                 osdid => { type => 'integer' },
 652                 vendor =>  { type => 'string', optional => 1 },
 653                 model =>  { type => 'string', optional => 1 },
 654                 serial =>  { type => 'string', optional => 1 },
 655             },
 656         },
 657         # links => [ { rel => 'child', href => "{}" } ],
 658     },
 659     code => sub {
 660         my ($param) = @_;
 661
 662         PVE::Ceph::Tools::check_ceph_inited();
 663
 664         my $disks = PVE::Diskmanage::get_disks(undef, 1);
 665
 666         my $res = [];
 667         foreach my $dev (keys %$disks) {
 668             my $d = $disks->{$dev};
 669             if ($param->{type}) {
 670                 if ($param->{type} eq 'journal_disks') {
 671                     next if $d->{osdid} >= 0;
 672                     next if !$d->{gpt};
 673                 } elsif ($param->{type} eq 'unused') {
 674                     next if $d->{used};
 675                 } else {
 676                     die "internal error"; # should not happen
 677                 }
 678             }
 679
 680             $d->{dev} = "/dev/$dev";
 681             push @$res, $d;
 682         }
 683
 684         return $res;
 685     }});
 686
 687 __PACKAGE__->register_method ({
 688     name => 'config',
 689     path => 'config',
 690     method => 'GET',
 691     permissions => {
 692         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 693     },
 694     description => "Get Ceph configuration.",
 695     parameters => {
 696         additionalProperties => 0,
 697         properties => {
 698             node => get_standard_option('pve-node'),
 699         },
 700     },
 701     returns => { type => 'string' },
 702     code => sub {
 703         my ($param) = @_;
 704
 705         PVE::Ceph::Tools::check_ceph_inited();
 706
 707         my $path = PVE::Ceph::Tools::get_config('pve_ceph_cfgpath');
 708         return file_get_contents($path);
 709
 710     }});
 711
 712 my $add_storage = sub {
 713     my ($pool, $storeid) = @_;
 714
 715     my $storage_params = {
 716         type => 'rbd',
 717         pool => $pool,
 718         storage => $storeid,
 719         krbd => 0,
 720         content => 'rootdir,images',
 721     };
 722
 723     PVE::API2::Storage::Config->create($storage_params);
 724 };
 725
 726 my $get_storages = sub {
 727     my ($pool) = @_;
 728
 729     my $cfg = PVE::Storage::config();
 730
 731     my $storages = $cfg->{ids};
 732     my $res = {};
 733     foreach my $storeid (keys %$storages) {
 734         my $curr = $storages->{$storeid};
 735         $res->{$storeid} = $storages->{$storeid}
 736             if $curr->{type} eq 'rbd' && $pool eq $curr->{pool};
 737     }
 738
 739     return $res;
 740 };
 741
 742 __PACKAGE__->register_method ({
 743     name => 'listmon',
 744     path => 'mon',
 745     method => 'GET',
 746     description => "Get Ceph monitor list.",
 747     proxyto => 'node',
 748     protected => 1,
 749     permissions => {
 750         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 751     },
 752     parameters => {
 753         additionalProperties => 0,
 754         properties => {
 755             node => get_standard_option('pve-node'),
 756         },
 757     },
 758     returns => {
 759         type => 'array',
 760         items => {
 761             type => "object",
 762             properties => {
 763                 name => { type => 'string' },
 764                 addr => { type => 'string' },
 765             },
 766         },
 767         links => [ { rel => 'child', href => "{name}" } ],
 768     },
 769     code => sub {
 770         my ($param) = @_;
 771
 772         PVE::Ceph::Tools::check_ceph_inited();
 773
 774         my $res = [];
 775
 776         my $cfg = cfs_read_file('ceph.conf');
 777
 778         my $monhash = {};
 779         foreach my $section (keys %$cfg) {
 780             my $d = $cfg->{$section};
 781             if ($section =~ m/^mon\.(\S+)$/) {
 782                 my $monid = $1;
 783                 if ($d->{'mon addr'} && $d->{'host'}) {
 784                     $monhash->{$monid} = {
 785                         addr => $d->{'mon addr'},
 786                         host => $d->{'host'},
 787                         name => $monid,
 788                     }
 789                 }
 790             }
 791         }
 792
 793         eval {
 794             my $rados = PVE::RADOS->new();
 795             my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 796             my $mons = $monstat->{monmap}->{mons};
 797             foreach my $d (@$mons) {
 798                 next if !defined($d->{name});
 799                 $monhash->{$d->{name}}->{rank} = $d->{rank};
 800                 $monhash->{$d->{name}}->{addr} = $d->{addr};
 801                 if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
 802                     $monhash->{$d->{name}}->{quorum} = 1;
 803                 }
 804             }
 805         };
 806         warn $@ if $@;
 807
 808         return PVE::RESTHandler::hash_to_array($monhash, 'name');
 809     }});
 810
 811 __PACKAGE__->register_method ({
 812     name => 'init',
 813     path => 'init',
 814     method => 'POST',
 815     description => "Create initial ceph default configuration and setup symlinks.",
 816     proxyto => 'node',
 817     protected => 1,
 818     permissions => {
 819         check => ['perm', '/', [ 'Sys.Modify' ]],
 820     },
 821     parameters => {
 822         additionalProperties => 0,
 823         properties => {
 824             node => get_standard_option('pve-node'),
 825             network => {
 826                 description => "Use specific network for all ceph related traffic",
 827                 type => 'string', format => 'CIDR',
 828                 optional => 1,
 829                 maxLength => 128,
 830             },
 831             'cluster-network' => {
 832                 description => "Declare a separate cluster network, OSDs will route" .
 833                     "heartbeat, object replication and recovery traffic over it",
 834                 type => 'string', format => 'CIDR',
 835                 requires => 'network',
 836                 optional => 1,
 837                 maxLength => 128,
 838             },
 839             size => {
 840                 description => 'Targeted number of replicas per object',
 841                 type => 'integer',
 842                 default => 3,
 843                 optional => 1,
 844                 minimum => 1,
 845                 maximum => 7,
 846             },
 847             min_size => {
 848                 description => 'Minimum number of available replicas per object to allow I/O',
 849                 type => 'integer',
 850                 default => 2,
 851                 optional => 1,
 852                 minimum => 1,
 853                 maximum => 7,
 854             },
 855             pg_bits => {
 856                 description => "Placement group bits, used to specify the " .
 857                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 858                     "default pg num' does not work for default pools.",
 859                 type => 'integer',
 860                 default => 6,
 861                 optional => 1,
 862                 minimum => 6,
 863                 maximum => 14,
 864             },
 865             disable_cephx => {
 866                 description => "Disable cephx authentification.\n\n" .
 867                     "WARNING: cephx is a security feature protecting against " .
 868                     "man-in-the-middle attacks. Only consider disabling cephx ".
 869                     "if your network is private!",
 870                 type => 'boolean',
 871                 optional => 1,
 872                 default => 0,
 873             },
 874         },
 875     },
 876     returns => { type => 'null' },
 877     code => sub {
 878         my ($param) = @_;
 879
 880         my $version = PVE::Ceph::Tools::get_local_version(1);
 881
 882         if (!$version || $version < 12) {
 883             die "Ceph Luminous required - please run 'pveceph install'\n";
 884         } else {
 885             PVE::Ceph::Tools::check_ceph_installed('ceph_bin');
 886         }
 887
 888         # simply load old config if it already exists
 889         my $cfg = cfs_read_file('ceph.conf');
 890
 891         if (!$cfg->{global}) {
 892
 893             my $fsid;
 894             my $uuid;
 895
 896             UUID::generate($uuid);
 897             UUID::unparse($uuid, $fsid);
 898
 899             my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 900
 901             $cfg->{global} = {
 902                 'fsid' => $fsid,
 903                 'auth cluster required' => $auth,
 904                 'auth service required' => $auth,
 905                 'auth client required' => $auth,
 906                 'osd journal size' => $pve_osd_default_journal_size,
 907                 'osd pool default size' => $param->{size} // 3,
 908                 'osd pool default min size' => $param->{min_size} // 2,
 909                 'mon allow pool delete' => 'true',
 910             };
 911
 912             # this does not work for default pools
 913             #'osd pool default pg num' => $pg_num,
 914             #'osd pool default pgp num' => $pg_num,
 915         }
 916
 917         $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 918         $cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring';
 919
 920         if ($param->{pg_bits}) {
 921             $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 922             $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 923         }
 924
 925         if ($param->{network}) {
 926             $cfg->{global}->{'public network'} = $param->{network};
 927             $cfg->{global}->{'cluster network'} = $param->{network};
 928         }
 929
 930         if ($param->{'cluster-network'}) {
 931             $cfg->{global}->{'cluster network'} = $param->{'cluster-network'};
 932         }
 933
 934         cfs_write_file('ceph.conf', $cfg);
 935
 936         PVE::Ceph::Tools::setup_pve_symlinks();
 937
 938         return undef;
 939     }});
 940
 941 my $find_mon_ip = sub {
 942     my ($pubnet, $node, $overwrite_ip) = @_;
 943
 944     if (!$pubnet) {
 945         return $overwrite_ip // PVE::Cluster::remote_node_ip($node);
 946     }
 947
 948     my $allowed_ips = PVE::Network::get_local_ip_from_cidr($pubnet);
 949     die "No IP configured and up from ceph public network '$pubnet'\n"
 950         if scalar(@$allowed_ips) < 1;
 951
 952     if (!$overwrite_ip) {
 953         if (scalar(@$allowed_ips) == 1) {
 954             return $allowed_ips->[0];
 955         }
 956         die "Multiple IPs for ceph public network '$pubnet' detected on $node:\n".
 957             join("\n", @$allowed_ips) ."\nuse 'mon-address' to specify one of them.\n";
 958     } else {
 959         if (grep { $_ eq $overwrite_ip } @$allowed_ips) {
 960             return $overwrite_ip;
 961         }
 962         die "Monitor IP '$overwrite_ip' not in ceph public network '$pubnet'\n"
 963             if !PVE::Network::is_ip_in_cidr($overwrite_ip, $pubnet);
 964
 965         die "Specified monitor IP '$overwrite_ip' not configured or up on $node!\n";
 966     }
 967 };
 968
 969 my $create_mgr = sub {
 970     my ($rados, $id) = @_;
 971
 972     my $clustername = PVE::Ceph::Tools::get_config('ccname');
 973     my $mgrdir = "/var/lib/ceph/mgr/$clustername-$id";
 974     my $mgrkeyring = "$mgrdir/keyring";
 975     my $mgrname = "mgr.$id";
 976
 977     die "ceph manager directory '$mgrdir' already exists\n"
 978         if -d $mgrdir;
 979
 980     print "creating manager directory '$mgrdir'\n";
 981     mkdir $mgrdir;
 982     print "creating keys for '$mgrname'\n";
 983     my $output = $rados->mon_command({ prefix => 'auth get-or-create',
 984                                        entity => $mgrname,
 985                                        caps => [
 986                                            mon => 'allow profile mgr',
 987                                            osd => 'allow *',
 988                                            mds => 'allow *',
 989                                        ],
 990                                        format => 'plain'});
 991     file_set_contents($mgrkeyring, $output);
 992
 993     print "setting owner for directory\n";
 994     run_command(["chown", 'ceph:ceph', '-R', $mgrdir]);
 995
 996     print "enabling service 'ceph-mgr\@$id.service'\n";
 997     PVE::Ceph::Tools::ceph_service_cmd('enable', $mgrname);
 998     print "starting service 'ceph-mgr\@$id.service'\n";
 999     PVE::Ceph::Tools::ceph_service_cmd('start', $mgrname);
1000 };
1001
1002 my $destroy_mgr = sub {
1003     my ($mgrid) = @_;
1004
1005     my $clustername = PVE::Ceph::Tools::get_config('ccname');
1006     my $mgrname = "mgr.$mgrid";
1007     my $mgrdir = "/var/lib/ceph/mgr/$clustername-$mgrid";
1008
1009     die "ceph manager directory '$mgrdir' not found\n"
1010         if ! -d $mgrdir;
1011
1012     print "disabling service 'ceph-mgr\@$mgrid.service'\n";
1013     PVE::Ceph::Tools::ceph_service_cmd('disable', $mgrname);
1014     print "stopping service 'ceph-mgr\@$mgrid.service'\n";
1015     PVE::Ceph::Tools::ceph_service_cmd('stop', $mgrname);
1016
1017     print "removing manager directory '$mgrdir'\n";
1018     File::Path::remove_tree($mgrdir);
1019 };
1020
1021 __PACKAGE__->register_method ({
1022     name => 'createmon',
1023     path => 'mon',
1024     method => 'POST',
1025     description => "Create Ceph Monitor and Manager",
1026     proxyto => 'node',
1027     protected => 1,
1028     permissions => {
1029         check => ['perm', '/', [ 'Sys.Modify' ]],
1030     },
1031     parameters => {
1032         additionalProperties => 0,
1033         properties => {
1034             node => get_standard_option('pve-node'),
1035             id => {
1036                 type => 'string',
1037                 optional => 1,
1038                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1039                 description => "The ID for the monitor, when omitted the same as the nodename",
1040             },
1041             'exclude-manager' => {
1042                 type => 'boolean',
1043                 optional => 1,
1044                 default => 0,
1045                 description => "When set, only a monitor will be created.",
1046             },
1047             'mon-address' => {
1048                 description => 'Overwrites autodetected monitor IP address. ' .
1049                                'Must be in the public network of ceph.',
1050                 type => 'string', format => 'ip',
1051                 optional => 1,
1052             },
1053         },
1054     },
1055     returns => { type => 'string' },
1056     code => sub {
1057         my ($param) = @_;
1058
1059         PVE::Ceph::Tools::check_ceph_installed('ceph_mon');
1060
1061         PVE::Ceph::Tools::check_ceph_installed('ceph_mgr')
1062             if (!$param->{'exclude-manager'});
1063
1064         PVE::Ceph::Tools::check_ceph_inited();
1065
1066         PVE::Ceph::Tools::setup_pve_symlinks();
1067
1068         my $rpcenv = PVE::RPCEnvironment::get();
1069
1070         my $authuser = $rpcenv->get_user();
1071
1072         my $cfg = cfs_read_file('ceph.conf');
1073
1074         my $moncount = 0;
1075
1076         my $monaddrhash = {};
1077
1078         my $systemd_managed = PVE::Ceph::Tools::systemd_managed();
1079
1080         foreach my $section (keys %$cfg) {
1081             next if $section eq 'global';
1082             my $d = $cfg->{$section};
1083             if ($section =~ m/^mon\./) {
1084                 $moncount++;
1085                 if ($d->{'mon addr'}) {
1086                     $monaddrhash->{$d->{'mon addr'}} = $section;
1087                 }
1088             }
1089         }
1090
1091         my $monid = $param->{id} // $param->{node};
1092
1093         my $monsection = "mon.$monid";
1094         my $pubnet = $cfg->{global}->{'public network'};
1095         my $ip = $find_mon_ip->($pubnet, $param->{node}, $param->{'mon-address'});
1096
1097         my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]:6789" : "$ip:6789";
1098         my $monname = $param->{node};
1099
1100         die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
1101         die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n"
1102             if $monaddrhash->{$monaddr};
1103
1104         my $worker = sub  {
1105             my $upid = shift;
1106
1107             my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
1108
1109             if (! -f $pve_ckeyring_path) {
1110                 run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
1111                             "--gen-key -n client.admin");
1112             }
1113
1114             my $pve_mon_key_path = PVE::Ceph::Tools::get_config('pve_mon_key_path');
1115             if (! -f $pve_mon_key_path) {
1116                 run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
1117                 run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " .
1118                             "--cap mds 'allow' " .
1119                             "--cap osd 'allow *' " .
1120                             "--cap mgr 'allow *' " .
1121                             "--cap mon 'allow *'");
1122                 run_command("cp $pve_mon_key_path.tmp /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
1123                 run_command("chown ceph:ceph /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
1124                 run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
1125                 run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
1126             }
1127
1128             my $ccname = PVE::Ceph::Tools::get_config('ccname');
1129
1130             my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
1131             -d $mondir && die "monitor filesystem '$mondir' already exist\n";
1132
1133             my $monmap = "/tmp/monmap";
1134
1135             eval {
1136                 mkdir $mondir;
1137
1138                 run_command("chown ceph:ceph $mondir") if $systemd_managed;
1139
1140                 if ($moncount > 0) {
1141                     my $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
1142                     my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
1143                     file_set_contents($monmap, $mapdata);
1144                 } else {
1145                     run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
1146                 }
1147
1148                 run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
1149                 run_command("chown ceph:ceph -R $mondir") if $systemd_managed;
1150             };
1151             my $err = $@;
1152             unlink $monmap;
1153             if ($err) {
1154                 File::Path::remove_tree($mondir);
1155                 die $err;
1156             }
1157
1158             $cfg->{$monsection} = {
1159                 'host' => $monname,
1160                 'mon addr' => $monaddr,
1161             };
1162
1163             cfs_write_file('ceph.conf', $cfg);
1164
1165             my $create_keys_pid = fork();
1166             if (!defined($create_keys_pid)) {
1167                 die "Could not spawn ceph-create-keys to create bootstrap keys\n";
1168             } elsif ($create_keys_pid == 0) {
1169                 exit PVE::Tools::run_command(['ceph-create-keys', '-i', $monid]);
1170             } else {
1171                 PVE::Ceph::Tools::ceph_service_cmd('start', $monsection);
1172
1173                 if ($systemd_managed) {
1174                     #to ensure we have the correct startup order.
1175                     eval { PVE::Tools::run_command(['/bin/systemctl', 'enable', "ceph-mon\@${monid}.service"]); };
1176                     warn "Enable ceph-mon\@${monid}.service manually"if $@;
1177                 }
1178                 waitpid($create_keys_pid, 0);
1179             }
1180
1181             # create manager
1182             if (!$param->{'exclude-manager'}) {
1183                 my $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
1184                 $create_mgr->($rados, $monid);
1185             }
1186         };
1187
1188         return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
1189     }});
1190
1191 __PACKAGE__->register_method ({
1192     name => 'destroymon',
1193     path => 'mon/{monid}',
1194     method => 'DELETE',
1195     description => "Destroy Ceph Monitor and Manager.",
1196     proxyto => 'node',
1197     protected => 1,
1198     permissions => {
1199         check => ['perm', '/', [ 'Sys.Modify' ]],
1200     },
1201     parameters => {
1202         additionalProperties => 0,
1203         properties => {
1204             node => get_standard_option('pve-node'),
1205             monid => {
1206                 description => 'Monitor ID',
1207                 type => 'string',
1208                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1209             },
1210             'exclude-manager' => {
1211                 type => 'boolean',
1212                 default => 0,
1213                 optional => 1,
1214                 description => "When set, removes only the monitor, not the manager"
1215             }
1216         },
1217     },
1218     returns => { type => 'string' },
1219     code => sub {
1220         my ($param) = @_;
1221
1222         my $rpcenv = PVE::RPCEnvironment::get();
1223
1224         my $authuser = $rpcenv->get_user();
1225
1226         PVE::Ceph::Tools::check_ceph_inited();
1227
1228         my $cfg = cfs_read_file('ceph.conf');
1229
1230         my $monid = $param->{monid};
1231         my $monsection = "mon.$monid";
1232
1233         my $rados = PVE::RADOS->new();
1234         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
1235         my $monlist = $monstat->{monmap}->{mons};
1236
1237         die "no such monitor id '$monid'\n"
1238             if !defined($cfg->{$monsection});
1239
1240         my $ccname = PVE::Ceph::Tools::get_config('ccname');
1241
1242         my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
1243         -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n";
1244
1245         die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
1246
1247         my $worker = sub {
1248             my $upid = shift;
1249
1250             # reopen with longer timeout
1251             $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
1252
1253             $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });
1254
1255             eval { PVE::Ceph::Tools::ceph_service_cmd('stop', $monsection); };
1256             warn $@ if $@;
1257
1258             delete $cfg->{$monsection};
1259             cfs_write_file('ceph.conf', $cfg);
1260             File::Path::remove_tree($mondir);
1261
1262             # remove manager
1263             if (!$param->{'exclude-manager'}) {
1264                 eval { $destroy_mgr->($monid); };
1265                 warn $@ if $@;
1266             }
1267         };
1268
1269         return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
1270     }});
1271
1272 __PACKAGE__->register_method ({
1273     name => 'createmgr',
1274     path => 'mgr',
1275     method => 'POST',
1276     description => "Create Ceph Manager",
1277     proxyto => 'node',
1278     protected => 1,
1279     permissions => {
1280         check => ['perm', '/', [ 'Sys.Modify' ]],
1281     },
1282     parameters => {
1283         additionalProperties => 0,
1284         properties => {
1285             node => get_standard_option('pve-node'),
1286             id => {
1287                 type => 'string',
1288                 optional => 1,
1289                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1290                 description => "The ID for the manager, when omitted the same as the nodename",
1291             },
1292         },
1293     },
1294     returns => { type => 'string' },
1295     code => sub {
1296         my ($param) = @_;
1297
1298         PVE::Ceph::Tools::check_ceph_installed('ceph_mgr');
1299
1300         PVE::Ceph::Tools::check_ceph_inited();
1301
1302         my $rpcenv = PVE::RPCEnvironment::get();
1303
1304         my $authuser = $rpcenv->get_user();
1305
1306         my $mgrid = $param->{id} // $param->{node};
1307
1308         my $worker = sub  {
1309             my $upid = shift;
1310
1311             my $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
1312
1313             $create_mgr->($rados, $mgrid);
1314         };
1315
1316         return $rpcenv->fork_worker('cephcreatemgr', "mgr.$mgrid", $authuser, $worker);
1317     }});
1318
1319 __PACKAGE__->register_method ({
1320     name => 'destroymgr',
1321     path => 'mgr/{id}',
1322     method => 'DELETE',
1323     description => "Destroy Ceph Manager.",
1324     proxyto => 'node',
1325     protected => 1,
1326     permissions => {
1327         check => ['perm', '/', [ 'Sys.Modify' ]],
1328     },
1329     parameters => {
1330         additionalProperties => 0,
1331         properties => {
1332             node => get_standard_option('pve-node'),
1333             id => {
1334                 description => 'The ID of the manager',
1335                 type => 'string',
1336                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1337             },
1338         },
1339     },
1340     returns => { type => 'string' },
1341     code => sub {
1342         my ($param) = @_;
1343
1344         my $rpcenv = PVE::RPCEnvironment::get();
1345
1346         my $authuser = $rpcenv->get_user();
1347
1348         PVE::Ceph::Tools::check_ceph_inited();
1349
1350         my $mgrid = $param->{id};
1351
1352         my $worker = sub {
1353             my $upid = shift;
1354
1355             $destroy_mgr->($mgrid);
1356         };
1357
1358         return $rpcenv->fork_worker('cephdestroymgr', "mgr.$mgrid",  $authuser, $worker);
1359     }});
1360
1361 __PACKAGE__->register_method ({
1362     name => 'stop',
1363     path => 'stop',
1364     method => 'POST',
1365     description => "Stop ceph services.",
1366     proxyto => 'node',
1367     protected => 1,
1368     permissions => {
1369         check => ['perm', '/', [ 'Sys.Modify' ]],
1370     },
1371     parameters => {
1372         additionalProperties => 0,
1373         properties => {
1374             node => get_standard_option('pve-node'),
1375             service => {
1376                 description => 'Ceph service name.',
1377                 type => 'string',
1378                 optional => 1,
1379                 default => 'ceph.target',
1380                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1381             },
1382         },
1383     },
1384     returns => { type => 'string' },
1385     code => sub {
1386         my ($param) = @_;
1387
1388         my $rpcenv = PVE::RPCEnvironment::get();
1389
1390         my $authuser = $rpcenv->get_user();
1391
1392         PVE::Ceph::Tools::check_ceph_inited();
1393
1394         my $cfg = cfs_read_file('ceph.conf');
1395         scalar(keys %$cfg) || die "no configuration\n";
1396
1397         my $worker = sub {
1398             my $upid = shift;
1399
1400             my $cmd = ['stop'];
1401             if ($param->{service}) {
1402                 push @$cmd, $param->{service};
1403             }
1404
1405             PVE::Ceph::Tools::ceph_service_cmd(@$cmd);
1406         };
1407
1408         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
1409                                     $authuser, $worker);
1410     }});
1411
1412 __PACKAGE__->register_method ({
1413     name => 'start',
1414     path => 'start',
1415     method => 'POST',
1416     description => "Start ceph services.",
1417     proxyto => 'node',
1418     protected => 1,
1419     permissions => {
1420         check => ['perm', '/', [ 'Sys.Modify' ]],
1421     },
1422     parameters => {
1423         additionalProperties => 0,
1424         properties => {
1425             node => get_standard_option('pve-node'),
1426             service => {
1427                 description => 'Ceph service name.',
1428                 type => 'string',
1429                 optional => 1,
1430                 default => 'ceph.target',
1431                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1432             },
1433         },
1434     },
1435     returns => { type => 'string' },
1436     code => sub {
1437         my ($param) = @_;
1438
1439         my $rpcenv = PVE::RPCEnvironment::get();
1440
1441         my $authuser = $rpcenv->get_user();
1442
1443         PVE::Ceph::Tools::check_ceph_inited();
1444
1445         my $cfg = cfs_read_file('ceph.conf');
1446         scalar(keys %$cfg) || die "no configuration\n";
1447
1448         my $worker = sub {
1449             my $upid = shift;
1450
1451             my $cmd = ['start'];
1452             if ($param->{service}) {
1453                 push @$cmd, $param->{service};
1454             }
1455
1456             PVE::Ceph::Tools::ceph_service_cmd(@$cmd);
1457         };
1458
1459         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
1460                                     $authuser, $worker);
1461     }});
1462
1463 __PACKAGE__->register_method ({
1464     name => 'restart',
1465     path => 'restart',
1466     method => 'POST',
1467     description => "Restart ceph services.",
1468     proxyto => 'node',
1469     protected => 1,
1470     permissions => {
1471         check => ['perm', '/', [ 'Sys.Modify' ]],
1472     },
1473     parameters => {
1474         additionalProperties => 0,
1475         properties => {
1476             node => get_standard_option('pve-node'),
1477             service => {
1478                 description => 'Ceph service name.',
1479                 type => 'string',
1480                 optional => 1,
1481                 default => 'ceph.target',
1482                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1483             },
1484         },
1485     },
1486     returns => { type => 'string' },
1487     code => sub {
1488         my ($param) = @_;
1489
1490         my $rpcenv = PVE::RPCEnvironment::get();
1491
1492         my $authuser = $rpcenv->get_user();
1493
1494         PVE::Ceph::Tools::check_ceph_inited();
1495
1496         my $cfg = cfs_read_file('ceph.conf');
1497         scalar(keys %$cfg) || die "no configuration\n";
1498
1499         my $worker = sub {
1500             my $upid = shift;
1501
1502             my $cmd = ['restart'];
1503             if ($param->{service}) {
1504                 push @$cmd, $param->{service};
1505             }
1506
1507             PVE::Ceph::Tools::ceph_service_cmd(@$cmd);
1508         };
1509
1510         return $rpcenv->fork_worker('srvrestart', $param->{service} || 'ceph',
1511                                     $authuser, $worker);
1512     }});
1513
1514 __PACKAGE__->register_method ({
1515     name => 'status',
1516     path => 'status',
1517     method => 'GET',
1518     description => "Get ceph status.",
1519     proxyto => 'node',
1520     protected => 1,
1521     permissions => {
1522         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1523     },
1524     parameters => {
1525         additionalProperties => 0,
1526         properties => {
1527             node => get_standard_option('pve-node'),
1528         },
1529     },
1530     returns => { type => 'object' },
1531     code => sub {
1532         my ($param) = @_;
1533
1534         PVE::Ceph::Tools::check_ceph_enabled();
1535
1536         my $rados = PVE::RADOS->new();
1537         my $status = $rados->mon_command({ prefix => 'status' });
1538         $status->{health} = $rados->mon_command({ prefix => 'health', detail => 'detail' });
1539         return $status;
1540     }});
1541
1542 __PACKAGE__->register_method ({
1543     name => 'lspools',
1544     path => 'pools',
1545     method => 'GET',
1546     description => "List all pools.",
1547     proxyto => 'node',
1548     protected => 1,
1549     permissions => {
1550         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1551     },
1552     parameters => {
1553         additionalProperties => 0,
1554         properties => {
1555             node => get_standard_option('pve-node'),
1556         },
1557     },
1558     returns => {
1559         type => 'array',
1560         items => {
1561             type => "object",
1562             properties => {
1563                 pool => { type => 'integer' },
1564                 pool_name => { type => 'string' },
1565                 size => { type => 'integer' },
1566             },
1567         },
1568         links => [ { rel => 'child', href => "{pool_name}" } ],
1569     },
1570     code => sub {
1571         my ($param) = @_;
1572
1573         PVE::Ceph::Tools::check_ceph_inited();
1574
1575         my $rados = PVE::RADOS->new();
1576
1577         my $stats = {};
1578         my $res = $rados->mon_command({ prefix => 'df' });
1579
1580         foreach my $d (@{$res->{pools}}) {
1581             next if !$d->{stats};
1582             next if !defined($d->{id});
1583             $stats->{$d->{id}} = $d->{stats};
1584         }
1585
1586         $res = $rados->mon_command({ prefix => 'osd dump' });
1587         my $rulestmp = $rados->mon_command({ prefix => 'osd crush rule dump'});
1588
1589         my $rules = {};
1590         for my $rule (@$rulestmp) {
1591             $rules->{$rule->{rule_id}} = $rule->{rule_name};
1592         }
1593
1594         my $data = [];
1595         foreach my $e (@{$res->{pools}}) {
1596             my $d = {};
1597             foreach my $attr (qw(pool pool_name size min_size pg_num crush_rule)) {
1598                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
1599             }
1600
1601             if (defined($d->{crush_rule}) && defined($rules->{$d->{crush_rule}})) {
1602                 $d->{crush_rule_name} = $rules->{$d->{crush_rule}};
1603             }
1604
1605             if (my $s = $stats->{$d->{pool}}) {
1606                 $d->{bytes_used} = $s->{bytes_used};
1607                 $d->{percent_used} = $s->{percent_used};
1608             }
1609             push @$data, $d;
1610         }
1611
1612
1613         return $data;
1614     }});
1615
1616 __PACKAGE__->register_method ({
1617     name => 'createpool',
1618     path => 'pools',
1619     method => 'POST',
1620     description => "Create POOL",
1621     proxyto => 'node',
1622     protected => 1,
1623     permissions => {
1624         check => ['perm', '/', [ 'Sys.Modify' ]],
1625     },
1626     parameters => {
1627         additionalProperties => 0,
1628         properties => {
1629             node => get_standard_option('pve-node'),
1630             name => {
1631                 description => "The name of the pool. It must be unique.",
1632                 type => 'string',
1633             },
1634             size => {
1635                 description => 'Number of replicas per object',
1636                 type => 'integer',
1637                 default => 3,
1638                 optional => 1,
1639                 minimum => 1,
1640                 maximum => 7,
1641             },
1642             min_size => {
1643                 description => 'Minimum number of replicas per object',
1644                 type => 'integer',
1645                 default => 2,
1646                 optional => 1,
1647                 minimum => 1,
1648                 maximum => 7,
1649             },
1650             pg_num => {
1651                 description => "Number of placement groups.",
1652                 type => 'integer',
1653                 default => 128,
1654                 optional => 1,
1655                 minimum => 8,
1656                 maximum => 32768,
1657             },
1658             crush_rule => {
1659                 description => "The rule to use for mapping object placement in the cluster.",
1660                 type => 'string',
1661                 optional => 1,
1662             },
1663             application => {
1664                 description => "The application of the pool, 'rbd' by default.",
1665                 type => 'string',
1666                 enum => ['rbd', 'cephfs', 'rgw'],
1667                 optional => 1,
1668             },
1669             add_storages => {
1670                 description => "Configure VM and CT storage using the new pool.",
1671                 type => 'boolean',
1672                 optional => 1,
1673             },
1674         },
1675     },
1676     returns => { type => 'string' },
1677     code => sub {
1678         my ($param) = @_;
1679
1680         PVE::Cluster::check_cfs_quorum();
1681         PVE::Ceph::Tools::check_ceph_inited();
1682
1683         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
1684
1685         die "not fully configured - missing '$pve_ckeyring_path'\n"
1686             if ! -f $pve_ckeyring_path;
1687
1688         my $pool = $param->{name};
1689         my $rpcenv = PVE::RPCEnvironment::get();
1690         my $user = $rpcenv->get_user();
1691
1692         if ($param->{add_storages}) {
1693             $rpcenv->check($user, '/storage', ['Datastore.Allocate']);
1694             die "pool name contains characters which are illegal for storage naming\n"
1695                 if !PVE::JSONSchema::parse_storage_id($pool);
1696         }
1697
1698         my $pg_num = $param->{pg_num} || 128;
1699         my $size = $param->{size} || 3;
1700         my $min_size = $param->{min_size} || 2;
1701         my $application = $param->{application} // 'rbd';
1702
1703         my $worker = sub {
1704
1705             PVE::Ceph::Tools::create_pool($pool, $param);
1706
1707             if ($param->{add_storages}) {
1708                 my $err;
1709                 eval { $add_storage->($pool, "${pool}"); };
1710                 if ($@) {
1711                     warn "failed to add storage: $@";
1712                     $err = 1;
1713                 }
1714                 die "adding storage for pool '$pool' failed, check log and add manually!\n"
1715                     if $err;
1716             }
1717         };
1718
1719         return $rpcenv->fork_worker('cephcreatepool', $pool,  $user, $worker);
1720     }});
1721
1722 __PACKAGE__->register_method ({
1723     name => 'get_flags',
1724     path => 'flags',
1725     method => 'GET',
1726     description => "get all set ceph flags",
1727     proxyto => 'node',
1728     protected => 1,
1729     permissions => {
1730         check => ['perm', '/', [ 'Sys.Audit' ]],
1731     },
1732     parameters => {
1733         additionalProperties => 0,
1734         properties => {
1735             node => get_standard_option('pve-node'),
1736         },
1737     },
1738     returns => { type => 'string' },
1739     code => sub {
1740         my ($param) = @_;
1741
1742         PVE::Ceph::Tools::check_ceph_inited();
1743
1744         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
1745
1746         die "not fully configured - missing '$pve_ckeyring_path'\n"
1747             if ! -f $pve_ckeyring_path;
1748
1749         my $rados = PVE::RADOS->new();
1750
1751         my $stat = $rados->mon_command({ prefix => 'osd dump' });
1752
1753         return $stat->{flags} // '';
1754     }});
1755
1756 __PACKAGE__->register_method ({
1757     name => 'set_flag',
1758     path => 'flags/{flag}',
1759     method => 'POST',
1760     description => "Set a ceph flag",
1761     proxyto => 'node',
1762     protected => 1,
1763     permissions => {
1764         check => ['perm', '/', [ 'Sys.Modify' ]],
1765     },
1766     parameters => {
1767         additionalProperties => 0,
1768         properties => {
1769             node => get_standard_option('pve-node'),
1770             flag => {
1771                 description => 'The ceph flag to set/unset',
1772                 type => 'string',
1773                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1774             },
1775         },
1776     },
1777     returns => { type => 'null' },
1778     code => sub {
1779         my ($param) = @_;
1780
1781         PVE::Ceph::Tools::check_ceph_inited();
1782
1783         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
1784
1785         die "not fully configured - missing '$pve_ckeyring_path'\n"
1786             if ! -f $pve_ckeyring_path;
1787
1788         my $set = $param->{set} // !$param->{unset};
1789         my $rados = PVE::RADOS->new();
1790
1791         $rados->mon_command({
1792             prefix => "osd set",
1793             key => $param->{flag},
1794         });
1795
1796         return undef;
1797     }});
1798
1799 __PACKAGE__->register_method ({
1800     name => 'unset_flag',
1801     path => 'flags/{flag}',
1802     method => 'DELETE',
1803     description => "Unset a ceph flag",
1804     proxyto => 'node',
1805     protected => 1,
1806     permissions => {
1807         check => ['perm', '/', [ 'Sys.Modify' ]],
1808     },
1809     parameters => {
1810         additionalProperties => 0,
1811         properties => {
1812             node => get_standard_option('pve-node'),
1813             flag => {
1814                 description => 'The ceph flag to set/unset',
1815                 type => 'string',
1816                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1817             },
1818         },
1819     },
1820     returns => { type => 'null' },
1821     code => sub {
1822         my ($param) = @_;
1823
1824         PVE::Ceph::Tools::check_ceph_inited();
1825
1826         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
1827
1828         die "not fully configured - missing '$pve_ckeyring_path'\n"
1829             if ! -f $pve_ckeyring_path;
1830
1831         my $set = $param->{set} // !$param->{unset};
1832         my $rados = PVE::RADOS->new();
1833
1834         $rados->mon_command({
1835             prefix => "osd unset",
1836             key => $param->{flag},
1837         });
1838
1839         return undef;
1840     }});
1841
1842 __PACKAGE__->register_method ({
1843     name => 'destroypool',
1844     path => 'pools/{name}',
1845     method => 'DELETE',
1846     description => "Destroy pool",
1847     proxyto => 'node',
1848     protected => 1,
1849     permissions => {
1850         check => ['perm', '/', [ 'Sys.Modify' ]],
1851     },
1852     parameters => {
1853         additionalProperties => 0,
1854         properties => {
1855             node => get_standard_option('pve-node'),
1856             name => {
1857                 description => "The name of the pool. It must be unique.",
1858                 type => 'string',
1859             },
1860             force => {
1861                 description => "If true, destroys pool even if in use",
1862                 type => 'boolean',
1863                 optional => 1,
1864                 default => 0,
1865             },
1866             remove_storages => {
1867                 description => "Remove all pveceph-managed storages configured for this pool",
1868                 type => 'boolean',
1869                 optional => 1,
1870                 default => 0,
1871             },
1872         },
1873     },
1874     returns => { type => 'string' },
1875     code => sub {
1876         my ($param) = @_;
1877
1878         PVE::Ceph::Tools::check_ceph_inited();
1879
1880         my $rpcenv = PVE::RPCEnvironment::get();
1881         my $user = $rpcenv->get_user();
1882         $rpcenv->check($user, '/storage', ['Datastore.Allocate'])
1883             if $param->{remove_storages};
1884
1885         my $pool = $param->{name};
1886
1887         my $worker = sub {
1888             my $storages = $get_storages->($pool);
1889
1890             # if not forced, destroy ceph pool only when no
1891             # vm disks are on it anymore
1892             if (!$param->{force}) {
1893                 my $storagecfg = PVE::Storage::config();
1894                 foreach my $storeid (keys %$storages) {
1895                     my $storage = $storages->{$storeid};
1896
1897                     # check if any vm disks are on the pool
1898                     print "checking storage '$storeid' for RBD images..\n";
1899                     my $res = PVE::Storage::vdisk_list($storagecfg, $storeid);
1900                     die "ceph pool '$pool' still in use by storage '$storeid'\n"
1901                         if @{$res->{$storeid}} != 0;
1902                 }
1903             }
1904
1905             PVE::Ceph::Tools::destroy_pool($pool);
1906
1907             if ($param->{remove_storages}) {
1908                 my $err;
1909                 foreach my $storeid (keys %$storages) {
1910                     # skip external clusters, not managed by pveceph
1911                     next if $storages->{$storeid}->{monhost};
1912                     eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
1913                     if ($@) {
1914                         warn "failed to remove storage '$storeid': $@\n";
1915                         $err = 1;
1916                     }
1917                 }
1918                 die "failed to remove (some) storages - check log and remove manually!\n"
1919                     if $err;
1920             }
1921         };
1922         return $rpcenv->fork_worker('cephdestroypool', $pool,  $user, $worker);
1923     }});
1924
1925
1926 __PACKAGE__->register_method ({
1927     name => 'crush',
1928     path => 'crush',
1929     method => 'GET',
1930     description => "Get OSD crush map",
1931     proxyto => 'node',
1932     protected => 1,
1933     permissions => {
1934         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1935     },
1936     parameters => {
1937         additionalProperties => 0,
1938         properties => {
1939             node => get_standard_option('pve-node'),
1940         },
1941     },
1942     returns => { type => 'string' },
1943     code => sub {
1944         my ($param) = @_;
1945
1946         PVE::Ceph::Tools::check_ceph_inited();
1947
1948         # this produces JSON (difficult to read for the user)
1949         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
1950
1951         my $txt = '';
1952
1953         my $mapfile = "/var/tmp/ceph-crush.map.$$";
1954         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
1955
1956         my $rados = PVE::RADOS->new();
1957
1958         eval {
1959             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
1960             file_set_contents($mapfile, $bindata);
1961             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
1962             $txt = file_get_contents($mapdata);
1963         };
1964         my $err = $@;
1965
1966         unlink $mapfile;
1967         unlink $mapdata;
1968
1969         die $err if $err;
1970
1971         return $txt;
1972     }});
1973
1974 __PACKAGE__->register_method({
1975     name => 'log',
1976     path => 'log',
1977     method => 'GET',
1978     description => "Read ceph log",
1979     proxyto => 'node',
1980     permissions => {
1981         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
1982     },
1983     protected => 1,
1984     parameters => {
1985         additionalProperties => 0,
1986         properties => {
1987             node => get_standard_option('pve-node'),
1988             start => {
1989                 type => 'integer',
1990                 minimum => 0,
1991                 optional => 1,
1992             },
1993             limit => {
1994                 type => 'integer',
1995                 minimum => 0,
1996                 optional => 1,
1997             },
1998         },
1999     },
2000     returns => {
2001         type => 'array',
2002         items => {
2003             type => "object",
2004             properties => {
2005                 n => {
2006                   description=>  "Line number",
2007                   type=> 'integer',
2008                 },
2009                 t => {
2010                   description=>  "Line text",
2011                   type => 'string',
2012                 }
2013             }
2014         }
2015     },
2016     code => sub {
2017         my ($param) = @_;
2018
2019         PVE::Ceph::Tools::check_ceph_inited();
2020
2021         my $rpcenv = PVE::RPCEnvironment::get();
2022         my $user = $rpcenv->get_user();
2023         my $node = $param->{node};
2024
2025         my $logfile = "/var/log/ceph/ceph.log";
2026         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
2027
2028         $rpcenv->set_result_attrib('total', $count);
2029
2030         return $lines;
2031     }});
2032
2033 __PACKAGE__->register_method ({
2034     name => 'rules',
2035     path => 'rules',
2036     method => 'GET',
2037     description => "List ceph rules.",
2038     proxyto => 'node',
2039     protected => 1,
2040     permissions => {
2041         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
2042     },
2043     parameters => {
2044         additionalProperties => 0,
2045         properties => {
2046             node => get_standard_option('pve-node'),
2047         },
2048     },
2049     returns => {
2050         type => 'array',
2051         items => {
2052             type => "object",
2053             properties => {},
2054         },
2055         links => [ { rel => 'child', href => "{name}" } ],
2056     },
2057     code => sub {
2058         my ($param) = @_;
2059
2060         PVE::Ceph::Tools::check_ceph_inited();
2061
2062         my $rados = PVE::RADOS->new();
2063
2064         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
2065
2066         my $res = [];
2067
2068         foreach my $rule (@$rules) {
2069             push @$res, { name => $rule };
2070         }
2071
2072         return $res;
2073     }});