PVE/API2/Ceph.pm

   1 package PVE::API2::CephOSD;
   2
   3 use strict;
   4 use warnings;
   5 use Cwd qw(abs_path);
   6 use Net::IP;
   7
   8 use PVE::SafeSyslog;
   9 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
  10 use PVE::Exception qw(raise raise_param_exc);
  11 use PVE::INotify;
  12 use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
  13 use PVE::AccessControl;
  14 use PVE::Storage;
  15 use PVE::RESTHandler;
  16 use PVE::RPCEnvironment;
  17 use PVE::JSONSchema qw(get_standard_option);
  18 use PVE::RADOS;
  19 use PVE::CephTools;
  20 use PVE::Diskmanage;
  21
  22 use base qw(PVE::RESTHandler);
  23
  24 use Data::Dumper; # fixme: remove
  25
  26 my $get_osd_status = sub {
  27     my ($rados, $osdid) = @_;
  28
  29     my $stat = $rados->mon_command({ prefix => 'osd dump' });
  30
  31     my $osdlist = $stat->{osds} || [];
  32
  33     my $flags = $stat->{flags} || undef;
  34
  35     my $osdstat;
  36     foreach my $d (@$osdlist) {
  37         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  38     }
  39     if (defined($osdid)) {
  40         die "no such OSD '$osdid'\n" if !$osdstat->{$osdid};
  41         return $osdstat->{$osdid};
  42     }
  43
  44     return wantarray? ($osdstat, $flags):$osdstat;
  45 };
  46
  47 my $get_osd_usage = sub {
  48     my ($rados) = @_;
  49
  50     my $osdlist = $rados->mon_command({ prefix => 'pg dump',
  51                                         dumpcontents => [ 'osds' ]}) || [];
  52
  53     my $osdstat;
  54     foreach my $d (@$osdlist) {
  55         $osdstat->{$d->{osd}} = $d if defined($d->{osd});
  56     }
  57
  58     return $osdstat;
  59 };
  60
  61 __PACKAGE__->register_method ({
  62     name => 'index',
  63     path => '',
  64     method => 'GET',
  65     description => "Get Ceph osd list/tree.",
  66     proxyto => 'node',
  67     protected => 1,
  68     permissions => {
  69         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  70     },
  71     parameters => {
  72         additionalProperties => 0,
  73         properties => {
  74             node => get_standard_option('pve-node'),
  75         },
  76     },
  77     # fixme: return a list instead of extjs tree format ?
  78     returns => {
  79         type => "object",
  80     },
  81     code => sub {
  82         my ($param) = @_;
  83
  84         PVE::CephTools::check_ceph_inited();
  85
  86         my $rados = PVE::RADOS->new();
  87         my $res = $rados->mon_command({ prefix => 'osd tree' });
  88
  89         die "no tree nodes found\n" if !($res && $res->{nodes});
  90
  91         my ($osdhash, $flags) = &$get_osd_status($rados);
  92
  93         my $usagehash = &$get_osd_usage($rados);
  94
  95         my $nodes = {};
  96         my $newnodes = {};
  97         foreach my $e (@{$res->{nodes}}) {
  98             $nodes->{$e->{id}} = $e;
  99
 100             my $new = {
 101                 id => $e->{id},
 102                 name => $e->{name},
 103                 type => $e->{type}
 104             };
 105
 106             foreach my $opt (qw(status crush_weight reweight)) {
 107                 $new->{$opt} = $e->{$opt} if defined($e->{$opt});
 108             }
 109
 110             if (my $stat = $osdhash->{$e->{id}}) {
 111                 $new->{in} = $stat->{in} if defined($stat->{in});
 112             }
 113
 114             if (my $stat = $usagehash->{$e->{id}}) {
 115                 $new->{total_space} = ($stat->{kb} || 1) * 1024;
 116                 $new->{bytes_used} = ($stat->{kb_used} || 0) * 1024;
 117                 $new->{percent_used} = ($new->{bytes_used}*100)/$new->{total_space};
 118                 if (my $d = $stat->{fs_perf_stat}) {
 119                     $new->{commit_latency_ms} = $d->{commit_latency_ms};
 120                     $new->{apply_latency_ms} = $d->{apply_latency_ms};
 121                 }
 122             }
 123
 124             $newnodes->{$e->{id}} = $new;
 125         }
 126
 127         foreach my $e (@{$res->{nodes}}) {
 128             my $new = $newnodes->{$e->{id}};
 129             if ($e->{children} && scalar(@{$e->{children}})) {
 130                 $new->{children} = [];
 131                 $new->{leaf} = 0;
 132                 foreach my $cid (@{$e->{children}}) {
 133                     $nodes->{$cid}->{parent} = $e->{id};
 134                     if ($nodes->{$cid}->{type} eq 'osd' &&
 135                         $e->{type} eq 'host') {
 136                         $newnodes->{$cid}->{host} = $e->{name};
 137                     }
 138                     push @{$new->{children}}, $newnodes->{$cid};
 139                 }
 140             } else {
 141                 $new->{leaf} = ($e->{id} >= 0) ? 1 : 0;
 142             }
 143         }
 144
 145         my $roots = [];
 146         foreach my $e (@{$res->{nodes}}) {
 147             if (!$nodes->{$e->{id}}->{parent}) {
 148                 push @$roots, $newnodes->{$e->{id}};
 149             }
 150         }
 151
 152         die "no root node\n" if !@$roots;
 153
 154         my $data = { root => { leaf =>  0, children => $roots } };
 155
 156         # we want this for the noout flag
 157         $data->{flags} = $flags if $flags;
 158
 159         return $data;
 160     }});
 161
 162 __PACKAGE__->register_method ({
 163     name => 'createosd',
 164     path => '',
 165     method => 'POST',
 166     description => "Create OSD",
 167     proxyto => 'node',
 168     protected => 1,
 169     parameters => {
 170         additionalProperties => 0,
 171         properties => {
 172             node => get_standard_option('pve-node'),
 173             dev => {
 174                 description => "Block device name.",
 175                 type => 'string',
 176             },
 177             journal_dev => {
 178                 description => "Block device name for journal.",
 179                 optional => 1,
 180                 type => 'string',
 181             },
 182             fstype => {
 183                 description => "File system type (filestore only).",
 184                 type => 'string',
 185                 enum => ['xfs', 'ext4', 'btrfs'],
 186                 default => 'xfs',
 187                 optional => 1,
 188             },
 189             bluestore => {
 190                 description => "Use bluestore instead of filestore.",
 191                 type => 'boolean',
 192                 default => 0,
 193                 optional => 1,
 194             },
 195         },
 196     },
 197     returns => { type => 'string' },
 198     code => sub {
 199         my ($param) = @_;
 200
 201         my $rpcenv = PVE::RPCEnvironment::get();
 202
 203         my $authuser = $rpcenv->get_user();
 204
 205         raise_param_exc({ 'bluestore' => "conflicts with parameter 'fstype'" })
 206             if (defined($param->{fstype}) && defined($param->{bluestore}) && $param->{bluestore});
 207
 208         PVE::CephTools::check_ceph_inited();
 209
 210         PVE::CephTools::setup_pve_symlinks();
 211
 212         my $journal_dev;
 213
 214         if ($param->{journal_dev} && ($param->{journal_dev} ne $param->{dev})) {
 215             $journal_dev = PVE::Diskmanage::verify_blockdev_path($param->{journal_dev});
 216         }
 217
 218         $param->{dev} = PVE::Diskmanage::verify_blockdev_path($param->{dev});
 219
 220         my $devname = $param->{dev};
 221         $devname =~ s|/dev/||;
 222
 223         my $disklist = PVE::Diskmanage::get_disks($devname, 1);
 224
 225         my $diskinfo = $disklist->{$devname};
 226         die "unable to get device info for '$devname'\n"
 227             if !$diskinfo;
 228
 229         die "device '$param->{dev}' is in use\n"
 230             if $diskinfo->{used};
 231
 232         my $devpath = $diskinfo->{devpath};
 233         my $rados = PVE::RADOS->new();
 234         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 235         die "unable to get fsid\n" if !$monstat->{monmap} || !$monstat->{monmap}->{fsid};
 236
 237         my $fsid = $monstat->{monmap}->{fsid};
 238         $fsid = $1 if $fsid =~ m/^([0-9a-f\-]+)$/;
 239
 240         my $ceph_bootstrap_osd_keyring = PVE::CephTools::get_config('ceph_bootstrap_osd_keyring');
 241
 242         if (! -f $ceph_bootstrap_osd_keyring) {
 243             my $bindata = $rados->mon_command({ prefix => 'auth get', entity => 'client.bootstrap-osd', format => 'plain' });
 244             PVE::Tools::file_set_contents($ceph_bootstrap_osd_keyring, $bindata);
 245         };
 246
 247         my $worker = sub {
 248             my $upid = shift;
 249
 250             my $fstype = $param->{fstype} || 'xfs';
 251
 252
 253             my $ccname = PVE::CephTools::get_config('ccname');
 254
 255             my $cmd = ['ceph-disk', 'prepare', '--zap-disk',
 256                        '--cluster', $ccname, '--cluster-uuid', $fsid ];
 257
 258             if ($param->{bluestore}) {
 259                 print "create OSD on $devpath (bluestore)\n";
 260                 push @$cmd, '--bluestore';
 261             } else {
 262                 print "create OSD on $devpath ($fstype)\n";
 263                 push @$cmd, '--filestore', '--fs-type', $fstype;
 264             }
 265
 266             if ($journal_dev) {
 267                 print "using device '$journal_dev' for journal\n";
 268                 push @$cmd, '--journal-dev', $devpath, $journal_dev;
 269             } else {
 270                 push @$cmd, $devpath;
 271             }
 272
 273             run_command($cmd);
 274         };
 275
 276         return $rpcenv->fork_worker('cephcreateosd', $devname,  $authuser, $worker);
 277     }});
 278
 279 __PACKAGE__->register_method ({
 280     name => 'destroyosd',
 281     path => '{osdid}',
 282     method => 'DELETE',
 283     description => "Destroy OSD",
 284     proxyto => 'node',
 285     protected => 1,
 286     parameters => {
 287         additionalProperties => 0,
 288         properties => {
 289             node => get_standard_option('pve-node'),
 290             osdid => {
 291                 description => 'OSD ID',
 292                 type => 'integer',
 293             },
 294             cleanup => {
 295                 description => "If set, we remove partition table entries.",
 296                 type => 'boolean',
 297                 optional => 1,
 298                 default => 0,
 299             },
 300         },
 301     },
 302     returns => { type => 'string' },
 303     code => sub {
 304         my ($param) = @_;
 305
 306         my $rpcenv = PVE::RPCEnvironment::get();
 307
 308         my $authuser = $rpcenv->get_user();
 309
 310         PVE::CephTools::check_ceph_inited();
 311
 312         my $osdid = $param->{osdid};
 313
 314         my $rados = PVE::RADOS->new();
 315         my $osdstat = &$get_osd_status($rados, $osdid);
 316
 317         die "osd is in use (in == 1)\n" if $osdstat->{in};
 318         #&$run_ceph_cmd(['osd', 'out', $osdid]);
 319
 320         die "osd is still runnung (up == 1)\n" if $osdstat->{up};
 321
 322         my $osdsection = "osd.$osdid";
 323
 324         my $worker = sub {
 325             my $upid = shift;
 326
 327             # reopen with longer timeout
 328             $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
 329
 330             print "destroy OSD $osdsection\n";
 331
 332             eval { PVE::CephTools::ceph_service_cmd('stop', $osdsection); };
 333             warn $@ if $@;
 334
 335             print "Remove $osdsection from the CRUSH map\n";
 336             $rados->mon_command({ prefix => "osd crush remove", name => $osdsection, format => 'plain' });
 337
 338             print "Remove the $osdsection authentication key.\n";
 339             $rados->mon_command({ prefix => "auth del", entity => $osdsection, format => 'plain' });
 340
 341             print "Remove OSD $osdsection\n";
 342             $rados->mon_command({ prefix => "osd rm", ids => [ $osdsection ], format => 'plain' });
 343
 344             # try to unmount from standard mount point
 345             my $mountpoint = "/var/lib/ceph/osd/ceph-$osdid";
 346
 347             my $remove_partition = sub {
 348                 my ($part) = @_;
 349
 350                 return if !$part || (! -b $part );
 351                 my $partnum = PVE::Diskmanage::get_partnum($part);
 352                 my $devpath = PVE::Diskmanage::get_blockdev($part);
 353
 354                 print "remove partition $part (disk '${devpath}', partnum $partnum)\n";
 355                 eval { run_command(['/sbin/sgdisk', '-d', $partnum, "${devpath}"]); };
 356                 warn $@ if $@;
 357             };
 358
 359             my $journal_part;
 360             my $data_part;
 361
 362             if ($param->{cleanup}) {
 363                 my $jpath = "$mountpoint/journal";
 364                 $journal_part = abs_path($jpath);
 365
 366                 if (my $fd = IO::File->new("/proc/mounts", "r")) {
 367                     while (defined(my $line = <$fd>)) {
 368                         my ($dev, $path, $fstype) = split(/\s+/, $line);
 369                         next if !($dev && $path && $fstype);
 370                         next if $dev !~ m|^/dev/|;
 371                         if ($path eq $mountpoint) {
 372                             $data_part = abs_path($dev);
 373                             last;
 374                         }
 375                     }
 376                     close($fd);
 377                 }
 378             }
 379
 380             print "Unmount OSD $osdsection from  $mountpoint\n";
 381             eval { run_command(['/bin/umount', $mountpoint]); };
 382             if (my $err = $@) {
 383                 warn $err;
 384             } elsif ($param->{cleanup}) {
 385                 #be aware of the ceph udev rules which can remount.
 386                 &$remove_partition($data_part);
 387                 &$remove_partition($journal_part);
 388             }
 389         };
 390
 391         return $rpcenv->fork_worker('cephdestroyosd', $osdsection,  $authuser, $worker);
 392     }});
 393
 394 __PACKAGE__->register_method ({
 395     name => 'in',
 396     path => '{osdid}/in',
 397     method => 'POST',
 398     description => "ceph osd in",
 399     proxyto => 'node',
 400     protected => 1,
 401     permissions => {
 402         check => ['perm', '/', [ 'Sys.Modify' ]],
 403     },
 404     parameters => {
 405         additionalProperties => 0,
 406         properties => {
 407             node => get_standard_option('pve-node'),
 408             osdid => {
 409                 description => 'OSD ID',
 410                 type => 'integer',
 411             },
 412         },
 413     },
 414     returns => { type => "null" },
 415     code => sub {
 416         my ($param) = @_;
 417
 418         PVE::CephTools::check_ceph_inited();
 419
 420         my $osdid = $param->{osdid};
 421
 422         my $rados = PVE::RADOS->new();
 423
 424         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 425
 426         my $osdsection = "osd.$osdid";
 427
 428         $rados->mon_command({ prefix => "osd in", ids => [ $osdsection ], format => 'plain' });
 429
 430         return undef;
 431     }});
 432
 433 __PACKAGE__->register_method ({
 434     name => 'out',
 435     path => '{osdid}/out',
 436     method => 'POST',
 437     description => "ceph osd out",
 438     proxyto => 'node',
 439     protected => 1,
 440     permissions => {
 441         check => ['perm', '/', [ 'Sys.Modify' ]],
 442     },
 443     parameters => {
 444         additionalProperties => 0,
 445         properties => {
 446             node => get_standard_option('pve-node'),
 447             osdid => {
 448                 description => 'OSD ID',
 449                 type => 'integer',
 450             },
 451         },
 452     },
 453     returns => { type => "null" },
 454     code => sub {
 455         my ($param) = @_;
 456
 457         PVE::CephTools::check_ceph_inited();
 458
 459         my $osdid = $param->{osdid};
 460
 461         my $rados = PVE::RADOS->new();
 462
 463         my $osdstat = &$get_osd_status($rados, $osdid); # osd exists?
 464
 465         my $osdsection = "osd.$osdid";
 466
 467         $rados->mon_command({ prefix => "osd out", ids => [ $osdsection ], format => 'plain' });
 468
 469         return undef;
 470     }});
 471
 472 package PVE::API2::Ceph;
 473
 474 use strict;
 475 use warnings;
 476 use File::Basename;
 477 use File::Path;
 478 use POSIX qw (LONG_MAX);
 479 use Cwd qw(abs_path);
 480 use IO::Dir;
 481 use UUID;
 482 use Net::IP;
 483
 484 use PVE::SafeSyslog;
 485 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
 486 use PVE::Exception qw(raise raise_param_exc);
 487 use PVE::INotify;
 488 use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
 489 use PVE::AccessControl;
 490 use PVE::Storage;
 491 use PVE::RESTHandler;
 492 use PVE::RPCEnvironment;
 493 use PVE::JSONSchema qw(get_standard_option);
 494 use JSON;
 495 use PVE::RADOS;
 496 use PVE::CephTools;
 497
 498 use base qw(PVE::RESTHandler);
 499
 500 use Data::Dumper; # fixme: remove
 501
 502 my $pve_osd_default_journal_size = 1024*5;
 503
 504 __PACKAGE__->register_method ({
 505     subclass => "PVE::API2::CephOSD",
 506     path => 'osd',
 507 });
 508
 509 __PACKAGE__->register_method ({
 510     name => 'index',
 511     path => '',
 512     method => 'GET',
 513     description => "Directory index.",
 514     permissions => { user => 'all' },
 515     permissions => {
 516         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 517     },
 518     parameters => {
 519         additionalProperties => 0,
 520         properties => {
 521             node => get_standard_option('pve-node'),
 522         },
 523     },
 524     returns => {
 525         type => 'array',
 526         items => {
 527             type => "object",
 528             properties => {},
 529         },
 530         links => [ { rel => 'child', href => "{name}" } ],
 531     },
 532     code => sub {
 533         my ($param) = @_;
 534
 535         my $result = [
 536             { name => 'init' },
 537             { name => 'mon' },
 538             { name => 'osd' },
 539             { name => 'pools' },
 540             { name => 'stop' },
 541             { name => 'start' },
 542             { name => 'status' },
 543             { name => 'crush' },
 544             { name => 'config' },
 545             { name => 'log' },
 546             { name => 'disks' },
 547             { name => 'flags' },
 548         ];
 549
 550         return $result;
 551     }});
 552
 553 __PACKAGE__->register_method ({
 554     name => 'disks',
 555     path => 'disks',
 556     method => 'GET',
 557     description => "List local disks.",
 558     proxyto => 'node',
 559     protected => 1,
 560     permissions => {
 561         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 562     },
 563     parameters => {
 564         additionalProperties => 0,
 565         properties => {
 566             node => get_standard_option('pve-node'),
 567             type => {
 568                 description => "Only list specific types of disks.",
 569                 type => 'string',
 570                 enum => ['unused', 'journal_disks'],
 571                 optional => 1,
 572             },
 573         },
 574     },
 575     returns => {
 576         type => 'array',
 577         items => {
 578             type => "object",
 579             properties => {
 580                 dev => { type => 'string' },
 581                 used => { type => 'string', optional => 1 },
 582                 gpt => { type => 'boolean' },
 583                 size => { type => 'integer' },
 584                 osdid => { type => 'integer' },
 585                 vendor =>  { type => 'string', optional => 1 },
 586                 model =>  { type => 'string', optional => 1 },
 587                 serial =>  { type => 'string', optional => 1 },
 588             },
 589         },
 590         # links => [ { rel => 'child', href => "{}" } ],
 591     },
 592     code => sub {
 593         my ($param) = @_;
 594
 595         PVE::CephTools::check_ceph_inited();
 596
 597         my $disks = PVE::Diskmanage::get_disks(undef, 1);
 598
 599         my $res = [];
 600         foreach my $dev (keys %$disks) {
 601             my $d = $disks->{$dev};
 602             if ($param->{type}) {
 603                 if ($param->{type} eq 'journal_disks') {
 604                     next if $d->{osdid} >= 0;
 605                     next if !$d->{gpt};
 606                 } elsif ($param->{type} eq 'unused') {
 607                     next if $d->{used};
 608                 } else {
 609                     die "internal error"; # should not happen
 610                 }
 611             }
 612
 613             $d->{dev} = "/dev/$dev";
 614             push @$res, $d;
 615         }
 616
 617         return $res;
 618     }});
 619
 620 __PACKAGE__->register_method ({
 621     name => 'config',
 622     path => 'config',
 623     method => 'GET',
 624     permissions => {
 625         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 626     },
 627     description => "Get Ceph configuration.",
 628     parameters => {
 629         additionalProperties => 0,
 630         properties => {
 631             node => get_standard_option('pve-node'),
 632         },
 633     },
 634     returns => { type => 'string' },
 635     code => sub {
 636         my ($param) = @_;
 637
 638         PVE::CephTools::check_ceph_inited();
 639
 640         my $path = PVE::CephTools::get_config('pve_ceph_cfgpath');
 641         return PVE::Tools::file_get_contents($path);
 642
 643     }});
 644
 645 __PACKAGE__->register_method ({
 646     name => 'listmon',
 647     path => 'mon',
 648     method => 'GET',
 649     description => "Get Ceph monitor list.",
 650     proxyto => 'node',
 651     protected => 1,
 652     permissions => {
 653         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 654     },
 655     parameters => {
 656         additionalProperties => 0,
 657         properties => {
 658             node => get_standard_option('pve-node'),
 659         },
 660     },
 661     returns => {
 662         type => 'array',
 663         items => {
 664             type => "object",
 665             properties => {
 666                 name => { type => 'string' },
 667                 addr => { type => 'string' },
 668             },
 669         },
 670         links => [ { rel => 'child', href => "{name}" } ],
 671     },
 672     code => sub {
 673         my ($param) = @_;
 674
 675         PVE::CephTools::check_ceph_inited();
 676
 677         my $res = [];
 678
 679         my $cfg = PVE::CephTools::parse_ceph_config();
 680
 681         my $monhash = {};
 682         foreach my $section (keys %$cfg) {
 683             my $d = $cfg->{$section};
 684             if ($section =~ m/^mon\.(\S+)$/) {
 685                 my $monid = $1;
 686                 if ($d->{'mon addr'} && $d->{'host'}) {
 687                     $monhash->{$monid} = {
 688                         addr => $d->{'mon addr'},
 689                         host => $d->{'host'},
 690                         name => $monid,
 691                     }
 692                 }
 693             }
 694         }
 695
 696         eval {
 697             my $rados = PVE::RADOS->new();
 698             my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 699             my $mons = $monstat->{monmap}->{mons};
 700             foreach my $d (@$mons) {
 701                 next if !defined($d->{name});
 702                 $monhash->{$d->{name}}->{rank} = $d->{rank};
 703                 $monhash->{$d->{name}}->{addr} = $d->{addr};
 704                 if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
 705                     $monhash->{$d->{name}}->{quorum} = 1;
 706                 }
 707             }
 708         };
 709         warn $@ if $@;
 710
 711         return PVE::RESTHandler::hash_to_array($monhash, 'name');
 712     }});
 713
 714 __PACKAGE__->register_method ({
 715     name => 'init',
 716     path => 'init',
 717     method => 'POST',
 718     description => "Create initial ceph default configuration and setup symlinks.",
 719     proxyto => 'node',
 720     protected => 1,
 721     permissions => {
 722         check => ['perm', '/', [ 'Sys.Modify' ]],
 723     },
 724     parameters => {
 725         additionalProperties => 0,
 726         properties => {
 727             node => get_standard_option('pve-node'),
 728             network => {
 729                 description => "Use specific network for all ceph related traffic",
 730                 type => 'string', format => 'CIDR',
 731                 optional => 1,
 732                 maxLength => 128,
 733             },
 734             size => {
 735                 description => 'Targeted number of replicas per object',
 736                 type => 'integer',
 737                 default => 3,
 738                 optional => 1,
 739                 minimum => 1,
 740                 maximum => 7,
 741             },
 742             min_size => {
 743                 description => 'Minimum number of available replicas per object to allow I/O',
 744                 type => 'integer',
 745                 default => 2,
 746                 optional => 1,
 747                 minimum => 1,
 748                 maximum => 7,
 749             },
 750             pg_bits => {
 751                 description => "Placement group bits, used to specify the " .
 752                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 753                     "default pg num' does not work for default pools.",
 754                 type => 'integer',
 755                 default => 6,
 756                 optional => 1,
 757                 minimum => 6,
 758                 maximum => 14,
 759             },
 760             disable_cephx => {
 761                 description => "Disable cephx authentification.\n\n" .
 762                     "WARNING: cephx is a security feature protecting against " .
 763                     "man-in-the-middle attacks. Only consider disabling cephx ".
 764                     "if your network is private!",
 765                 type => 'boolean',
 766                 optional => 1,
 767                 default => 0,
 768             },
 769         },
 770     },
 771     returns => { type => 'null' },
 772     code => sub {
 773         my ($param) = @_;
 774
 775         PVE::CephTools::check_ceph_installed();
 776
 777         # simply load old config if it already exists
 778         my $cfg = PVE::CephTools::parse_ceph_config();
 779
 780         if (!$cfg->{global}) {
 781
 782             my $fsid;
 783             my $uuid;
 784
 785             UUID::generate($uuid);
 786             UUID::unparse($uuid, $fsid);
 787
 788             my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 789
 790             $cfg->{global} = {
 791                 'fsid' => $fsid,
 792                 'auth cluster required' => $auth,
 793                 'auth service required' => $auth,
 794                 'auth client required' => $auth,
 795                 'osd journal size' => $pve_osd_default_journal_size,
 796                 'osd pool default size' => $param->{size} // 3,
 797                 'osd pool default min size' => $param->{min_size} // 2,
 798                 'mon allow pool delete' => 'true',
 799             };
 800
 801             # this does not work for default pools
 802             #'osd pool default pg num' => $pg_num,
 803             #'osd pool default pgp num' => $pg_num,
 804         }
 805
 806         $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 807         $cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring';
 808
 809         if ($param->{pg_bits}) {
 810             $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 811             $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 812         }
 813
 814         if ($param->{network}) {
 815             $cfg->{global}->{'public network'} = $param->{network};
 816             $cfg->{global}->{'cluster network'} = $param->{network};
 817         }
 818
 819         PVE::CephTools::write_ceph_config($cfg);
 820
 821         PVE::CephTools::setup_pve_symlinks();
 822
 823         return undef;
 824     }});
 825
 826 my $find_node_ip = sub {
 827     my ($cidr) = @_;
 828
 829     my $net = Net::IP->new($cidr) || die Net::IP::Error() . "\n";
 830     my $id = $net->version == 6 ? 'address6' : 'address';
 831
 832     my $config = PVE::INotify::read_file('interfaces');
 833     my $ifaces = $config->{ifaces};
 834
 835     foreach my $iface (keys %$ifaces) {
 836         my $d = $ifaces->{$iface};
 837         next if !$d->{$id};
 838         my $a = Net::IP->new($d->{$id});
 839         next if !$a;
 840         return $d->{$id} if $net->overlaps($a);
 841     }
 842
 843     die "unable to find local address within network '$cidr'\n";
 844 };
 845
 846 my $create_mgr = sub {
 847     my ($rados, $id) = @_;
 848
 849     my $clustername = PVE::CephTools::get_config('ccname');
 850     my $mgrdir = "/var/lib/ceph/mgr/$clustername-$id";
 851     my $mgrkeyring = "$mgrdir/keyring";
 852     my $mgrname = "mgr.$id";
 853
 854     die "ceph manager directory '$mgrdir' already exists\n"
 855         if -d $mgrdir;
 856
 857     print "creating manager directory '$mgrdir'\n";
 858     mkdir $mgrdir;
 859     print "creating keys for '$mgrname'\n";
 860     my $output = $rados->mon_command({ prefix => 'auth get-or-create',
 861                                        entity => $mgrname,
 862                                        caps => [
 863                                            mon => 'allow profile mgr',
 864                                            osd => 'allow *',
 865                                            mds => 'allow *',
 866                                        ],
 867                                        format => 'plain'});
 868     PVE::Tools::file_set_contents($mgrkeyring, $output);
 869
 870     print "setting owner for directory\n";
 871     run_command(["chown", 'ceph:ceph', '-R', $mgrdir]);
 872
 873     print "enabling service 'ceph-mgr\@$id.service'\n";
 874     PVE::CephTools::ceph_service_cmd('enable', $mgrname);
 875     print "starting service 'ceph-mgr\@$id.service'\n";
 876     PVE::CephTools::ceph_service_cmd('start', $mgrname);
 877 };
 878
 879 my $destroy_mgr = sub {
 880     my ($mgrid) = @_;
 881
 882     my $clustername = PVE::CephTools::get_config('ccname');
 883     my $mgrname = "mgr.$mgrid";
 884     my $mgrdir = "/var/lib/ceph/mgr/$clustername-$mgrid";
 885
 886     die "ceph manager directory '$mgrdir' not found\n"
 887         if ! -d $mgrdir;
 888
 889     print "disabling service 'ceph-mgr\@$mgrid.service'\n";
 890     PVE::CephTools::ceph_service_cmd('disable', $mgrname);
 891     print "stopping service 'ceph-mgr\@$mgrid.service'\n";
 892     PVE::CephTools::ceph_service_cmd('stop', $mgrname);
 893
 894     print "removing manager directory '$mgrdir'\n";
 895     File::Path::remove_tree($mgrdir);
 896 };
 897
 898 __PACKAGE__->register_method ({
 899     name => 'createmon',
 900     path => 'mon',
 901     method => 'POST',
 902     description => "Create Ceph Monitor and Manager",
 903     proxyto => 'node',
 904     protected => 1,
 905     permissions => {
 906         check => ['perm', '/', [ 'Sys.Modify' ]],
 907     },
 908     parameters => {
 909         additionalProperties => 0,
 910         properties => {
 911             node => get_standard_option('pve-node'),
 912             id => {
 913                 type => 'string',
 914                 optional => 1,
 915                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
 916                 description => "The ID for the monitor, when omitted the same as the nodename",
 917             },
 918             'exclude-manager' => {
 919                 type => 'boolean',
 920                 optional => 1,
 921                 default => 0,
 922                 description => "When set, only a monitor will be created.",
 923             },
 924         },
 925     },
 926     returns => { type => 'string' },
 927     code => sub {
 928         my ($param) = @_;
 929
 930         PVE::CephTools::check_ceph_inited();
 931
 932         PVE::CephTools::setup_pve_symlinks();
 933
 934         my $rpcenv = PVE::RPCEnvironment::get();
 935
 936         my $authuser = $rpcenv->get_user();
 937
 938         my $cfg = PVE::CephTools::parse_ceph_config();
 939
 940         my $moncount = 0;
 941
 942         my $monaddrhash = {};
 943
 944         my $systemd_managed = PVE::CephTools::systemd_managed();
 945
 946         foreach my $section (keys %$cfg) {
 947             next if $section eq 'global';
 948             my $d = $cfg->{$section};
 949             if ($section =~ m/^mon\./) {
 950                 $moncount++;
 951                 if ($d->{'mon addr'}) {
 952                     $monaddrhash->{$d->{'mon addr'}} = $section;
 953                 }
 954             }
 955         }
 956
 957         my $monid = $param->{id} // $param->{node};
 958
 959         my $monsection = "mon.$monid";
 960         my $ip;
 961         if (my $pubnet = $cfg->{global}->{'public network'}) {
 962             $ip = &$find_node_ip($pubnet);
 963         } else {
 964             $ip = PVE::Cluster::remote_node_ip($param->{node});
 965         }
 966
 967         my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]:6789" : "$ip:6789";
 968         my $monname = $param->{node};
 969
 970         die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
 971         die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n"
 972             if $monaddrhash->{$monaddr};
 973
 974         my $worker = sub  {
 975             my $upid = shift;
 976
 977             my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
 978
 979             if (! -f $pve_ckeyring_path) {
 980                 run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
 981                             "--gen-key -n client.admin");
 982             }
 983
 984             my $pve_mon_key_path = PVE::CephTools::get_config('pve_mon_key_path');
 985             if (! -f $pve_mon_key_path) {
 986                 run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
 987                 run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " .
 988                             "--cap mds 'allow' " .
 989                             "--cap osd 'allow *' " .
 990                             "--cap mgr 'allow *' " .
 991                             "--cap mon 'allow *'");
 992                 run_command("cp $pve_mon_key_path.tmp /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
 993                 run_command("chown ceph:ceph /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
 994                 run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
 995                 run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
 996             }
 997
 998             my $ccname = PVE::CephTools::get_config('ccname');
 999
1000             my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
1001             -d $mondir && die "monitor filesystem '$mondir' already exist\n";
1002
1003             my $monmap = "/tmp/monmap";
1004
1005             eval {
1006                 mkdir $mondir;
1007
1008                 run_command("chown ceph:ceph $mondir") if $systemd_managed;
1009
1010                 if ($moncount > 0) {
1011                     my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1012                     my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
1013                     PVE::Tools::file_set_contents($monmap, $mapdata);
1014                 } else {
1015                     run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
1016                 }
1017
1018                 run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
1019                 run_command("chown ceph:ceph -R $mondir") if $systemd_managed;
1020             };
1021             my $err = $@;
1022             unlink $monmap;
1023             if ($err) {
1024                 File::Path::remove_tree($mondir);
1025                 die $err;
1026             }
1027
1028             $cfg->{$monsection} = {
1029                 'host' => $monname,
1030                 'mon addr' => $monaddr,
1031             };
1032
1033             PVE::CephTools::write_ceph_config($cfg);
1034
1035             my $create_keys_pid = fork();
1036             if (!defined($create_keys_pid)) {
1037                 die "Could not spawn ceph-create-keys to create bootstrap keys\n";
1038             } elsif ($create_keys_pid == 0) {
1039                 exit PVE::Tools::run_command(['ceph-create-keys', '-i', $monid]);
1040             } else {
1041                 PVE::CephTools::ceph_service_cmd('start', $monsection);
1042
1043                 if ($systemd_managed) {
1044                     #to ensure we have the correct startup order.
1045                     eval { PVE::Tools::run_command(['/bin/systemctl', 'enable', "ceph-mon\@${monid}.service"]); };
1046                     warn "Enable ceph-mon\@${monid}.service manually"if $@;
1047                 }
1048                 waitpid($create_keys_pid, 0);
1049             }
1050
1051             # create manager
1052             if (!$param->{'exclude-manager'}) {
1053                 my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1054                 $create_mgr->($rados, $monid);
1055             }
1056         };
1057
1058         return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
1059     }});
1060
1061 __PACKAGE__->register_method ({
1062     name => 'destroymon',
1063     path => 'mon/{monid}',
1064     method => 'DELETE',
1065     description => "Destroy Ceph Monitor and Manager.",
1066     proxyto => 'node',
1067     protected => 1,
1068     permissions => {
1069         check => ['perm', '/', [ 'Sys.Modify' ]],
1070     },
1071     parameters => {
1072         additionalProperties => 0,
1073         properties => {
1074             node => get_standard_option('pve-node'),
1075             monid => {
1076                 description => 'Monitor ID',
1077                 type => 'string',
1078                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1079             },
1080             'exclude-manager' => {
1081                 type => 'boolean',
1082                 default => 0,
1083                 optional => 1,
1084                 description => "When set, removes only the monitor, not the manager"
1085             }
1086         },
1087     },
1088     returns => { type => 'string' },
1089     code => sub {
1090         my ($param) = @_;
1091
1092         my $rpcenv = PVE::RPCEnvironment::get();
1093
1094         my $authuser = $rpcenv->get_user();
1095
1096         PVE::CephTools::check_ceph_inited();
1097
1098         my $cfg = PVE::CephTools::parse_ceph_config();
1099
1100         my $monid = $param->{monid};
1101         my $monsection = "mon.$monid";
1102
1103         my $rados = PVE::RADOS->new();
1104         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
1105         my $monlist = $monstat->{monmap}->{mons};
1106
1107         die "no such monitor id '$monid'\n"
1108             if !defined($cfg->{$monsection});
1109
1110         my $ccname = PVE::CephTools::get_config('ccname');
1111
1112         my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
1113         -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n";
1114
1115         die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
1116
1117         my $worker = sub {
1118             my $upid = shift;
1119
1120             # reopen with longer timeout
1121             $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1122
1123             $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });
1124
1125             eval { PVE::CephTools::ceph_service_cmd('stop', $monsection); };
1126             warn $@ if $@;
1127
1128             delete $cfg->{$monsection};
1129             PVE::CephTools::write_ceph_config($cfg);
1130             File::Path::remove_tree($mondir);
1131
1132             # remove manager
1133             if (!$param->{'exclude-manager'}) {
1134                 eval { $destroy_mgr->($monid); };
1135                 warn $@ if $@;
1136             }
1137         };
1138
1139         return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
1140     }});
1141
1142 __PACKAGE__->register_method ({
1143     name => 'createmgr',
1144     path => 'mgr',
1145     method => 'POST',
1146     description => "Create Ceph Manager",
1147     proxyto => 'node',
1148     protected => 1,
1149     permissions => {
1150         check => ['perm', '/', [ 'Sys.Modify' ]],
1151     },
1152     parameters => {
1153         additionalProperties => 0,
1154         properties => {
1155             node => get_standard_option('pve-node'),
1156             id => {
1157                 type => 'string',
1158                 optional => 1,
1159                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1160                 description => "The ID for the manager, when omitted the same as the nodename",
1161             },
1162         },
1163     },
1164     returns => { type => 'string' },
1165     code => sub {
1166         my ($param) = @_;
1167
1168         PVE::CephTools::check_ceph_inited();
1169
1170         my $rpcenv = PVE::RPCEnvironment::get();
1171
1172         my $authuser = $rpcenv->get_user();
1173
1174         my $mgrid = $param->{id} // $param->{node};
1175
1176         my $worker = sub  {
1177             my $upid = shift;
1178
1179             my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
1180
1181             $create_mgr->($rados, $mgrid);
1182         };
1183
1184         return $rpcenv->fork_worker('cephcreatemgr', "mgr.$mgrid", $authuser, $worker);
1185     }});
1186
1187 __PACKAGE__->register_method ({
1188     name => 'destroymgr',
1189     path => 'mgr/{id}',
1190     method => 'DELETE',
1191     description => "Destroy Ceph Manager.",
1192     proxyto => 'node',
1193     protected => 1,
1194     permissions => {
1195         check => ['perm', '/', [ 'Sys.Modify' ]],
1196     },
1197     parameters => {
1198         additionalProperties => 0,
1199         properties => {
1200             node => get_standard_option('pve-node'),
1201             id => {
1202                 description => 'The ID of the manager',
1203                 type => 'string',
1204                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
1205             },
1206         },
1207     },
1208     returns => { type => 'string' },
1209     code => sub {
1210         my ($param) = @_;
1211
1212         my $rpcenv = PVE::RPCEnvironment::get();
1213
1214         my $authuser = $rpcenv->get_user();
1215
1216         PVE::CephTools::check_ceph_inited();
1217
1218         my $mgrid = $param->{id};
1219
1220         my $worker = sub {
1221             my $upid = shift;
1222
1223             $destroy_mgr->($mgrid);
1224         };
1225
1226         return $rpcenv->fork_worker('cephdestroymgr', "mgr.$mgrid",  $authuser, $worker);
1227     }});
1228
1229 __PACKAGE__->register_method ({
1230     name => 'stop',
1231     path => 'stop',
1232     method => 'POST',
1233     description => "Stop ceph services.",
1234     proxyto => 'node',
1235     protected => 1,
1236     permissions => {
1237         check => ['perm', '/', [ 'Sys.Modify' ]],
1238     },
1239     parameters => {
1240         additionalProperties => 0,
1241         properties => {
1242             node => get_standard_option('pve-node'),
1243             service => {
1244                 description => 'Ceph service name.',
1245                 type => 'string',
1246                 optional => 1,
1247                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1248             },
1249         },
1250     },
1251     returns => { type => 'string' },
1252     code => sub {
1253         my ($param) = @_;
1254
1255         my $rpcenv = PVE::RPCEnvironment::get();
1256
1257         my $authuser = $rpcenv->get_user();
1258
1259         PVE::CephTools::check_ceph_inited();
1260
1261         my $cfg = PVE::CephTools::parse_ceph_config();
1262         scalar(keys %$cfg) || die "no configuration\n";
1263
1264         my $worker = sub {
1265             my $upid = shift;
1266
1267             my $cmd = ['stop'];
1268             if ($param->{service}) {
1269                 push @$cmd, $param->{service};
1270             }
1271
1272             PVE::CephTools::ceph_service_cmd(@$cmd);
1273         };
1274
1275         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
1276                                     $authuser, $worker);
1277     }});
1278
1279 __PACKAGE__->register_method ({
1280     name => 'start',
1281     path => 'start',
1282     method => 'POST',
1283     description => "Start ceph services.",
1284     proxyto => 'node',
1285     protected => 1,
1286     permissions => {
1287         check => ['perm', '/', [ 'Sys.Modify' ]],
1288     },
1289     parameters => {
1290         additionalProperties => 0,
1291         properties => {
1292             node => get_standard_option('pve-node'),
1293             service => {
1294                 description => 'Ceph service name.',
1295                 type => 'string',
1296                 optional => 1,
1297                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
1298             },
1299         },
1300     },
1301     returns => { type => 'string' },
1302     code => sub {
1303         my ($param) = @_;
1304
1305         my $rpcenv = PVE::RPCEnvironment::get();
1306
1307         my $authuser = $rpcenv->get_user();
1308
1309         PVE::CephTools::check_ceph_inited();
1310
1311         my $cfg = PVE::CephTools::parse_ceph_config();
1312         scalar(keys %$cfg) || die "no configuration\n";
1313
1314         my $worker = sub {
1315             my $upid = shift;
1316
1317             my $cmd = ['start'];
1318             if ($param->{service}) {
1319                 push @$cmd, $param->{service};
1320             }
1321
1322             PVE::CephTools::ceph_service_cmd(@$cmd);
1323         };
1324
1325         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
1326                                     $authuser, $worker);
1327     }});
1328
1329 __PACKAGE__->register_method ({
1330     name => 'status',
1331     path => 'status',
1332     method => 'GET',
1333     description => "Get ceph status.",
1334     proxyto => 'node',
1335     protected => 1,
1336     permissions => {
1337         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1338     },
1339     parameters => {
1340         additionalProperties => 0,
1341         properties => {
1342             node => get_standard_option('pve-node'),
1343         },
1344     },
1345     returns => { type => 'object' },
1346     code => sub {
1347         my ($param) = @_;
1348
1349         PVE::CephTools::check_ceph_enabled();
1350
1351         my $rados = PVE::RADOS->new();
1352         my $status = $rados->mon_command({ prefix => 'status' });
1353         $status->{health} = $rados->mon_command({ prefix => 'health', detail => 'detail' });
1354         return $status;
1355     }});
1356
1357 __PACKAGE__->register_method ({
1358     name => 'lspools',
1359     path => 'pools',
1360     method => 'GET',
1361     description => "List all pools.",
1362     proxyto => 'node',
1363     protected => 1,
1364     permissions => {
1365         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1366     },
1367     parameters => {
1368         additionalProperties => 0,
1369         properties => {
1370             node => get_standard_option('pve-node'),
1371         },
1372     },
1373     returns => {
1374         type => 'array',
1375         items => {
1376             type => "object",
1377             properties => {
1378                 pool => { type => 'integer' },
1379                 pool_name => { type => 'string' },
1380                 size => { type => 'integer' },
1381             },
1382         },
1383         links => [ { rel => 'child', href => "{pool_name}" } ],
1384     },
1385     code => sub {
1386         my ($param) = @_;
1387
1388         PVE::CephTools::check_ceph_inited();
1389
1390         my $rados = PVE::RADOS->new();
1391
1392         my $stats = {};
1393         my $res = $rados->mon_command({ prefix => 'df' });
1394         my $total = $res->{stats}->{total_avail_bytes} || 0;
1395
1396         foreach my $d (@{$res->{pools}}) {
1397             next if !$d->{stats};
1398             next if !defined($d->{id});
1399             $stats->{$d->{id}} = $d->{stats};
1400         }
1401
1402         $res = $rados->mon_command({ prefix => 'osd dump' });
1403
1404         my $data = [];
1405         foreach my $e (@{$res->{pools}}) {
1406             my $d = {};
1407             foreach my $attr (qw(pool pool_name size min_size pg_num crush_ruleset)) {
1408                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
1409             }
1410             if (my $s = $stats->{$d->{pool}}) {
1411                 $d->{bytes_used} = $s->{bytes_used};
1412                 $d->{percent_used} = ($s->{bytes_used} / $total)*100
1413                     if $s->{max_avail} && $total;
1414             }
1415             push @$data, $d;
1416         }
1417
1418
1419         return $data;
1420     }});
1421
1422 __PACKAGE__->register_method ({
1423     name => 'createpool',
1424     path => 'pools',
1425     method => 'POST',
1426     description => "Create POOL",
1427     proxyto => 'node',
1428     protected => 1,
1429     permissions => {
1430         check => ['perm', '/', [ 'Sys.Modify' ]],
1431     },
1432     parameters => {
1433         additionalProperties => 0,
1434         properties => {
1435             node => get_standard_option('pve-node'),
1436             name => {
1437                 description => "The name of the pool. It must be unique.",
1438                 type => 'string',
1439             },
1440             size => {
1441                 description => 'Number of replicas per object',
1442                 type => 'integer',
1443                 default => 2,
1444                 optional => 1,
1445                 minimum => 1,
1446                 maximum => 7,
1447             },
1448             min_size => {
1449                 description => 'Minimum number of replicas per object',
1450                 type => 'integer',
1451                 default => 1,
1452                 optional => 1,
1453                 minimum => 1,
1454                 maximum => 7,
1455             },
1456             pg_num => {
1457                 description => "Number of placement groups.",
1458                 type => 'integer',
1459                 default => 64,
1460                 optional => 1,
1461                 minimum => 8,
1462                 maximum => 32768,
1463             },
1464             crush_ruleset => {
1465                 description => "The ruleset to use for mapping object placement in the cluster.",
1466                 type => 'integer',
1467                 minimum => 0,
1468                 maximum => 32768,
1469                 default => 0,
1470                 optional => 1,
1471             },
1472         },
1473     },
1474     returns => { type => 'null' },
1475     code => sub {
1476         my ($param) = @_;
1477
1478         PVE::CephTools::check_ceph_inited();
1479
1480         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1481
1482         die "not fully configured - missing '$pve_ckeyring_path'\n"
1483             if ! -f $pve_ckeyring_path;
1484
1485         my $pg_num = $param->{pg_num} || 64;
1486         my $size = $param->{size} || 2;
1487         my $min_size = $param->{min_size} || 1;
1488         my $ruleset = $param->{crush_ruleset} || 0;
1489         my $rados = PVE::RADOS->new();
1490
1491         $rados->mon_command({
1492             prefix => "osd pool create",
1493             pool => $param->{name},
1494             pg_num => int($pg_num),
1495 # this does not work for unknown reason
1496 #           properties => ["size=$size", "min_size=$min_size", "crush_ruleset=$ruleset"],
1497             format => 'plain',
1498         });
1499
1500         $rados->mon_command({
1501             prefix => "osd pool set",
1502             pool => $param->{name},
1503             var => 'min_size',
1504             val => $min_size,
1505             format => 'plain',
1506         });
1507
1508         $rados->mon_command({
1509             prefix => "osd pool set",
1510             pool => $param->{name},
1511             var => 'size',
1512             val => $size,
1513             format => 'plain',
1514         });
1515
1516         if (defined($param->{crush_ruleset})) {
1517             $rados->mon_command({
1518                 prefix => "osd pool set",
1519                 pool => $param->{name},
1520                 var => 'crush_ruleset',
1521                 val => $param->{crush_ruleset},
1522                 format => 'plain',
1523             });
1524         }
1525
1526         return undef;
1527     }});
1528
1529 __PACKAGE__->register_method ({
1530     name => 'get_flags',
1531     path => 'flags',
1532     method => 'GET',
1533     description => "get all set ceph flags",
1534     proxyto => 'node',
1535     protected => 1,
1536     permissions => {
1537         check => ['perm', '/', [ 'Sys.Audit' ]],
1538     },
1539     parameters => {
1540         additionalProperties => 0,
1541         properties => {
1542             node => get_standard_option('pve-node'),
1543         },
1544     },
1545     returns => { type => 'string' },
1546     code => sub {
1547         my ($param) = @_;
1548
1549         PVE::CephTools::check_ceph_inited();
1550
1551         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1552
1553         die "not fully configured - missing '$pve_ckeyring_path'\n"
1554             if ! -f $pve_ckeyring_path;
1555
1556         my $rados = PVE::RADOS->new();
1557
1558         my $stat = $rados->mon_command({ prefix => 'osd dump' });
1559
1560         return $stat->{flags} // '';
1561     }});
1562
1563 __PACKAGE__->register_method ({
1564     name => 'set_flag',
1565     path => 'flags/{flag}',
1566     method => 'POST',
1567     description => "Set a ceph flag",
1568     proxyto => 'node',
1569     protected => 1,
1570     permissions => {
1571         check => ['perm', '/', [ 'Sys.Modify' ]],
1572     },
1573     parameters => {
1574         additionalProperties => 0,
1575         properties => {
1576             node => get_standard_option('pve-node'),
1577             flag => {
1578                 description => 'The ceph flag to set/unset',
1579                 type => 'string',
1580                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1581             },
1582         },
1583     },
1584     returns => { type => 'null' },
1585     code => sub {
1586         my ($param) = @_;
1587
1588         PVE::CephTools::check_ceph_inited();
1589
1590         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1591
1592         die "not fully configured - missing '$pve_ckeyring_path'\n"
1593             if ! -f $pve_ckeyring_path;
1594
1595         my $set = $param->{set} // !$param->{unset};
1596         my $rados = PVE::RADOS->new();
1597
1598         $rados->mon_command({
1599             prefix => "osd set",
1600             key => $param->{flag},
1601         });
1602
1603         return undef;
1604     }});
1605
1606 __PACKAGE__->register_method ({
1607     name => 'unset_flag',
1608     path => 'flags/{flag}',
1609     method => 'DELETE',
1610     description => "Unset a ceph flag",
1611     proxyto => 'node',
1612     protected => 1,
1613     permissions => {
1614         check => ['perm', '/', [ 'Sys.Modify' ]],
1615     },
1616     parameters => {
1617         additionalProperties => 0,
1618         properties => {
1619             node => get_standard_option('pve-node'),
1620             flag => {
1621                 description => 'The ceph flag to set/unset',
1622                 type => 'string',
1623                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1624             },
1625         },
1626     },
1627     returns => { type => 'null' },
1628     code => sub {
1629         my ($param) = @_;
1630
1631         PVE::CephTools::check_ceph_inited();
1632
1633         my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
1634
1635         die "not fully configured - missing '$pve_ckeyring_path'\n"
1636             if ! -f $pve_ckeyring_path;
1637
1638         my $set = $param->{set} // !$param->{unset};
1639         my $rados = PVE::RADOS->new();
1640
1641         $rados->mon_command({
1642             prefix => "osd unset",
1643             key => $param->{flag},
1644         });
1645
1646         return undef;
1647     }});
1648
1649 __PACKAGE__->register_method ({
1650     name => 'destroypool',
1651     path => 'pools/{name}',
1652     method => 'DELETE',
1653     description => "Destroy pool",
1654     proxyto => 'node',
1655     protected => 1,
1656     permissions => {
1657         check => ['perm', '/', [ 'Sys.Modify' ]],
1658     },
1659     parameters => {
1660         additionalProperties => 0,
1661         properties => {
1662             node => get_standard_option('pve-node'),
1663             name => {
1664                 description => "The name of the pool. It must be unique.",
1665                 type => 'string',
1666             },
1667             force => {
1668                 description => "If true, destroys pool even if in use",
1669                 type => 'boolean',
1670                 optional => 1,
1671                 default => 0,
1672             }
1673         },
1674     },
1675     returns => { type => 'null' },
1676     code => sub {
1677         my ($param) = @_;
1678
1679         PVE::CephTools::check_ceph_inited();
1680
1681         # if not forced, destroy ceph pool only when no
1682         # vm disks are on it anymore
1683         if (!$param->{force}) {
1684             my $storagecfg = PVE::Storage::config();
1685             foreach my $storageid (keys %{$storagecfg->{ids}}) {
1686                 my $storage = $storagecfg->{ids}->{$storageid};
1687                 next if $storage->{type} ne 'rbd';
1688                 next if $storage->{pool} ne $param->{name};
1689
1690                 # check if any vm disks are on the pool
1691                 my $res = PVE::Storage::vdisk_list($storagecfg, $storageid);
1692                 die "ceph pool '$param->{name}' still in use by storage '$storageid'\n"
1693                     if @{$res->{$storageid}} != 0;
1694             }
1695         }
1696
1697         my $rados = PVE::RADOS->new();
1698         # fixme: '--yes-i-really-really-mean-it'
1699         $rados->mon_command({
1700             prefix => "osd pool delete",
1701             pool => $param->{name},
1702             pool2 => $param->{name},
1703             sure => '--yes-i-really-really-mean-it',
1704             format => 'plain',
1705         });
1706
1707         return undef;
1708     }});
1709
1710
1711 __PACKAGE__->register_method ({
1712     name => 'crush',
1713     path => 'crush',
1714     method => 'GET',
1715     description => "Get OSD crush map",
1716     proxyto => 'node',
1717     protected => 1,
1718     permissions => {
1719         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1720     },
1721     parameters => {
1722         additionalProperties => 0,
1723         properties => {
1724             node => get_standard_option('pve-node'),
1725         },
1726     },
1727     returns => { type => 'string' },
1728     code => sub {
1729         my ($param) = @_;
1730
1731         PVE::CephTools::check_ceph_inited();
1732
1733         # this produces JSON (difficult to read for the user)
1734         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
1735
1736         my $txt = '';
1737
1738         my $mapfile = "/var/tmp/ceph-crush.map.$$";
1739         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
1740
1741         my $rados = PVE::RADOS->new();
1742
1743         eval {
1744             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
1745             PVE::Tools::file_set_contents($mapfile, $bindata);
1746             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
1747             $txt = PVE::Tools::file_get_contents($mapdata);
1748         };
1749         my $err = $@;
1750
1751         unlink $mapfile;
1752         unlink $mapdata;
1753
1754         die $err if $err;
1755
1756         return $txt;
1757     }});
1758
1759 __PACKAGE__->register_method({
1760     name => 'log',
1761     path => 'log',
1762     method => 'GET',
1763     description => "Read ceph log",
1764     proxyto => 'node',
1765     permissions => {
1766         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
1767     },
1768     protected => 1,
1769     parameters => {
1770         additionalProperties => 0,
1771         properties => {
1772             node => get_standard_option('pve-node'),
1773             start => {
1774                 type => 'integer',
1775                 minimum => 0,
1776                 optional => 1,
1777             },
1778             limit => {
1779                 type => 'integer',
1780                 minimum => 0,
1781                 optional => 1,
1782             },
1783         },
1784     },
1785     returns => {
1786         type => 'array',
1787         items => {
1788             type => "object",
1789             properties => {
1790                 n => {
1791                   description=>  "Line number",
1792                   type=> 'integer',
1793                 },
1794                 t => {
1795                   description=>  "Line text",
1796                   type => 'string',
1797                 }
1798             }
1799         }
1800     },
1801     code => sub {
1802         my ($param) = @_;
1803
1804         my $rpcenv = PVE::RPCEnvironment::get();
1805         my $user = $rpcenv->get_user();
1806         my $node = $param->{node};
1807
1808         my $logfile = "/var/log/ceph/ceph.log";
1809         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
1810
1811         $rpcenv->set_result_attrib('total', $count);
1812
1813         return $lines;
1814     }});
1815
1816