PVE/API2/Ceph.pm

   1 package PVE::API2::Ceph;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use Net::IP;
   8 use UUID;
   9
  10 use PVE::Ceph::Tools;
  11 use PVE::Ceph::Services;
  12 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  13 use PVE::JSONSchema qw(get_standard_option);
  14 use PVE::Network;
  15 use PVE::RADOS;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::Storage;
  19 use PVE::Tools qw(run_command file_get_contents file_set_contents);
  20
  21 use PVE::API2::Ceph::OSD;
  22 use PVE::API2::Ceph::FS;
  23 use PVE::API2::Ceph::MDS;
  24 use PVE::API2::Storage::Config;
  25
  26 use base qw(PVE::RESTHandler);
  27
  28 my $pve_osd_default_journal_size = 1024*5;
  29
  30 __PACKAGE__->register_method ({
  31     subclass => "PVE::API2::Ceph::OSD",
  32     path => 'osd',
  33 });
  34
  35 __PACKAGE__->register_method ({
  36     subclass => "PVE::API2::Ceph::MDS",
  37     path => 'mds',
  38 });
  39
  40 __PACKAGE__->register_method ({
  41     subclass => "PVE::API2::Ceph::FS",
  42     path => 'fs',
  43 });
  44
  45 __PACKAGE__->register_method ({
  46     name => 'index',
  47     path => '',
  48     method => 'GET',
  49     description => "Directory index.",
  50     permissions => { user => 'all' },
  51     permissions => {
  52         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  53     },
  54     parameters => {
  55         additionalProperties => 0,
  56         properties => {
  57             node => get_standard_option('pve-node'),
  58         },
  59     },
  60     returns => {
  61         type => 'array',
  62         items => {
  63             type => "object",
  64             properties => {},
  65         },
  66         links => [ { rel => 'child', href => "{name}" } ],
  67     },
  68     code => sub {
  69         my ($param) = @_;
  70
  71         my $result = [
  72             { name => 'init' },
  73             { name => 'mon' },
  74             { name => 'osd' },
  75             { name => 'pools' },
  76             { name => 'fs' },
  77             { name => 'mds' },
  78             { name => 'stop' },
  79             { name => 'start' },
  80             { name => 'status' },
  81             { name => 'crush' },
  82             { name => 'config' },
  83             { name => 'log' },
  84             { name => 'disks' },
  85             { name => 'flags' },
  86             { name => 'rules' },
  87         ];
  88
  89         return $result;
  90     }});
  91
  92 __PACKAGE__->register_method ({
  93     name => 'disks',
  94     path => 'disks',
  95     method => 'GET',
  96     description => "List local disks.",
  97     proxyto => 'node',
  98     protected => 1,
  99     permissions => {
 100         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 101     },
 102     parameters => {
 103         additionalProperties => 0,
 104         properties => {
 105             node => get_standard_option('pve-node'),
 106             type => {
 107                 description => "Only list specific types of disks.",
 108                 type => 'string',
 109                 enum => ['unused', 'journal_disks'],
 110                 optional => 1,
 111             },
 112         },
 113     },
 114     returns => {
 115         type => 'array',
 116         items => {
 117             type => "object",
 118             properties => {
 119                 dev => { type => 'string' },
 120                 used => { type => 'string', optional => 1 },
 121                 gpt => { type => 'boolean' },
 122                 size => { type => 'integer' },
 123                 osdid => { type => 'integer' },
 124                 vendor =>  { type => 'string', optional => 1 },
 125                 model =>  { type => 'string', optional => 1 },
 126                 serial =>  { type => 'string', optional => 1 },
 127             },
 128         },
 129         # links => [ { rel => 'child', href => "{}" } ],
 130     },
 131     code => sub {
 132         my ($param) = @_;
 133
 134         PVE::Ceph::Tools::check_ceph_inited();
 135
 136         my $disks = PVE::Diskmanage::get_disks(undef, 1);
 137
 138         my $res = [];
 139         foreach my $dev (keys %$disks) {
 140             my $d = $disks->{$dev};
 141             if ($param->{type}) {
 142                 if ($param->{type} eq 'journal_disks') {
 143                     next if $d->{osdid} >= 0;
 144                     next if !$d->{gpt};
 145                 } elsif ($param->{type} eq 'unused') {
 146                     next if $d->{used};
 147                 } else {
 148                     die "internal error"; # should not happen
 149                 }
 150             }
 151
 152             $d->{dev} = "/dev/$dev";
 153             push @$res, $d;
 154         }
 155
 156         return $res;
 157     }});
 158
 159 __PACKAGE__->register_method ({
 160     name => 'config',
 161     path => 'config',
 162     method => 'GET',
 163     permissions => {
 164         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 165     },
 166     description => "Get Ceph configuration.",
 167     parameters => {
 168         additionalProperties => 0,
 169         properties => {
 170             node => get_standard_option('pve-node'),
 171         },
 172     },
 173     returns => { type => 'string' },
 174     code => sub {
 175         my ($param) = @_;
 176
 177         PVE::Ceph::Tools::check_ceph_inited();
 178
 179         my $path = PVE::Ceph::Tools::get_config('pve_ceph_cfgpath');
 180         return file_get_contents($path);
 181
 182     }});
 183
 184 my $add_storage = sub {
 185     my ($pool, $storeid) = @_;
 186
 187     my $storage_params = {
 188         type => 'rbd',
 189         pool => $pool,
 190         storage => $storeid,
 191         krbd => 0,
 192         content => 'rootdir,images',
 193     };
 194
 195     PVE::API2::Storage::Config->create($storage_params);
 196 };
 197
 198 my $get_storages = sub {
 199     my ($pool) = @_;
 200
 201     my $cfg = PVE::Storage::config();
 202
 203     my $storages = $cfg->{ids};
 204     my $res = {};
 205     foreach my $storeid (keys %$storages) {
 206         my $curr = $storages->{$storeid};
 207         $res->{$storeid} = $storages->{$storeid}
 208             if $curr->{type} eq 'rbd' && $pool eq $curr->{pool};
 209     }
 210
 211     return $res;
 212 };
 213
 214 __PACKAGE__->register_method ({
 215     name => 'listmon',
 216     path => 'mon',
 217     method => 'GET',
 218     description => "Get Ceph monitor list.",
 219     proxyto => 'node',
 220     protected => 1,
 221     permissions => {
 222         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 223     },
 224     parameters => {
 225         additionalProperties => 0,
 226         properties => {
 227             node => get_standard_option('pve-node'),
 228         },
 229     },
 230     returns => {
 231         type => 'array',
 232         items => {
 233             type => "object",
 234             properties => {
 235                 name => { type => 'string' },
 236                 addr => { type => 'string' },
 237             },
 238         },
 239         links => [ { rel => 'child', href => "{name}" } ],
 240     },
 241     code => sub {
 242         my ($param) = @_;
 243
 244         PVE::Ceph::Tools::check_ceph_inited();
 245
 246         my $res = [];
 247
 248         my $cfg = cfs_read_file('ceph.conf');
 249
 250         my $monhash = {};
 251         foreach my $section (keys %$cfg) {
 252             my $d = $cfg->{$section};
 253             if ($section =~ m/^mon\.(\S+)$/) {
 254                 my $monid = $1;
 255                 if ($d->{'mon addr'} && $d->{'host'}) {
 256                     $monhash->{$monid} = {
 257                         addr => $d->{'mon addr'},
 258                         host => $d->{'host'},
 259                         name => $monid,
 260                     }
 261                 }
 262             }
 263         }
 264
 265         eval {
 266             my $rados = PVE::RADOS->new();
 267             my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 268             my $mons = $monstat->{monmap}->{mons};
 269             foreach my $d (@$mons) {
 270                 next if !defined($d->{name});
 271                 $monhash->{$d->{name}}->{rank} = $d->{rank};
 272                 $monhash->{$d->{name}}->{addr} = $d->{addr};
 273                 if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
 274                     $monhash->{$d->{name}}->{quorum} = 1;
 275                 }
 276             }
 277         };
 278         warn $@ if $@;
 279
 280         return PVE::RESTHandler::hash_to_array($monhash, 'name');
 281     }});
 282
 283 __PACKAGE__->register_method ({
 284     name => 'init',
 285     path => 'init',
 286     method => 'POST',
 287     description => "Create initial ceph default configuration and setup symlinks.",
 288     proxyto => 'node',
 289     protected => 1,
 290     permissions => {
 291         check => ['perm', '/', [ 'Sys.Modify' ]],
 292     },
 293     parameters => {
 294         additionalProperties => 0,
 295         properties => {
 296             node => get_standard_option('pve-node'),
 297             network => {
 298                 description => "Use specific network for all ceph related traffic",
 299                 type => 'string', format => 'CIDR',
 300                 optional => 1,
 301                 maxLength => 128,
 302             },
 303             'cluster-network' => {
 304                 description => "Declare a separate cluster network, OSDs will route" .
 305                     "heartbeat, object replication and recovery traffic over it",
 306                 type => 'string', format => 'CIDR',
 307                 requires => 'network',
 308                 optional => 1,
 309                 maxLength => 128,
 310             },
 311             size => {
 312                 description => 'Targeted number of replicas per object',
 313                 type => 'integer',
 314                 default => 3,
 315                 optional => 1,
 316                 minimum => 1,
 317                 maximum => 7,
 318             },
 319             min_size => {
 320                 description => 'Minimum number of available replicas per object to allow I/O',
 321                 type => 'integer',
 322                 default => 2,
 323                 optional => 1,
 324                 minimum => 1,
 325                 maximum => 7,
 326             },
 327             pg_bits => {
 328                 description => "Placement group bits, used to specify the " .
 329                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 330                     "default pg num' does not work for default pools.",
 331                 type => 'integer',
 332                 default => 6,
 333                 optional => 1,
 334                 minimum => 6,
 335                 maximum => 14,
 336             },
 337             disable_cephx => {
 338                 description => "Disable cephx authentification.\n\n" .
 339                     "WARNING: cephx is a security feature protecting against " .
 340                     "man-in-the-middle attacks. Only consider disabling cephx ".
 341                     "if your network is private!",
 342                 type => 'boolean',
 343                 optional => 1,
 344                 default => 0,
 345             },
 346         },
 347     },
 348     returns => { type => 'null' },
 349     code => sub {
 350         my ($param) = @_;
 351
 352         my $version = PVE::Ceph::Tools::get_local_version(1);
 353
 354         if (!$version || $version < 12) {
 355             die "Ceph Luminous required - please run 'pveceph install'\n";
 356         } else {
 357             PVE::Ceph::Tools::check_ceph_installed('ceph_bin');
 358         }
 359
 360         # simply load old config if it already exists
 361         my $cfg = cfs_read_file('ceph.conf');
 362
 363         if (!$cfg->{global}) {
 364
 365             my $fsid;
 366             my $uuid;
 367
 368             UUID::generate($uuid);
 369             UUID::unparse($uuid, $fsid);
 370
 371             my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 372
 373             $cfg->{global} = {
 374                 'fsid' => $fsid,
 375                 'auth cluster required' => $auth,
 376                 'auth service required' => $auth,
 377                 'auth client required' => $auth,
 378                 'osd journal size' => $pve_osd_default_journal_size,
 379                 'osd pool default size' => $param->{size} // 3,
 380                 'osd pool default min size' => $param->{min_size} // 2,
 381                 'mon allow pool delete' => 'true',
 382             };
 383
 384             # this does not work for default pools
 385             #'osd pool default pg num' => $pg_num,
 386             #'osd pool default pgp num' => $pg_num,
 387         }
 388
 389         $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 390         $cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring';
 391
 392         if ($param->{pg_bits}) {
 393             $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 394             $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 395         }
 396
 397         if ($param->{network}) {
 398             $cfg->{global}->{'public network'} = $param->{network};
 399             $cfg->{global}->{'cluster network'} = $param->{network};
 400         }
 401
 402         if ($param->{'cluster-network'}) {
 403             $cfg->{global}->{'cluster network'} = $param->{'cluster-network'};
 404         }
 405
 406         cfs_write_file('ceph.conf', $cfg);
 407
 408         PVE::Ceph::Tools::setup_pve_symlinks();
 409
 410         return undef;
 411     }});
 412
 413 my $find_mon_ip = sub {
 414     my ($pubnet, $node, $overwrite_ip) = @_;
 415
 416     if (!$pubnet) {
 417         return $overwrite_ip // PVE::Cluster::remote_node_ip($node);
 418     }
 419
 420     my $allowed_ips = PVE::Network::get_local_ip_from_cidr($pubnet);
 421     die "No IP configured and up from ceph public network '$pubnet'\n"
 422         if scalar(@$allowed_ips) < 1;
 423
 424     if (!$overwrite_ip) {
 425         if (scalar(@$allowed_ips) == 1) {
 426             return $allowed_ips->[0];
 427         }
 428         die "Multiple IPs for ceph public network '$pubnet' detected on $node:\n".
 429             join("\n", @$allowed_ips) ."\nuse 'mon-address' to specify one of them.\n";
 430     } else {
 431         if (grep { $_ eq $overwrite_ip } @$allowed_ips) {
 432             return $overwrite_ip;
 433         }
 434         die "Monitor IP '$overwrite_ip' not in ceph public network '$pubnet'\n"
 435             if !PVE::Network::is_ip_in_cidr($overwrite_ip, $pubnet);
 436
 437         die "Specified monitor IP '$overwrite_ip' not configured or up on $node!\n";
 438     }
 439 };
 440
 441 __PACKAGE__->register_method ({
 442     name => 'createmon',
 443     path => 'mon',
 444     method => 'POST',
 445     description => "Create Ceph Monitor and Manager",
 446     proxyto => 'node',
 447     protected => 1,
 448     permissions => {
 449         check => ['perm', '/', [ 'Sys.Modify' ]],
 450     },
 451     parameters => {
 452         additionalProperties => 0,
 453         properties => {
 454             node => get_standard_option('pve-node'),
 455             id => {
 456                 type => 'string',
 457                 optional => 1,
 458                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
 459                 description => "The ID for the monitor, when omitted the same as the nodename",
 460             },
 461             'exclude-manager' => {
 462                 type => 'boolean',
 463                 optional => 1,
 464                 default => 0,
 465                 description => "When set, only a monitor will be created.",
 466             },
 467             'mon-address' => {
 468                 description => 'Overwrites autodetected monitor IP address. ' .
 469                                'Must be in the public network of ceph.',
 470                 type => 'string', format => 'ip',
 471                 optional => 1,
 472             },
 473         },
 474     },
 475     returns => { type => 'string' },
 476     code => sub {
 477         my ($param) = @_;
 478
 479         PVE::Ceph::Tools::check_ceph_installed('ceph_mon');
 480
 481         PVE::Ceph::Tools::check_ceph_installed('ceph_mgr')
 482             if (!$param->{'exclude-manager'});
 483
 484         PVE::Ceph::Tools::check_ceph_inited();
 485
 486         PVE::Ceph::Tools::setup_pve_symlinks();
 487
 488         my $rpcenv = PVE::RPCEnvironment::get();
 489
 490         my $authuser = $rpcenv->get_user();
 491
 492         my $cfg = cfs_read_file('ceph.conf');
 493
 494         my $moncount = 0;
 495
 496         my $monaddrhash = {};
 497
 498         my $systemd_managed = PVE::Ceph::Tools::systemd_managed();
 499
 500         foreach my $section (keys %$cfg) {
 501             next if $section eq 'global';
 502             my $d = $cfg->{$section};
 503             if ($section =~ m/^mon\./) {
 504                 $moncount++;
 505                 if ($d->{'mon addr'}) {
 506                     $monaddrhash->{$d->{'mon addr'}} = $section;
 507                 }
 508             }
 509         }
 510
 511         my $monid = $param->{id} // $param->{node};
 512
 513         my $monsection = "mon.$monid";
 514         my $pubnet = $cfg->{global}->{'public network'};
 515         my $ip = $find_mon_ip->($pubnet, $param->{node}, $param->{'mon-address'});
 516
 517         my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]:6789" : "$ip:6789";
 518         my $monname = $param->{node};
 519
 520         die "monitor '$monsection' already exists\n" if $cfg->{$monsection};
 521         die "monitor address '$monaddr' already in use by '$monaddrhash->{$monaddr}'\n"
 522             if $monaddrhash->{$monaddr};
 523
 524         my $worker = sub  {
 525             my $upid = shift;
 526
 527             my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
 528
 529             if (! -f $pve_ckeyring_path) {
 530                 run_command("ceph-authtool $pve_ckeyring_path --create-keyring " .
 531                             "--gen-key -n client.admin");
 532             }
 533
 534             my $pve_mon_key_path = PVE::Ceph::Tools::get_config('pve_mon_key_path');
 535             if (! -f $pve_mon_key_path) {
 536                 run_command("cp $pve_ckeyring_path $pve_mon_key_path.tmp");
 537                 run_command("ceph-authtool $pve_mon_key_path.tmp -n client.admin --set-uid=0 " .
 538                             "--cap mds 'allow' " .
 539                             "--cap osd 'allow *' " .
 540                             "--cap mgr 'allow *' " .
 541                             "--cap mon 'allow *'");
 542                 run_command("cp $pve_mon_key_path.tmp /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
 543                 run_command("chown ceph:ceph /etc/ceph/ceph.client.admin.keyring") if $systemd_managed;
 544                 run_command("ceph-authtool $pve_mon_key_path.tmp --gen-key -n mon. --cap mon 'allow *'");
 545                 run_command("mv $pve_mon_key_path.tmp $pve_mon_key_path");
 546             }
 547
 548             my $ccname = PVE::Ceph::Tools::get_config('ccname');
 549
 550             my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 551             -d $mondir && die "monitor filesystem '$mondir' already exist\n";
 552
 553             my $monmap = "/tmp/monmap";
 554
 555             eval {
 556                 mkdir $mondir;
 557
 558                 run_command("chown ceph:ceph $mondir") if $systemd_managed;
 559
 560                 if ($moncount > 0) {
 561                     my $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
 562                     my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
 563                     file_set_contents($monmap, $mapdata);
 564                 } else {
 565                     run_command("monmaptool --create --clobber --add $monid $monaddr --print $monmap");
 566                 }
 567
 568                 run_command("ceph-mon --mkfs -i $monid --monmap $monmap --keyring $pve_mon_key_path");
 569                 run_command("chown ceph:ceph -R $mondir") if $systemd_managed;
 570             };
 571             my $err = $@;
 572             unlink $monmap;
 573             if ($err) {
 574                 File::Path::remove_tree($mondir);
 575                 die $err;
 576             }
 577
 578             $cfg->{$monsection} = {
 579                 'host' => $monname,
 580                 'mon addr' => $monaddr,
 581             };
 582
 583             cfs_write_file('ceph.conf', $cfg);
 584
 585             my $create_keys_pid = fork();
 586             if (!defined($create_keys_pid)) {
 587                 die "Could not spawn ceph-create-keys to create bootstrap keys\n";
 588             } elsif ($create_keys_pid == 0) {
 589                 exit PVE::Tools::run_command(['ceph-create-keys', '-i', $monid]);
 590             } else {
 591                 PVE::Ceph::Services::ceph_service_cmd('start', $monsection);
 592
 593                 if ($systemd_managed) {
 594                     #to ensure we have the correct startup order.
 595                     eval { PVE::Tools::run_command(['/bin/systemctl', 'enable', "ceph-mon\@${monid}.service"]); };
 596                     warn "Enable ceph-mon\@${monid}.service manually"if $@;
 597                 }
 598                 waitpid($create_keys_pid, 0);
 599             }
 600
 601             # create manager
 602             if (!$param->{'exclude-manager'}) {
 603                 my $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
 604                 PVE::Ceph::Services::create_mgr($monid, $rados);
 605             }
 606         };
 607
 608         return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
 609     }});
 610
 611 __PACKAGE__->register_method ({
 612     name => 'destroymon',
 613     path => 'mon/{monid}',
 614     method => 'DELETE',
 615     description => "Destroy Ceph Monitor and Manager.",
 616     proxyto => 'node',
 617     protected => 1,
 618     permissions => {
 619         check => ['perm', '/', [ 'Sys.Modify' ]],
 620     },
 621     parameters => {
 622         additionalProperties => 0,
 623         properties => {
 624             node => get_standard_option('pve-node'),
 625             monid => {
 626                 description => 'Monitor ID',
 627                 type => 'string',
 628                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
 629             },
 630             'exclude-manager' => {
 631                 type => 'boolean',
 632                 default => 0,
 633                 optional => 1,
 634                 description => "When set, removes only the monitor, not the manager"
 635             }
 636         },
 637     },
 638     returns => { type => 'string' },
 639     code => sub {
 640         my ($param) = @_;
 641
 642         my $rpcenv = PVE::RPCEnvironment::get();
 643
 644         my $authuser = $rpcenv->get_user();
 645
 646         PVE::Ceph::Tools::check_ceph_inited();
 647
 648         my $cfg = cfs_read_file('ceph.conf');
 649
 650         my $monid = $param->{monid};
 651         my $monsection = "mon.$monid";
 652
 653         my $rados = PVE::RADOS->new();
 654         my $monstat = $rados->mon_command({ prefix => 'mon_status' });
 655         my $monlist = $monstat->{monmap}->{mons};
 656
 657         die "no such monitor id '$monid'\n"
 658             if !defined($cfg->{$monsection});
 659
 660         my $ccname = PVE::Ceph::Tools::get_config('ccname');
 661
 662         my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 663         -d $mondir || die "monitor filesystem '$mondir' does not exist on this node\n";
 664
 665         die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
 666
 667         my $worker = sub {
 668             my $upid = shift;
 669
 670             # reopen with longer timeout
 671             $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
 672
 673             $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });
 674
 675             eval { PVE::Ceph::Services::ceph_service_cmd('stop', $monsection); };
 676             warn $@ if $@;
 677
 678             delete $cfg->{$monsection};
 679             cfs_write_file('ceph.conf', $cfg);
 680             File::Path::remove_tree($mondir);
 681
 682             # remove manager
 683             if (!$param->{'exclude-manager'}) {
 684                 eval { PVE::Ceph::Services::destroy_mgr($mgrid) };
 685                 warn $@ if $@;
 686             }
 687         };
 688
 689         return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
 690     }});
 691
 692 __PACKAGE__->register_method ({
 693     name => 'createmgr',
 694     path => 'mgr',
 695     method => 'POST',
 696     description => "Create Ceph Manager",
 697     proxyto => 'node',
 698     protected => 1,
 699     permissions => {
 700         check => ['perm', '/', [ 'Sys.Modify' ]],
 701     },
 702     parameters => {
 703         additionalProperties => 0,
 704         properties => {
 705             node => get_standard_option('pve-node'),
 706             id => {
 707                 type => 'string',
 708                 optional => 1,
 709                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
 710                 description => "The ID for the manager, when omitted the same as the nodename",
 711             },
 712         },
 713     },
 714     returns => { type => 'string' },
 715     code => sub {
 716         my ($param) = @_;
 717
 718         PVE::Ceph::Tools::check_ceph_installed('ceph_mgr');
 719
 720         PVE::Ceph::Tools::check_ceph_inited();
 721
 722         my $rpcenv = PVE::RPCEnvironment::get();
 723
 724         my $authuser = $rpcenv->get_user();
 725
 726         my $mgrid = $param->{id} // $param->{node};
 727
 728         my $worker = sub  {
 729             my $upid = shift;
 730
 731             my $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
 732
 733             PVE::Ceph::Services::create_mgr($mgrid, $rados);
 734         };
 735
 736         return $rpcenv->fork_worker('cephcreatemgr', "mgr.$mgrid", $authuser, $worker);
 737     }});
 738
 739 __PACKAGE__->register_method ({
 740     name => 'destroymgr',
 741     path => 'mgr/{id}',
 742     method => 'DELETE',
 743     description => "Destroy Ceph Manager.",
 744     proxyto => 'node',
 745     protected => 1,
 746     permissions => {
 747         check => ['perm', '/', [ 'Sys.Modify' ]],
 748     },
 749     parameters => {
 750         additionalProperties => 0,
 751         properties => {
 752             node => get_standard_option('pve-node'),
 753             id => {
 754                 description => 'The ID of the manager',
 755                 type => 'string',
 756                 pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
 757             },
 758         },
 759     },
 760     returns => { type => 'string' },
 761     code => sub {
 762         my ($param) = @_;
 763
 764         my $rpcenv = PVE::RPCEnvironment::get();
 765
 766         my $authuser = $rpcenv->get_user();
 767
 768         PVE::Ceph::Tools::check_ceph_inited();
 769
 770         my $mgrid = $param->{id};
 771
 772         my $worker = sub {
 773             my $upid = shift;
 774
 775             PVE::Ceph::Services::destroy_mgr($mgrid);
 776         };
 777
 778         return $rpcenv->fork_worker('cephdestroymgr', "mgr.$mgrid",  $authuser, $worker);
 779     }});
 780
 781 __PACKAGE__->register_method ({
 782     name => 'stop',
 783     path => 'stop',
 784     method => 'POST',
 785     description => "Stop ceph services.",
 786     proxyto => 'node',
 787     protected => 1,
 788     permissions => {
 789         check => ['perm', '/', [ 'Sys.Modify' ]],
 790     },
 791     parameters => {
 792         additionalProperties => 0,
 793         properties => {
 794             node => get_standard_option('pve-node'),
 795             service => {
 796                 description => 'Ceph service name.',
 797                 type => 'string',
 798                 optional => 1,
 799                 default => 'ceph.target',
 800                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
 801             },
 802         },
 803     },
 804     returns => { type => 'string' },
 805     code => sub {
 806         my ($param) = @_;
 807
 808         my $rpcenv = PVE::RPCEnvironment::get();
 809
 810         my $authuser = $rpcenv->get_user();
 811
 812         PVE::Ceph::Tools::check_ceph_inited();
 813
 814         my $cfg = cfs_read_file('ceph.conf');
 815         scalar(keys %$cfg) || die "no configuration\n";
 816
 817         my $worker = sub {
 818             my $upid = shift;
 819
 820             my $cmd = ['stop'];
 821             if ($param->{service}) {
 822                 push @$cmd, $param->{service};
 823             }
 824
 825             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 826         };
 827
 828         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
 829                                     $authuser, $worker);
 830     }});
 831
 832 __PACKAGE__->register_method ({
 833     name => 'start',
 834     path => 'start',
 835     method => 'POST',
 836     description => "Start ceph services.",
 837     proxyto => 'node',
 838     protected => 1,
 839     permissions => {
 840         check => ['perm', '/', [ 'Sys.Modify' ]],
 841     },
 842     parameters => {
 843         additionalProperties => 0,
 844         properties => {
 845             node => get_standard_option('pve-node'),
 846             service => {
 847                 description => 'Ceph service name.',
 848                 type => 'string',
 849                 optional => 1,
 850                 default => 'ceph.target',
 851                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
 852             },
 853         },
 854     },
 855     returns => { type => 'string' },
 856     code => sub {
 857         my ($param) = @_;
 858
 859         my $rpcenv = PVE::RPCEnvironment::get();
 860
 861         my $authuser = $rpcenv->get_user();
 862
 863         PVE::Ceph::Tools::check_ceph_inited();
 864
 865         my $cfg = cfs_read_file('ceph.conf');
 866         scalar(keys %$cfg) || die "no configuration\n";
 867
 868         my $worker = sub {
 869             my $upid = shift;
 870
 871             my $cmd = ['start'];
 872             if ($param->{service}) {
 873                 push @$cmd, $param->{service};
 874             }
 875
 876             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 877         };
 878
 879         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
 880                                     $authuser, $worker);
 881     }});
 882
 883 __PACKAGE__->register_method ({
 884     name => 'restart',
 885     path => 'restart',
 886     method => 'POST',
 887     description => "Restart ceph services.",
 888     proxyto => 'node',
 889     protected => 1,
 890     permissions => {
 891         check => ['perm', '/', [ 'Sys.Modify' ]],
 892     },
 893     parameters => {
 894         additionalProperties => 0,
 895         properties => {
 896             node => get_standard_option('pve-node'),
 897             service => {
 898                 description => 'Ceph service name.',
 899                 type => 'string',
 900                 optional => 1,
 901                 default => 'ceph.target',
 902                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
 903             },
 904         },
 905     },
 906     returns => { type => 'string' },
 907     code => sub {
 908         my ($param) = @_;
 909
 910         my $rpcenv = PVE::RPCEnvironment::get();
 911
 912         my $authuser = $rpcenv->get_user();
 913
 914         PVE::Ceph::Tools::check_ceph_inited();
 915
 916         my $cfg = cfs_read_file('ceph.conf');
 917         scalar(keys %$cfg) || die "no configuration\n";
 918
 919         my $worker = sub {
 920             my $upid = shift;
 921
 922             my $cmd = ['restart'];
 923             if ($param->{service}) {
 924                 push @$cmd, $param->{service};
 925             }
 926
 927             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 928         };
 929
 930         return $rpcenv->fork_worker('srvrestart', $param->{service} || 'ceph',
 931                                     $authuser, $worker);
 932     }});
 933
 934 __PACKAGE__->register_method ({
 935     name => 'status',
 936     path => 'status',
 937     method => 'GET',
 938     description => "Get ceph status.",
 939     proxyto => 'node',
 940     protected => 1,
 941     permissions => {
 942         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 943     },
 944     parameters => {
 945         additionalProperties => 0,
 946         properties => {
 947             node => get_standard_option('pve-node'),
 948         },
 949     },
 950     returns => { type => 'object' },
 951     code => sub {
 952         my ($param) = @_;
 953
 954         PVE::Ceph::Tools::check_ceph_enabled();
 955
 956         my $rados = PVE::RADOS->new();
 957         my $status = $rados->mon_command({ prefix => 'status' });
 958         $status->{health} = $rados->mon_command({ prefix => 'health', detail => 'detail' });
 959         return $status;
 960     }});
 961
 962 __PACKAGE__->register_method ({
 963     name => 'lspools',
 964     path => 'pools',
 965     method => 'GET',
 966     description => "List all pools.",
 967     proxyto => 'node',
 968     protected => 1,
 969     permissions => {
 970         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 971     },
 972     parameters => {
 973         additionalProperties => 0,
 974         properties => {
 975             node => get_standard_option('pve-node'),
 976         },
 977     },
 978     returns => {
 979         type => 'array',
 980         items => {
 981             type => "object",
 982             properties => {
 983                 pool => { type => 'integer' },
 984                 pool_name => { type => 'string' },
 985                 size => { type => 'integer' },
 986             },
 987         },
 988         links => [ { rel => 'child', href => "{pool_name}" } ],
 989     },
 990     code => sub {
 991         my ($param) = @_;
 992
 993         PVE::Ceph::Tools::check_ceph_inited();
 994
 995         my $rados = PVE::RADOS->new();
 996
 997         my $stats = {};
 998         my $res = $rados->mon_command({ prefix => 'df' });
 999
1000         foreach my $d (@{$res->{pools}}) {
1001             next if !$d->{stats};
1002             next if !defined($d->{id});
1003             $stats->{$d->{id}} = $d->{stats};
1004         }
1005
1006         $res = $rados->mon_command({ prefix => 'osd dump' });
1007         my $rulestmp = $rados->mon_command({ prefix => 'osd crush rule dump'});
1008
1009         my $rules = {};
1010         for my $rule (@$rulestmp) {
1011             $rules->{$rule->{rule_id}} = $rule->{rule_name};
1012         }
1013
1014         my $data = [];
1015         foreach my $e (@{$res->{pools}}) {
1016             my $d = {};
1017             foreach my $attr (qw(pool pool_name size min_size pg_num crush_rule)) {
1018                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
1019             }
1020
1021             if (defined($d->{crush_rule}) && defined($rules->{$d->{crush_rule}})) {
1022                 $d->{crush_rule_name} = $rules->{$d->{crush_rule}};
1023             }
1024
1025             if (my $s = $stats->{$d->{pool}}) {
1026                 $d->{bytes_used} = $s->{bytes_used};
1027                 $d->{percent_used} = $s->{percent_used};
1028             }
1029             push @$data, $d;
1030         }
1031
1032
1033         return $data;
1034     }});
1035
1036 __PACKAGE__->register_method ({
1037     name => 'createpool',
1038     path => 'pools',
1039     method => 'POST',
1040     description => "Create POOL",
1041     proxyto => 'node',
1042     protected => 1,
1043     permissions => {
1044         check => ['perm', '/', [ 'Sys.Modify' ]],
1045     },
1046     parameters => {
1047         additionalProperties => 0,
1048         properties => {
1049             node => get_standard_option('pve-node'),
1050             name => {
1051                 description => "The name of the pool. It must be unique.",
1052                 type => 'string',
1053             },
1054             size => {
1055                 description => 'Number of replicas per object',
1056                 type => 'integer',
1057                 default => 3,
1058                 optional => 1,
1059                 minimum => 1,
1060                 maximum => 7,
1061             },
1062             min_size => {
1063                 description => 'Minimum number of replicas per object',
1064                 type => 'integer',
1065                 default => 2,
1066                 optional => 1,
1067                 minimum => 1,
1068                 maximum => 7,
1069             },
1070             pg_num => {
1071                 description => "Number of placement groups.",
1072                 type => 'integer',
1073                 default => 128,
1074                 optional => 1,
1075                 minimum => 8,
1076                 maximum => 32768,
1077             },
1078             crush_rule => {
1079                 description => "The rule to use for mapping object placement in the cluster.",
1080                 type => 'string',
1081                 optional => 1,
1082             },
1083             application => {
1084                 description => "The application of the pool, 'rbd' by default.",
1085                 type => 'string',
1086                 enum => ['rbd', 'cephfs', 'rgw'],
1087                 optional => 1,
1088             },
1089             add_storages => {
1090                 description => "Configure VM and CT storage using the new pool.",
1091                 type => 'boolean',
1092                 optional => 1,
1093             },
1094         },
1095     },
1096     returns => { type => 'string' },
1097     code => sub {
1098         my ($param) = @_;
1099
1100         PVE::Cluster::check_cfs_quorum();
1101         PVE::Ceph::Tools::check_ceph_inited();
1102
1103         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
1104
1105         die "not fully configured - missing '$pve_ckeyring_path'\n"
1106             if ! -f $pve_ckeyring_path;
1107
1108         my $pool = $param->{name};
1109         my $rpcenv = PVE::RPCEnvironment::get();
1110         my $user = $rpcenv->get_user();
1111
1112         if ($param->{add_storages}) {
1113             $rpcenv->check($user, '/storage', ['Datastore.Allocate']);
1114             die "pool name contains characters which are illegal for storage naming\n"
1115                 if !PVE::JSONSchema::parse_storage_id($pool);
1116         }
1117
1118         my $pg_num = $param->{pg_num} || 128;
1119         my $size = $param->{size} || 3;
1120         my $min_size = $param->{min_size} || 2;
1121         my $application = $param->{application} // 'rbd';
1122
1123         my $worker = sub {
1124
1125             PVE::Ceph::Tools::create_pool($pool, $param);
1126
1127             if ($param->{add_storages}) {
1128                 my $err;
1129                 eval { $add_storage->($pool, "${pool}"); };
1130                 if ($@) {
1131                     warn "failed to add storage: $@";
1132                     $err = 1;
1133                 }
1134                 die "adding storage for pool '$pool' failed, check log and add manually!\n"
1135                     if $err;
1136             }
1137         };
1138
1139         return $rpcenv->fork_worker('cephcreatepool', $pool,  $user, $worker);
1140     }});
1141
1142 __PACKAGE__->register_method ({
1143     name => 'get_flags',
1144     path => 'flags',
1145     method => 'GET',
1146     description => "get all set ceph flags",
1147     proxyto => 'node',
1148     protected => 1,
1149     permissions => {
1150         check => ['perm', '/', [ 'Sys.Audit' ]],
1151     },
1152     parameters => {
1153         additionalProperties => 0,
1154         properties => {
1155             node => get_standard_option('pve-node'),
1156         },
1157     },
1158     returns => { type => 'string' },
1159     code => sub {
1160         my ($param) = @_;
1161
1162         PVE::Ceph::Tools::check_ceph_inited();
1163
1164         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
1165
1166         die "not fully configured - missing '$pve_ckeyring_path'\n"
1167             if ! -f $pve_ckeyring_path;
1168
1169         my $rados = PVE::RADOS->new();
1170
1171         my $stat = $rados->mon_command({ prefix => 'osd dump' });
1172
1173         return $stat->{flags} // '';
1174     }});
1175
1176 __PACKAGE__->register_method ({
1177     name => 'set_flag',
1178     path => 'flags/{flag}',
1179     method => 'POST',
1180     description => "Set a ceph flag",
1181     proxyto => 'node',
1182     protected => 1,
1183     permissions => {
1184         check => ['perm', '/', [ 'Sys.Modify' ]],
1185     },
1186     parameters => {
1187         additionalProperties => 0,
1188         properties => {
1189             node => get_standard_option('pve-node'),
1190             flag => {
1191                 description => 'The ceph flag to set/unset',
1192                 type => 'string',
1193                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1194             },
1195         },
1196     },
1197     returns => { type => 'null' },
1198     code => sub {
1199         my ($param) = @_;
1200
1201         PVE::Ceph::Tools::check_ceph_inited();
1202
1203         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
1204
1205         die "not fully configured - missing '$pve_ckeyring_path'\n"
1206             if ! -f $pve_ckeyring_path;
1207
1208         my $set = $param->{set} // !$param->{unset};
1209         my $rados = PVE::RADOS->new();
1210
1211         $rados->mon_command({
1212             prefix => "osd set",
1213             key => $param->{flag},
1214         });
1215
1216         return undef;
1217     }});
1218
1219 __PACKAGE__->register_method ({
1220     name => 'unset_flag',
1221     path => 'flags/{flag}',
1222     method => 'DELETE',
1223     description => "Unset a ceph flag",
1224     proxyto => 'node',
1225     protected => 1,
1226     permissions => {
1227         check => ['perm', '/', [ 'Sys.Modify' ]],
1228     },
1229     parameters => {
1230         additionalProperties => 0,
1231         properties => {
1232             node => get_standard_option('pve-node'),
1233             flag => {
1234                 description => 'The ceph flag to set/unset',
1235                 type => 'string',
1236                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
1237             },
1238         },
1239     },
1240     returns => { type => 'null' },
1241     code => sub {
1242         my ($param) = @_;
1243
1244         PVE::Ceph::Tools::check_ceph_inited();
1245
1246         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
1247
1248         die "not fully configured - missing '$pve_ckeyring_path'\n"
1249             if ! -f $pve_ckeyring_path;
1250
1251         my $set = $param->{set} // !$param->{unset};
1252         my $rados = PVE::RADOS->new();
1253
1254         $rados->mon_command({
1255             prefix => "osd unset",
1256             key => $param->{flag},
1257         });
1258
1259         return undef;
1260     }});
1261
1262 __PACKAGE__->register_method ({
1263     name => 'destroypool',
1264     path => 'pools/{name}',
1265     method => 'DELETE',
1266     description => "Destroy pool",
1267     proxyto => 'node',
1268     protected => 1,
1269     permissions => {
1270         check => ['perm', '/', [ 'Sys.Modify' ]],
1271     },
1272     parameters => {
1273         additionalProperties => 0,
1274         properties => {
1275             node => get_standard_option('pve-node'),
1276             name => {
1277                 description => "The name of the pool. It must be unique.",
1278                 type => 'string',
1279             },
1280             force => {
1281                 description => "If true, destroys pool even if in use",
1282                 type => 'boolean',
1283                 optional => 1,
1284                 default => 0,
1285             },
1286             remove_storages => {
1287                 description => "Remove all pveceph-managed storages configured for this pool",
1288                 type => 'boolean',
1289                 optional => 1,
1290                 default => 0,
1291             },
1292         },
1293     },
1294     returns => { type => 'string' },
1295     code => sub {
1296         my ($param) = @_;
1297
1298         PVE::Ceph::Tools::check_ceph_inited();
1299
1300         my $rpcenv = PVE::RPCEnvironment::get();
1301         my $user = $rpcenv->get_user();
1302         $rpcenv->check($user, '/storage', ['Datastore.Allocate'])
1303             if $param->{remove_storages};
1304
1305         my $pool = $param->{name};
1306
1307         my $worker = sub {
1308             my $storages = $get_storages->($pool);
1309
1310             # if not forced, destroy ceph pool only when no
1311             # vm disks are on it anymore
1312             if (!$param->{force}) {
1313                 my $storagecfg = PVE::Storage::config();
1314                 foreach my $storeid (keys %$storages) {
1315                     my $storage = $storages->{$storeid};
1316
1317                     # check if any vm disks are on the pool
1318                     print "checking storage '$storeid' for RBD images..\n";
1319                     my $res = PVE::Storage::vdisk_list($storagecfg, $storeid);
1320                     die "ceph pool '$pool' still in use by storage '$storeid'\n"
1321                         if @{$res->{$storeid}} != 0;
1322                 }
1323             }
1324
1325             PVE::Ceph::Tools::destroy_pool($pool);
1326
1327             if ($param->{remove_storages}) {
1328                 my $err;
1329                 foreach my $storeid (keys %$storages) {
1330                     # skip external clusters, not managed by pveceph
1331                     next if $storages->{$storeid}->{monhost};
1332                     eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
1333                     if ($@) {
1334                         warn "failed to remove storage '$storeid': $@\n";
1335                         $err = 1;
1336                     }
1337                 }
1338                 die "failed to remove (some) storages - check log and remove manually!\n"
1339                     if $err;
1340             }
1341         };
1342         return $rpcenv->fork_worker('cephdestroypool', $pool,  $user, $worker);
1343     }});
1344
1345
1346 __PACKAGE__->register_method ({
1347     name => 'crush',
1348     path => 'crush',
1349     method => 'GET',
1350     description => "Get OSD crush map",
1351     proxyto => 'node',
1352     protected => 1,
1353     permissions => {
1354         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1355     },
1356     parameters => {
1357         additionalProperties => 0,
1358         properties => {
1359             node => get_standard_option('pve-node'),
1360         },
1361     },
1362     returns => { type => 'string' },
1363     code => sub {
1364         my ($param) = @_;
1365
1366         PVE::Ceph::Tools::check_ceph_inited();
1367
1368         # this produces JSON (difficult to read for the user)
1369         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
1370
1371         my $txt = '';
1372
1373         my $mapfile = "/var/tmp/ceph-crush.map.$$";
1374         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
1375
1376         my $rados = PVE::RADOS->new();
1377
1378         eval {
1379             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
1380             file_set_contents($mapfile, $bindata);
1381             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
1382             $txt = file_get_contents($mapdata);
1383         };
1384         my $err = $@;
1385
1386         unlink $mapfile;
1387         unlink $mapdata;
1388
1389         die $err if $err;
1390
1391         return $txt;
1392     }});
1393
1394 __PACKAGE__->register_method({
1395     name => 'log',
1396     path => 'log',
1397     method => 'GET',
1398     description => "Read ceph log",
1399     proxyto => 'node',
1400     permissions => {
1401         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
1402     },
1403     protected => 1,
1404     parameters => {
1405         additionalProperties => 0,
1406         properties => {
1407             node => get_standard_option('pve-node'),
1408             start => {
1409                 type => 'integer',
1410                 minimum => 0,
1411                 optional => 1,
1412             },
1413             limit => {
1414                 type => 'integer',
1415                 minimum => 0,
1416                 optional => 1,
1417             },
1418         },
1419     },
1420     returns => {
1421         type => 'array',
1422         items => {
1423             type => "object",
1424             properties => {
1425                 n => {
1426                   description=>  "Line number",
1427                   type=> 'integer',
1428                 },
1429                 t => {
1430                   description=>  "Line text",
1431                   type => 'string',
1432                 }
1433             }
1434         }
1435     },
1436     code => sub {
1437         my ($param) = @_;
1438
1439         PVE::Ceph::Tools::check_ceph_inited();
1440
1441         my $rpcenv = PVE::RPCEnvironment::get();
1442         my $user = $rpcenv->get_user();
1443         my $node = $param->{node};
1444
1445         my $logfile = "/var/log/ceph/ceph.log";
1446         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
1447
1448         $rpcenv->set_result_attrib('total', $count);
1449
1450         return $lines;
1451     }});
1452
1453 __PACKAGE__->register_method ({
1454     name => 'rules',
1455     path => 'rules',
1456     method => 'GET',
1457     description => "List ceph rules.",
1458     proxyto => 'node',
1459     protected => 1,
1460     permissions => {
1461         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1462     },
1463     parameters => {
1464         additionalProperties => 0,
1465         properties => {
1466             node => get_standard_option('pve-node'),
1467         },
1468     },
1469     returns => {
1470         type => 'array',
1471         items => {
1472             type => "object",
1473             properties => {},
1474         },
1475         links => [ { rel => 'child', href => "{name}" } ],
1476     },
1477     code => sub {
1478         my ($param) = @_;
1479
1480         PVE::Ceph::Tools::check_ceph_inited();
1481
1482         my $rados = PVE::RADOS->new();
1483
1484         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
1485
1486         my $res = [];
1487
1488         foreach my $rule (@$rules) {
1489             push @$res, { name => $rule };
1490         }
1491
1492         return $res;
1493     }});
1494
1495 1;