PVE/API2/Ceph.pm

   1 package PVE::API2::Ceph;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use Net::IP;
   8 use UUID;
   9
  10 use PVE::Ceph::Tools;
  11 use PVE::Ceph::Services;
  12 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  13 use PVE::JSONSchema qw(get_standard_option);
  14 use PVE::Network;
  15 use PVE::RADOS;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::Storage;
  19 use PVE::Tools qw(run_command file_get_contents file_set_contents);
  20
  21 use PVE::API2::Ceph::OSD;
  22 use PVE::API2::Ceph::FS;
  23 use PVE::API2::Ceph::MDS;
  24 use PVE::API2::Ceph::MGR;
  25 use PVE::API2::Ceph::MON;
  26 use PVE::API2::Storage::Config;
  27
  28 use base qw(PVE::RESTHandler);
  29
  30 my $pve_osd_default_journal_size = 1024*5;
  31
  32 __PACKAGE__->register_method ({
  33     subclass => "PVE::API2::Ceph::OSD",
  34     path => 'osd',
  35 });
  36
  37 __PACKAGE__->register_method ({
  38     subclass => "PVE::API2::Ceph::MDS",
  39     path => 'mds',
  40 });
  41
  42 __PACKAGE__->register_method ({
  43     subclass => "PVE::API2::Ceph::MGR",
  44     path => 'mgr',
  45 });
  46
  47 __PACKAGE__->register_method ({
  48     subclass => "PVE::API2::Ceph::MON",
  49     path => 'mon',
  50 });
  51
  52 __PACKAGE__->register_method ({
  53     subclass => "PVE::API2::Ceph::FS",
  54     path => 'fs',
  55 });
  56
  57 __PACKAGE__->register_method ({
  58     name => 'index',
  59     path => '',
  60     method => 'GET',
  61     description => "Directory index.",
  62     permissions => { user => 'all' },
  63     permissions => {
  64         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  65     },
  66     parameters => {
  67         additionalProperties => 0,
  68         properties => {
  69             node => get_standard_option('pve-node'),
  70         },
  71     },
  72     returns => {
  73         type => 'array',
  74         items => {
  75             type => "object",
  76             properties => {},
  77         },
  78         links => [ { rel => 'child', href => "{name}" } ],
  79     },
  80     code => sub {
  81         my ($param) = @_;
  82
  83         my $result = [
  84             { name => 'init' },
  85             { name => 'mon' },
  86             { name => 'osd' },
  87             { name => 'pools' },
  88             { name => 'fs' },
  89             { name => 'mds' },
  90             { name => 'stop' },
  91             { name => 'start' },
  92             { name => 'restart' },
  93             { name => 'status' },
  94             { name => 'crush' },
  95             { name => 'config' },
  96             { name => 'log' },
  97             { name => 'disks' },
  98             { name => 'flags' }, # FIXME: remove with 7.0
  99             { name => 'rules' },
 100         ];
 101
 102         return $result;
 103     }});
 104
 105 __PACKAGE__->register_method ({
 106     name => 'disks',
 107     path => 'disks',
 108     method => 'GET',
 109     description => "List local disks.",
 110     proxyto => 'node',
 111     protected => 1,
 112     permissions => {
 113         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 114     },
 115     parameters => {
 116         additionalProperties => 0,
 117         properties => {
 118             node => get_standard_option('pve-node'),
 119             type => {
 120                 description => "Only list specific types of disks.",
 121                 type => 'string',
 122                 enum => ['unused', 'journal_disks'],
 123                 optional => 1,
 124             },
 125         },
 126     },
 127     returns => {
 128         type => 'array',
 129         items => {
 130             type => "object",
 131             properties => {
 132                 dev => { type => 'string' },
 133                 used => { type => 'string', optional => 1 },
 134                 gpt => { type => 'boolean' },
 135                 size => { type => 'integer' },
 136                 osdid => { type => 'integer' },
 137                 vendor =>  { type => 'string', optional => 1 },
 138                 model =>  { type => 'string', optional => 1 },
 139                 serial =>  { type => 'string', optional => 1 },
 140             },
 141         },
 142         # links => [ { rel => 'child', href => "{}" } ],
 143     },
 144     code => sub {
 145         my ($param) = @_;
 146
 147         PVE::Ceph::Tools::check_ceph_inited();
 148
 149         my $disks = PVE::Diskmanage::get_disks(undef, 1);
 150
 151         my $res = [];
 152         foreach my $dev (keys %$disks) {
 153             my $d = $disks->{$dev};
 154             if ($param->{type}) {
 155                 if ($param->{type} eq 'journal_disks') {
 156                     next if $d->{osdid} >= 0;
 157                     next if !$d->{gpt};
 158                 } elsif ($param->{type} eq 'unused') {
 159                     next if $d->{used};
 160                 } else {
 161                     die "internal error"; # should not happen
 162                 }
 163             }
 164
 165             $d->{dev} = "/dev/$dev";
 166             push @$res, $d;
 167         }
 168
 169         return $res;
 170     }});
 171
 172 __PACKAGE__->register_method ({
 173     name => 'config',
 174     path => 'config',
 175     method => 'GET',
 176     proxyto => 'node',
 177     permissions => {
 178         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 179     },
 180     description => "Get Ceph configuration.",
 181     parameters => {
 182         additionalProperties => 0,
 183         properties => {
 184             node => get_standard_option('pve-node'),
 185         },
 186     },
 187     returns => { type => 'string' },
 188     code => sub {
 189         my ($param) = @_;
 190
 191         PVE::Ceph::Tools::check_ceph_inited();
 192
 193         my $path = PVE::Ceph::Tools::get_config('pve_ceph_cfgpath');
 194         return file_get_contents($path);
 195
 196     }});
 197
 198 __PACKAGE__->register_method ({
 199     name => 'configdb',
 200     path => 'configdb',
 201     method => 'GET',
 202     proxyto => 'node',
 203     protected => 1,
 204     permissions => {
 205         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 206     },
 207     description => "Get Ceph configuration database.",
 208     parameters => {
 209         additionalProperties => 0,
 210         properties => {
 211             node => get_standard_option('pve-node'),
 212         },
 213     },
 214     returns => {
 215         type => 'array',
 216         items => {
 217             type => 'object',
 218             properties => {
 219                 section => { type => "string", },
 220                 name => { type => "string", },
 221                 value => { type => "string", },
 222                 level => { type => "string", },
 223                 'can_update_at_runtime' => { type => "boolean", },
 224                 mask => { type => "string" },
 225             },
 226         },
 227     },
 228     code => sub {
 229         my ($param) = @_;
 230
 231         PVE::Ceph::Tools::check_ceph_inited();
 232
 233         my $rados = PVE::RADOS->new();
 234         my $res = $rados->mon_command( { prefix => 'config dump', format => 'json' });
 235         foreach my $entry (@$res) {
 236             $entry->{can_update_at_runtime} = $entry->{can_update_at_runtime}? 1 : 0; # JSON::true/false -> 1/0
 237         }
 238
 239         return $res;
 240     }});
 241
 242 my $add_storage = sub {
 243     my ($pool, $storeid) = @_;
 244
 245     my $storage_params = {
 246         type => 'rbd',
 247         pool => $pool,
 248         storage => $storeid,
 249         krbd => 0,
 250         content => 'rootdir,images',
 251     };
 252
 253     PVE::API2::Storage::Config->create($storage_params);
 254 };
 255
 256 my $get_storages = sub {
 257     my ($pool) = @_;
 258
 259     my $cfg = PVE::Storage::config();
 260
 261     my $storages = $cfg->{ids};
 262     my $res = {};
 263     foreach my $storeid (keys %$storages) {
 264         my $curr = $storages->{$storeid};
 265         $res->{$storeid} = $storages->{$storeid}
 266             if $curr->{type} eq 'rbd' && $pool eq $curr->{pool};
 267     }
 268
 269     return $res;
 270 };
 271
 272 __PACKAGE__->register_method ({
 273     name => 'init',
 274     path => 'init',
 275     method => 'POST',
 276     description => "Create initial ceph default configuration and setup symlinks.",
 277     proxyto => 'node',
 278     protected => 1,
 279     permissions => {
 280         check => ['perm', '/', [ 'Sys.Modify' ]],
 281     },
 282     parameters => {
 283         additionalProperties => 0,
 284         properties => {
 285             node => get_standard_option('pve-node'),
 286             network => {
 287                 description => "Use specific network for all ceph related traffic",
 288                 type => 'string', format => 'CIDR',
 289                 optional => 1,
 290                 maxLength => 128,
 291             },
 292             'cluster-network' => {
 293                 description => "Declare a separate cluster network, OSDs will route" .
 294                     "heartbeat, object replication and recovery traffic over it",
 295                 type => 'string', format => 'CIDR',
 296                 requires => 'network',
 297                 optional => 1,
 298                 maxLength => 128,
 299             },
 300             size => {
 301                 description => 'Targeted number of replicas per object',
 302                 type => 'integer',
 303                 default => 3,
 304                 optional => 1,
 305                 minimum => 1,
 306                 maximum => 7,
 307             },
 308             min_size => {
 309                 description => 'Minimum number of available replicas per object to allow I/O',
 310                 type => 'integer',
 311                 default => 2,
 312                 optional => 1,
 313                 minimum => 1,
 314                 maximum => 7,
 315             },
 316             pg_bits => {
 317                 description => "Placement group bits, used to specify the " .
 318                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 319                     "default pg num' does not work for default pools.",
 320                 type => 'integer',
 321                 default => 6,
 322                 optional => 1,
 323                 minimum => 6,
 324                 maximum => 14,
 325             },
 326             disable_cephx => {
 327                 description => "Disable cephx authentication.\n\n" .
 328                     "WARNING: cephx is a security feature protecting against " .
 329                     "man-in-the-middle attacks. Only consider disabling cephx ".
 330                     "if your network is private!",
 331                 type => 'boolean',
 332                 optional => 1,
 333                 default => 0,
 334             },
 335         },
 336     },
 337     returns => { type => 'null' },
 338     code => sub {
 339         my ($param) = @_;
 340
 341         my $version = PVE::Ceph::Tools::get_local_version(1);
 342
 343         if (!$version || $version < 14) {
 344             die "Ceph Nautilus required - please run 'pveceph install'\n";
 345         } else {
 346             PVE::Ceph::Tools::check_ceph_installed('ceph_bin');
 347         }
 348
 349         my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 350
 351         # simply load old config if it already exists
 352         PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
 353             my $cfg = cfs_read_file('ceph.conf');
 354
 355             if (!$cfg->{global}) {
 356
 357                 my $fsid;
 358                 my $uuid;
 359
 360                 UUID::generate($uuid);
 361                 UUID::unparse($uuid, $fsid);
 362
 363                 $cfg->{global} = {
 364                     'fsid' => $fsid,
 365                     'auth cluster required' => $auth,
 366                     'auth service required' => $auth,
 367                     'auth client required' => $auth,
 368                     'osd pool default size' => $param->{size} // 3,
 369                     'osd pool default min size' => $param->{min_size} // 2,
 370                     'mon allow pool delete' => 'true',
 371                 };
 372
 373                 # this does not work for default pools
 374                 #'osd pool default pg num' => $pg_num,
 375                 #'osd pool default pgp num' => $pg_num,
 376             }
 377
 378             if ($auth eq 'cephx') {
 379                 $cfg->{client}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 380             }
 381
 382             if ($param->{pg_bits}) {
 383                 $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 384                 $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 385             }
 386
 387             if ($param->{network}) {
 388                 $cfg->{global}->{'public network'} = $param->{network};
 389                 $cfg->{global}->{'cluster network'} = $param->{network};
 390             }
 391
 392             if ($param->{'cluster-network'}) {
 393                 $cfg->{global}->{'cluster network'} = $param->{'cluster-network'};
 394             }
 395
 396             cfs_write_file('ceph.conf', $cfg);
 397
 398             if ($auth eq 'cephx') {
 399                 PVE::Ceph::Tools::get_or_create_admin_keyring();
 400             }
 401             PVE::Ceph::Tools::setup_pve_symlinks();
 402         });
 403         die $@ if $@;
 404
 405         return undef;
 406     }});
 407
 408 __PACKAGE__->register_method ({
 409     name => 'stop',
 410     path => 'stop',
 411     method => 'POST',
 412     description => "Stop ceph services.",
 413     proxyto => 'node',
 414     protected => 1,
 415     permissions => {
 416         check => ['perm', '/', [ 'Sys.Modify' ]],
 417     },
 418     parameters => {
 419         additionalProperties => 0,
 420         properties => {
 421             node => get_standard_option('pve-node'),
 422             service => {
 423                 description => 'Ceph service name.',
 424                 type => 'string',
 425                 optional => 1,
 426                 default => 'ceph.target',
 427                 pattern => '(ceph|mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 428             },
 429         },
 430     },
 431     returns => { type => 'string' },
 432     code => sub {
 433         my ($param) = @_;
 434
 435         my $rpcenv = PVE::RPCEnvironment::get();
 436
 437         my $authuser = $rpcenv->get_user();
 438
 439         PVE::Ceph::Tools::check_ceph_inited();
 440
 441         my $cfg = cfs_read_file('ceph.conf');
 442         scalar(keys %$cfg) || die "no configuration\n";
 443
 444         my $worker = sub {
 445             my $upid = shift;
 446
 447             my $cmd = ['stop'];
 448             if ($param->{service}) {
 449                 push @$cmd, $param->{service};
 450             }
 451
 452             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 453         };
 454
 455         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
 456                                     $authuser, $worker);
 457     }});
 458
 459 __PACKAGE__->register_method ({
 460     name => 'start',
 461     path => 'start',
 462     method => 'POST',
 463     description => "Start ceph services.",
 464     proxyto => 'node',
 465     protected => 1,
 466     permissions => {
 467         check => ['perm', '/', [ 'Sys.Modify' ]],
 468     },
 469     parameters => {
 470         additionalProperties => 0,
 471         properties => {
 472             node => get_standard_option('pve-node'),
 473             service => {
 474                 description => 'Ceph service name.',
 475                 type => 'string',
 476                 optional => 1,
 477                 default => 'ceph.target',
 478                 pattern => '(ceph|mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 479             },
 480         },
 481     },
 482     returns => { type => 'string' },
 483     code => sub {
 484         my ($param) = @_;
 485
 486         my $rpcenv = PVE::RPCEnvironment::get();
 487
 488         my $authuser = $rpcenv->get_user();
 489
 490         PVE::Ceph::Tools::check_ceph_inited();
 491
 492         my $cfg = cfs_read_file('ceph.conf');
 493         scalar(keys %$cfg) || die "no configuration\n";
 494
 495         my $worker = sub {
 496             my $upid = shift;
 497
 498             my $cmd = ['start'];
 499             if ($param->{service}) {
 500                 push @$cmd, $param->{service};
 501             }
 502
 503             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 504         };
 505
 506         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
 507                                     $authuser, $worker);
 508     }});
 509
 510 __PACKAGE__->register_method ({
 511     name => 'restart',
 512     path => 'restart',
 513     method => 'POST',
 514     description => "Restart ceph services.",
 515     proxyto => 'node',
 516     protected => 1,
 517     permissions => {
 518         check => ['perm', '/', [ 'Sys.Modify' ]],
 519     },
 520     parameters => {
 521         additionalProperties => 0,
 522         properties => {
 523             node => get_standard_option('pve-node'),
 524             service => {
 525                 description => 'Ceph service name.',
 526                 type => 'string',
 527                 optional => 1,
 528                 default => 'ceph.target',
 529                 pattern => '(mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 530             },
 531         },
 532     },
 533     returns => { type => 'string' },
 534     code => sub {
 535         my ($param) = @_;
 536
 537         my $rpcenv = PVE::RPCEnvironment::get();
 538
 539         my $authuser = $rpcenv->get_user();
 540
 541         PVE::Ceph::Tools::check_ceph_inited();
 542
 543         my $cfg = cfs_read_file('ceph.conf');
 544         scalar(keys %$cfg) || die "no configuration\n";
 545
 546         my $worker = sub {
 547             my $upid = shift;
 548
 549             my $cmd = ['restart'];
 550             if ($param->{service}) {
 551                 push @$cmd, $param->{service};
 552             }
 553
 554             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 555         };
 556
 557         return $rpcenv->fork_worker('srvrestart', $param->{service} || 'ceph',
 558                                     $authuser, $worker);
 559     }});
 560
 561 __PACKAGE__->register_method ({
 562     name => 'status',
 563     path => 'status',
 564     method => 'GET',
 565     description => "Get ceph status.",
 566     proxyto => 'node',
 567     protected => 1,
 568     permissions => {
 569         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 570     },
 571     parameters => {
 572         additionalProperties => 0,
 573         properties => {
 574             node => get_standard_option('pve-node'),
 575         },
 576     },
 577     returns => { type => 'object' },
 578     code => sub {
 579         my ($param) = @_;
 580
 581         PVE::Ceph::Tools::check_ceph_inited();
 582
 583         return PVE::Ceph::Tools::ceph_cluster_status();
 584     }});
 585
 586 __PACKAGE__->register_method ({
 587     name => 'lspools',
 588     path => 'pools',
 589     method => 'GET',
 590     description => "List all pools.",
 591     proxyto => 'node',
 592     protected => 1,
 593     permissions => {
 594         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 595     },
 596     parameters => {
 597         additionalProperties => 0,
 598         properties => {
 599             node => get_standard_option('pve-node'),
 600         },
 601     },
 602     returns => {
 603         type => 'array',
 604         items => {
 605             type => "object",
 606             properties => {
 607                 pool => { type => 'integer' },
 608                 pool_name => { type => 'string' },
 609                 size => { type => 'integer' },
 610             },
 611         },
 612         links => [ { rel => 'child', href => "{pool_name}" } ],
 613     },
 614     code => sub {
 615         my ($param) = @_;
 616
 617         PVE::Ceph::Tools::check_ceph_inited();
 618
 619         my $rados = PVE::RADOS->new();
 620
 621         my $stats = {};
 622         my $res = $rados->mon_command({ prefix => 'df' });
 623
 624         foreach my $d (@{$res->{pools}}) {
 625             next if !$d->{stats};
 626             next if !defined($d->{id});
 627             $stats->{$d->{id}} = $d->{stats};
 628         }
 629
 630         $res = $rados->mon_command({ prefix => 'osd dump' });
 631         my $rulestmp = $rados->mon_command({ prefix => 'osd crush rule dump'});
 632
 633         my $rules = {};
 634         for my $rule (@$rulestmp) {
 635             $rules->{$rule->{rule_id}} = $rule->{rule_name};
 636         }
 637
 638         my $data = [];
 639         foreach my $e (@{$res->{pools}}) {
 640             my $d = {};
 641             foreach my $attr (qw(pool pool_name size min_size pg_num crush_rule)) {
 642                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
 643             }
 644
 645             if (defined($d->{crush_rule}) && defined($rules->{$d->{crush_rule}})) {
 646                 $d->{crush_rule_name} = $rules->{$d->{crush_rule}};
 647             }
 648
 649             if (my $s = $stats->{$d->{pool}}) {
 650                 $d->{bytes_used} = $s->{bytes_used};
 651                 $d->{percent_used} = $s->{percent_used};
 652             }
 653             push @$data, $d;
 654         }
 655
 656
 657         return $data;
 658     }});
 659
 660 __PACKAGE__->register_method ({
 661     name => 'createpool',
 662     path => 'pools',
 663     method => 'POST',
 664     description => "Create POOL",
 665     proxyto => 'node',
 666     protected => 1,
 667     permissions => {
 668         check => ['perm', '/', [ 'Sys.Modify' ]],
 669     },
 670     parameters => {
 671         additionalProperties => 0,
 672         properties => {
 673             node => get_standard_option('pve-node'),
 674             name => {
 675                 description => "The name of the pool. It must be unique.",
 676                 type => 'string',
 677             },
 678             size => {
 679                 description => 'Number of replicas per object',
 680                 type => 'integer',
 681                 default => 3,
 682                 optional => 1,
 683                 minimum => 1,
 684                 maximum => 7,
 685             },
 686             min_size => {
 687                 description => 'Minimum number of replicas per object',
 688                 type => 'integer',
 689                 default => 2,
 690                 optional => 1,
 691                 minimum => 1,
 692                 maximum => 7,
 693             },
 694             pg_num => {
 695                 description => "Number of placement groups.",
 696                 type => 'integer',
 697                 default => 128,
 698                 optional => 1,
 699                 minimum => 8,
 700                 maximum => 32768,
 701             },
 702             crush_rule => {
 703                 description => "The rule to use for mapping object placement in the cluster.",
 704                 type => 'string',
 705                 optional => 1,
 706             },
 707             application => {
 708                 description => "The application of the pool, 'rbd' by default.",
 709                 type => 'string',
 710                 enum => ['rbd', 'cephfs', 'rgw'],
 711                 optional => 1,
 712             },
 713             add_storages => {
 714                 description => "Configure VM and CT storage using the new pool.",
 715                 type => 'boolean',
 716                 optional => 1,
 717             },
 718         },
 719     },
 720     returns => { type => 'string' },
 721     code => sub {
 722         my ($param) = @_;
 723
 724         PVE::Cluster::check_cfs_quorum();
 725         PVE::Ceph::Tools::check_ceph_configured();
 726
 727         my $pool = $param->{name};
 728         my $rpcenv = PVE::RPCEnvironment::get();
 729         my $user = $rpcenv->get_user();
 730
 731         if ($param->{add_storages}) {
 732             $rpcenv->check($user, '/storage', ['Datastore.Allocate']);
 733             die "pool name contains characters which are illegal for storage naming\n"
 734                 if !PVE::JSONSchema::parse_storage_id($pool);
 735         }
 736
 737         my $pg_num = $param->{pg_num} || 128;
 738         my $size = $param->{size} || 3;
 739         my $min_size = $param->{min_size} || 2;
 740         my $application = $param->{application} // 'rbd';
 741
 742         my $worker = sub {
 743
 744             PVE::Ceph::Tools::create_pool($pool, $param);
 745
 746             if ($param->{add_storages}) {
 747                 my $err;
 748                 eval { $add_storage->($pool, "${pool}"); };
 749                 if ($@) {
 750                     warn "failed to add storage: $@";
 751                     $err = 1;
 752                 }
 753                 die "adding storage for pool '$pool' failed, check log and add manually!\n"
 754                     if $err;
 755             }
 756         };
 757
 758         return $rpcenv->fork_worker('cephcreatepool', $pool,  $user, $worker);
 759     }});
 760
 761 my $possible_flags = PVE::Ceph::Tools::get_possible_osd_flags();
 762 my $possible_flags_list = [ sort keys %$possible_flags ];
 763
 764 # FIXME: Remove with PVE 7.0
 765 __PACKAGE__->register_method ({
 766     name => 'get_flags',
 767     path => 'flags',
 768     method => 'GET',
 769     description => "get all set ceph flags",
 770     proxyto => 'node',
 771     protected => 1,
 772     permissions => {
 773         check => ['perm', '/', [ 'Sys.Audit' ]],
 774     },
 775     parameters => {
 776         additionalProperties => 0,
 777         properties => {
 778             node => get_standard_option('pve-node'),
 779         },
 780     },
 781     returns => { type => 'string' },
 782     code => sub {
 783         my ($param) = @_;
 784
 785         PVE::Ceph::Tools::check_ceph_configured();
 786
 787         my $rados = PVE::RADOS->new();
 788
 789         my $stat = $rados->mon_command({ prefix => 'osd dump' });
 790
 791         return $stat->{flags} // '';
 792     }});
 793
 794 # FIXME: Remove with PVE 7.0
 795 __PACKAGE__->register_method ({
 796     name => 'set_flag',
 797     path => 'flags/{flag}',
 798     method => 'POST',
 799     description => "Set a specific ceph flag",
 800     proxyto => 'node',
 801     protected => 1,
 802     permissions => {
 803         check => ['perm', '/', [ 'Sys.Modify' ]],
 804     },
 805     parameters => {
 806         additionalProperties => 0,
 807         properties => {
 808             node => get_standard_option('pve-node'),
 809             flag => {
 810                 description => 'The ceph flag to set',
 811                 type => 'string',
 812                 enum => $possible_flags_list,
 813             },
 814         },
 815     },
 816     returns => { type => 'null' },
 817     code => sub {
 818         my ($param) = @_;
 819
 820         PVE::Ceph::Tools::check_ceph_configured();
 821
 822         my $rados = PVE::RADOS->new();
 823
 824         $rados->mon_command({
 825             prefix => "osd set",
 826             key => $param->{flag},
 827         });
 828
 829         return undef;
 830     }});
 831
 832 __PACKAGE__->register_method ({
 833     name => 'unset_flag',
 834     path => 'flags/{flag}',
 835     method => 'DELETE',
 836     description => "Unset a ceph flag",
 837     proxyto => 'node',
 838     protected => 1,
 839     permissions => {
 840         check => ['perm', '/', [ 'Sys.Modify' ]],
 841     },
 842     parameters => {
 843         additionalProperties => 0,
 844         properties => {
 845             node => get_standard_option('pve-node'),
 846             flag => {
 847                 description => 'The ceph flag to unset',
 848                 type => 'string',
 849                 enum => $possible_flags_list,
 850             },
 851         },
 852     },
 853     returns => { type => 'null' },
 854     code => sub {
 855         my ($param) = @_;
 856
 857         PVE::Ceph::Tools::check_ceph_configured();
 858
 859         my $rados = PVE::RADOS->new();
 860
 861         $rados->mon_command({
 862             prefix => "osd unset",
 863             key => $param->{flag},
 864         });
 865
 866         return undef;
 867     }});
 868
 869 __PACKAGE__->register_method ({
 870     name => 'destroypool',
 871     path => 'pools/{name}',
 872     method => 'DELETE',
 873     description => "Destroy pool",
 874     proxyto => 'node',
 875     protected => 1,
 876     permissions => {
 877         check => ['perm', '/', [ 'Sys.Modify' ]],
 878     },
 879     parameters => {
 880         additionalProperties => 0,
 881         properties => {
 882             node => get_standard_option('pve-node'),
 883             name => {
 884                 description => "The name of the pool. It must be unique.",
 885                 type => 'string',
 886             },
 887             force => {
 888                 description => "If true, destroys pool even if in use",
 889                 type => 'boolean',
 890                 optional => 1,
 891                 default => 0,
 892             },
 893             remove_storages => {
 894                 description => "Remove all pveceph-managed storages configured for this pool",
 895                 type => 'boolean',
 896                 optional => 1,
 897                 default => 0,
 898             },
 899         },
 900     },
 901     returns => { type => 'string' },
 902     code => sub {
 903         my ($param) = @_;
 904
 905         PVE::Ceph::Tools::check_ceph_inited();
 906
 907         my $rpcenv = PVE::RPCEnvironment::get();
 908         my $user = $rpcenv->get_user();
 909         $rpcenv->check($user, '/storage', ['Datastore.Allocate'])
 910             if $param->{remove_storages};
 911
 912         my $pool = $param->{name};
 913
 914         my $worker = sub {
 915             my $storages = $get_storages->($pool);
 916
 917             # if not forced, destroy ceph pool only when no
 918             # vm disks are on it anymore
 919             if (!$param->{force}) {
 920                 my $storagecfg = PVE::Storage::config();
 921                 foreach my $storeid (keys %$storages) {
 922                     my $storage = $storages->{$storeid};
 923
 924                     # check if any vm disks are on the pool
 925                     print "checking storage '$storeid' for RBD images..\n";
 926                     my $res = PVE::Storage::vdisk_list($storagecfg, $storeid);
 927                     die "ceph pool '$pool' still in use by storage '$storeid'\n"
 928                         if @{$res->{$storeid}} != 0;
 929                 }
 930             }
 931
 932             PVE::Ceph::Tools::destroy_pool($pool);
 933
 934             if ($param->{remove_storages}) {
 935                 my $err;
 936                 foreach my $storeid (keys %$storages) {
 937                     # skip external clusters, not managed by pveceph
 938                     next if $storages->{$storeid}->{monhost};
 939                     eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
 940                     if ($@) {
 941                         warn "failed to remove storage '$storeid': $@\n";
 942                         $err = 1;
 943                     }
 944                 }
 945                 die "failed to remove (some) storages - check log and remove manually!\n"
 946                     if $err;
 947             }
 948         };
 949         return $rpcenv->fork_worker('cephdestroypool', $pool,  $user, $worker);
 950     }});
 951
 952
 953 __PACKAGE__->register_method ({
 954     name => 'crush',
 955     path => 'crush',
 956     method => 'GET',
 957     description => "Get OSD crush map",
 958     proxyto => 'node',
 959     protected => 1,
 960     permissions => {
 961         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 962     },
 963     parameters => {
 964         additionalProperties => 0,
 965         properties => {
 966             node => get_standard_option('pve-node'),
 967         },
 968     },
 969     returns => { type => 'string' },
 970     code => sub {
 971         my ($param) = @_;
 972
 973         PVE::Ceph::Tools::check_ceph_inited();
 974
 975         # this produces JSON (difficult to read for the user)
 976         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
 977
 978         my $txt = '';
 979
 980         my $mapfile = "/var/tmp/ceph-crush.map.$$";
 981         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
 982
 983         my $rados = PVE::RADOS->new();
 984
 985         eval {
 986             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
 987             file_set_contents($mapfile, $bindata);
 988             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
 989             $txt = file_get_contents($mapdata);
 990         };
 991         my $err = $@;
 992
 993         unlink $mapfile;
 994         unlink $mapdata;
 995
 996         die $err if $err;
 997
 998         return $txt;
 999     }});
1000
1001 __PACKAGE__->register_method({
1002     name => 'log',
1003     path => 'log',
1004     method => 'GET',
1005     description => "Read ceph log",
1006     proxyto => 'node',
1007     permissions => {
1008         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
1009     },
1010     protected => 1,
1011     parameters => {
1012         additionalProperties => 0,
1013         properties => {
1014             node => get_standard_option('pve-node'),
1015             start => {
1016                 type => 'integer',
1017                 minimum => 0,
1018                 optional => 1,
1019             },
1020             limit => {
1021                 type => 'integer',
1022                 minimum => 0,
1023                 optional => 1,
1024             },
1025         },
1026     },
1027     returns => {
1028         type => 'array',
1029         items => {
1030             type => "object",
1031             properties => {
1032                 n => {
1033                   description=>  "Line number",
1034                   type=> 'integer',
1035                 },
1036                 t => {
1037                   description=>  "Line text",
1038                   type => 'string',
1039                 }
1040             }
1041         }
1042     },
1043     code => sub {
1044         my ($param) = @_;
1045
1046         PVE::Ceph::Tools::check_ceph_inited();
1047
1048         my $rpcenv = PVE::RPCEnvironment::get();
1049         my $user = $rpcenv->get_user();
1050         my $node = $param->{node};
1051
1052         my $logfile = "/var/log/ceph/ceph.log";
1053         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
1054
1055         $rpcenv->set_result_attrib('total', $count);
1056
1057         return $lines;
1058     }});
1059
1060 __PACKAGE__->register_method ({
1061     name => 'rules',
1062     path => 'rules',
1063     method => 'GET',
1064     description => "List ceph rules.",
1065     proxyto => 'node',
1066     protected => 1,
1067     permissions => {
1068         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1069     },
1070     parameters => {
1071         additionalProperties => 0,
1072         properties => {
1073             node => get_standard_option('pve-node'),
1074         },
1075     },
1076     returns => {
1077         type => 'array',
1078         items => {
1079             type => "object",
1080             properties => {},
1081         },
1082         links => [ { rel => 'child', href => "{name}" } ],
1083     },
1084     code => sub {
1085         my ($param) = @_;
1086
1087         PVE::Ceph::Tools::check_ceph_inited();
1088
1089         my $rados = PVE::RADOS->new();
1090
1091         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
1092
1093         my $res = [];
1094
1095         foreach my $rule (@$rules) {
1096             push @$res, { name => $rule };
1097         }
1098
1099         return $res;
1100     }});
1101
1102 1;