PVE/API2/Ceph.pm

   1 package PVE::API2::Ceph;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use Net::IP;
   8 use UUID;
   9
  10 use PVE::Ceph::Tools;
  11 use PVE::Ceph::Services;
  12 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  13 use PVE::JSONSchema qw(get_standard_option);
  14 use PVE::Network;
  15 use PVE::RADOS;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::Storage;
  19 use PVE::Tools qw(run_command file_get_contents file_set_contents);
  20
  21 use PVE::API2::Ceph::OSD;
  22 use PVE::API2::Ceph::FS;
  23 use PVE::API2::Ceph::MDS;
  24 use PVE::API2::Ceph::MGR;
  25 use PVE::API2::Ceph::MON;
  26 use PVE::API2::Storage::Config;
  27
  28 use base qw(PVE::RESTHandler);
  29
  30 my $pve_osd_default_journal_size = 1024*5;
  31
  32 __PACKAGE__->register_method ({
  33     subclass => "PVE::API2::Ceph::OSD",
  34     path => 'osd',
  35 });
  36
  37 __PACKAGE__->register_method ({
  38     subclass => "PVE::API2::Ceph::MDS",
  39     path => 'mds',
  40 });
  41
  42 __PACKAGE__->register_method ({
  43     subclass => "PVE::API2::Ceph::MGR",
  44     path => 'mgr',
  45 });
  46
  47 __PACKAGE__->register_method ({
  48     subclass => "PVE::API2::Ceph::MON",
  49     path => 'mon',
  50 });
  51
  52 __PACKAGE__->register_method ({
  53     subclass => "PVE::API2::Ceph::FS",
  54     path => 'fs',
  55 });
  56
  57 __PACKAGE__->register_method ({
  58     name => 'index',
  59     path => '',
  60     method => 'GET',
  61     description => "Directory index.",
  62     permissions => { user => 'all' },
  63     permissions => {
  64         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  65     },
  66     parameters => {
  67         additionalProperties => 0,
  68         properties => {
  69             node => get_standard_option('pve-node'),
  70         },
  71     },
  72     returns => {
  73         type => 'array',
  74         items => {
  75             type => "object",
  76             properties => {},
  77         },
  78         links => [ { rel => 'child', href => "{name}" } ],
  79     },
  80     code => sub {
  81         my ($param) = @_;
  82
  83         my $result = [
  84             { name => 'init' },
  85             { name => 'mon' },
  86             { name => 'osd' },
  87             { name => 'pools' },
  88             { name => 'fs' },
  89             { name => 'mds' },
  90             { name => 'stop' },
  91             { name => 'start' },
  92             { name => 'restart' },
  93             { name => 'status' },
  94             { name => 'crush' },
  95             { name => 'config' },
  96             { name => 'log' },
  97             { name => 'disks' },
  98             { name => 'flags' },
  99             { name => 'flags2' },
 100             { name => 'rules' },
 101         ];
 102
 103         return $result;
 104     }});
 105
 106 __PACKAGE__->register_method ({
 107     name => 'disks',
 108     path => 'disks',
 109     method => 'GET',
 110     description => "List local disks.",
 111     proxyto => 'node',
 112     protected => 1,
 113     permissions => {
 114         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 115     },
 116     parameters => {
 117         additionalProperties => 0,
 118         properties => {
 119             node => get_standard_option('pve-node'),
 120             type => {
 121                 description => "Only list specific types of disks.",
 122                 type => 'string',
 123                 enum => ['unused', 'journal_disks'],
 124                 optional => 1,
 125             },
 126         },
 127     },
 128     returns => {
 129         type => 'array',
 130         items => {
 131             type => "object",
 132             properties => {
 133                 dev => { type => 'string' },
 134                 used => { type => 'string', optional => 1 },
 135                 gpt => { type => 'boolean' },
 136                 size => { type => 'integer' },
 137                 osdid => { type => 'integer' },
 138                 vendor =>  { type => 'string', optional => 1 },
 139                 model =>  { type => 'string', optional => 1 },
 140                 serial =>  { type => 'string', optional => 1 },
 141             },
 142         },
 143         # links => [ { rel => 'child', href => "{}" } ],
 144     },
 145     code => sub {
 146         my ($param) = @_;
 147
 148         PVE::Ceph::Tools::check_ceph_inited();
 149
 150         my $disks = PVE::Diskmanage::get_disks(undef, 1);
 151
 152         my $res = [];
 153         foreach my $dev (keys %$disks) {
 154             my $d = $disks->{$dev};
 155             if ($param->{type}) {
 156                 if ($param->{type} eq 'journal_disks') {
 157                     next if $d->{osdid} >= 0;
 158                     next if !$d->{gpt};
 159                 } elsif ($param->{type} eq 'unused') {
 160                     next if $d->{used};
 161                 } else {
 162                     die "internal error"; # should not happen
 163                 }
 164             }
 165
 166             $d->{dev} = "/dev/$dev";
 167             push @$res, $d;
 168         }
 169
 170         return $res;
 171     }});
 172
 173 __PACKAGE__->register_method ({
 174     name => 'config',
 175     path => 'config',
 176     method => 'GET',
 177     proxyto => 'node',
 178     permissions => {
 179         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 180     },
 181     description => "Get Ceph configuration.",
 182     parameters => {
 183         additionalProperties => 0,
 184         properties => {
 185             node => get_standard_option('pve-node'),
 186         },
 187     },
 188     returns => { type => 'string' },
 189     code => sub {
 190         my ($param) = @_;
 191
 192         PVE::Ceph::Tools::check_ceph_inited();
 193
 194         my $path = PVE::Ceph::Tools::get_config('pve_ceph_cfgpath');
 195         return file_get_contents($path);
 196
 197     }});
 198
 199 __PACKAGE__->register_method ({
 200     name => 'configdb',
 201     path => 'configdb',
 202     method => 'GET',
 203     proxyto => 'node',
 204     protected => 1,
 205     permissions => {
 206         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 207     },
 208     description => "Get Ceph configuration database.",
 209     parameters => {
 210         additionalProperties => 0,
 211         properties => {
 212             node => get_standard_option('pve-node'),
 213         },
 214     },
 215     returns => {
 216         type => 'array',
 217         items => {
 218             type => 'object',
 219             properties => {
 220                 section => { type => "string", },
 221                 name => { type => "string", },
 222                 value => { type => "string", },
 223                 level => { type => "string", },
 224                 'can_update_at_runtime' => { type => "boolean", },
 225                 mask => { type => "string" },
 226             },
 227         },
 228     },
 229     code => sub {
 230         my ($param) = @_;
 231
 232         PVE::Ceph::Tools::check_ceph_inited();
 233
 234         my $rados = PVE::RADOS->new();
 235         my $res = $rados->mon_command( { prefix => 'config dump', format => 'json' });
 236         foreach my $entry (@$res) {
 237             $entry->{can_update_at_runtime} = $entry->{can_update_at_runtime}? 1 : 0; # JSON::true/false -> 1/0
 238         }
 239
 240         return $res;
 241     }});
 242
 243 my $add_storage = sub {
 244     my ($pool, $storeid) = @_;
 245
 246     my $storage_params = {
 247         type => 'rbd',
 248         pool => $pool,
 249         storage => $storeid,
 250         krbd => 0,
 251         content => 'rootdir,images',
 252     };
 253
 254     PVE::API2::Storage::Config->create($storage_params);
 255 };
 256
 257 my $get_storages = sub {
 258     my ($pool) = @_;
 259
 260     my $cfg = PVE::Storage::config();
 261
 262     my $storages = $cfg->{ids};
 263     my $res = {};
 264     foreach my $storeid (keys %$storages) {
 265         my $curr = $storages->{$storeid};
 266         $res->{$storeid} = $storages->{$storeid}
 267             if $curr->{type} eq 'rbd' && $pool eq $curr->{pool};
 268     }
 269
 270     return $res;
 271 };
 272
 273 __PACKAGE__->register_method ({
 274     name => 'init',
 275     path => 'init',
 276     method => 'POST',
 277     description => "Create initial ceph default configuration and setup symlinks.",
 278     proxyto => 'node',
 279     protected => 1,
 280     permissions => {
 281         check => ['perm', '/', [ 'Sys.Modify' ]],
 282     },
 283     parameters => {
 284         additionalProperties => 0,
 285         properties => {
 286             node => get_standard_option('pve-node'),
 287             network => {
 288                 description => "Use specific network for all ceph related traffic",
 289                 type => 'string', format => 'CIDR',
 290                 optional => 1,
 291                 maxLength => 128,
 292             },
 293             'cluster-network' => {
 294                 description => "Declare a separate cluster network, OSDs will route" .
 295                     "heartbeat, object replication and recovery traffic over it",
 296                 type => 'string', format => 'CIDR',
 297                 requires => 'network',
 298                 optional => 1,
 299                 maxLength => 128,
 300             },
 301             size => {
 302                 description => 'Targeted number of replicas per object',
 303                 type => 'integer',
 304                 default => 3,
 305                 optional => 1,
 306                 minimum => 1,
 307                 maximum => 7,
 308             },
 309             min_size => {
 310                 description => 'Minimum number of available replicas per object to allow I/O',
 311                 type => 'integer',
 312                 default => 2,
 313                 optional => 1,
 314                 minimum => 1,
 315                 maximum => 7,
 316             },
 317             pg_bits => {
 318                 description => "Placement group bits, used to specify the " .
 319                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 320                     "default pg num' does not work for default pools.",
 321                 type => 'integer',
 322                 default => 6,
 323                 optional => 1,
 324                 minimum => 6,
 325                 maximum => 14,
 326             },
 327             disable_cephx => {
 328                 description => "Disable cephx authentication.\n\n" .
 329                     "WARNING: cephx is a security feature protecting against " .
 330                     "man-in-the-middle attacks. Only consider disabling cephx ".
 331                     "if your network is private!",
 332                 type => 'boolean',
 333                 optional => 1,
 334                 default => 0,
 335             },
 336         },
 337     },
 338     returns => { type => 'null' },
 339     code => sub {
 340         my ($param) = @_;
 341
 342         my $version = PVE::Ceph::Tools::get_local_version(1);
 343
 344         if (!$version || $version < 14) {
 345             die "Ceph Nautilus required - please run 'pveceph install'\n";
 346         } else {
 347             PVE::Ceph::Tools::check_ceph_installed('ceph_bin');
 348         }
 349
 350         my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 351
 352         # simply load old config if it already exists
 353         PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
 354             my $cfg = cfs_read_file('ceph.conf');
 355
 356             if (!$cfg->{global}) {
 357
 358                 my $fsid;
 359                 my $uuid;
 360
 361                 UUID::generate($uuid);
 362                 UUID::unparse($uuid, $fsid);
 363
 364                 $cfg->{global} = {
 365                     'fsid' => $fsid,
 366                     'auth cluster required' => $auth,
 367                     'auth service required' => $auth,
 368                     'auth client required' => $auth,
 369                     'osd pool default size' => $param->{size} // 3,
 370                     'osd pool default min size' => $param->{min_size} // 2,
 371                     'mon allow pool delete' => 'true',
 372                 };
 373
 374                 # this does not work for default pools
 375                 #'osd pool default pg num' => $pg_num,
 376                 #'osd pool default pgp num' => $pg_num,
 377             }
 378
 379             if ($auth eq 'cephx') {
 380                 $cfg->{client}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 381             }
 382
 383             if ($param->{pg_bits}) {
 384                 $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 385                 $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 386             }
 387
 388             if ($param->{network}) {
 389                 $cfg->{global}->{'public network'} = $param->{network};
 390                 $cfg->{global}->{'cluster network'} = $param->{network};
 391             }
 392
 393             if ($param->{'cluster-network'}) {
 394                 $cfg->{global}->{'cluster network'} = $param->{'cluster-network'};
 395             }
 396
 397             cfs_write_file('ceph.conf', $cfg);
 398
 399             if ($auth eq 'cephx') {
 400                 PVE::Ceph::Tools::get_or_create_admin_keyring();
 401             }
 402             PVE::Ceph::Tools::setup_pve_symlinks();
 403         });
 404         die $@ if $@;
 405
 406         return undef;
 407     }});
 408
 409 __PACKAGE__->register_method ({
 410     name => 'stop',
 411     path => 'stop',
 412     method => 'POST',
 413     description => "Stop ceph services.",
 414     proxyto => 'node',
 415     protected => 1,
 416     permissions => {
 417         check => ['perm', '/', [ 'Sys.Modify' ]],
 418     },
 419     parameters => {
 420         additionalProperties => 0,
 421         properties => {
 422             node => get_standard_option('pve-node'),
 423             service => {
 424                 description => 'Ceph service name.',
 425                 type => 'string',
 426                 optional => 1,
 427                 default => 'ceph.target',
 428                 pattern => '(ceph|mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
 429             },
 430         },
 431     },
 432     returns => { type => 'string' },
 433     code => sub {
 434         my ($param) = @_;
 435
 436         my $rpcenv = PVE::RPCEnvironment::get();
 437
 438         my $authuser = $rpcenv->get_user();
 439
 440         PVE::Ceph::Tools::check_ceph_inited();
 441
 442         my $cfg = cfs_read_file('ceph.conf');
 443         scalar(keys %$cfg) || die "no configuration\n";
 444
 445         my $worker = sub {
 446             my $upid = shift;
 447
 448             my $cmd = ['stop'];
 449             if ($param->{service}) {
 450                 push @$cmd, $param->{service};
 451             }
 452
 453             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 454         };
 455
 456         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
 457                                     $authuser, $worker);
 458     }});
 459
 460 __PACKAGE__->register_method ({
 461     name => 'start',
 462     path => 'start',
 463     method => 'POST',
 464     description => "Start ceph services.",
 465     proxyto => 'node',
 466     protected => 1,
 467     permissions => {
 468         check => ['perm', '/', [ 'Sys.Modify' ]],
 469     },
 470     parameters => {
 471         additionalProperties => 0,
 472         properties => {
 473             node => get_standard_option('pve-node'),
 474             service => {
 475                 description => 'Ceph service name.',
 476                 type => 'string',
 477                 optional => 1,
 478                 default => 'ceph.target',
 479                 pattern => '(ceph|mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
 480             },
 481         },
 482     },
 483     returns => { type => 'string' },
 484     code => sub {
 485         my ($param) = @_;
 486
 487         my $rpcenv = PVE::RPCEnvironment::get();
 488
 489         my $authuser = $rpcenv->get_user();
 490
 491         PVE::Ceph::Tools::check_ceph_inited();
 492
 493         my $cfg = cfs_read_file('ceph.conf');
 494         scalar(keys %$cfg) || die "no configuration\n";
 495
 496         my $worker = sub {
 497             my $upid = shift;
 498
 499             my $cmd = ['start'];
 500             if ($param->{service}) {
 501                 push @$cmd, $param->{service};
 502             }
 503
 504             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 505         };
 506
 507         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
 508                                     $authuser, $worker);
 509     }});
 510
 511 __PACKAGE__->register_method ({
 512     name => 'restart',
 513     path => 'restart',
 514     method => 'POST',
 515     description => "Restart ceph services.",
 516     proxyto => 'node',
 517     protected => 1,
 518     permissions => {
 519         check => ['perm', '/', [ 'Sys.Modify' ]],
 520     },
 521     parameters => {
 522         additionalProperties => 0,
 523         properties => {
 524             node => get_standard_option('pve-node'),
 525             service => {
 526                 description => 'Ceph service name.',
 527                 type => 'string',
 528                 optional => 1,
 529                 default => 'ceph.target',
 530                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
 531             },
 532         },
 533     },
 534     returns => { type => 'string' },
 535     code => sub {
 536         my ($param) = @_;
 537
 538         my $rpcenv = PVE::RPCEnvironment::get();
 539
 540         my $authuser = $rpcenv->get_user();
 541
 542         PVE::Ceph::Tools::check_ceph_inited();
 543
 544         my $cfg = cfs_read_file('ceph.conf');
 545         scalar(keys %$cfg) || die "no configuration\n";
 546
 547         my $worker = sub {
 548             my $upid = shift;
 549
 550             my $cmd = ['restart'];
 551             if ($param->{service}) {
 552                 push @$cmd, $param->{service};
 553             }
 554
 555             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 556         };
 557
 558         return $rpcenv->fork_worker('srvrestart', $param->{service} || 'ceph',
 559                                     $authuser, $worker);
 560     }});
 561
 562 __PACKAGE__->register_method ({
 563     name => 'status',
 564     path => 'status',
 565     method => 'GET',
 566     description => "Get ceph status.",
 567     proxyto => 'node',
 568     protected => 1,
 569     permissions => {
 570         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 571     },
 572     parameters => {
 573         additionalProperties => 0,
 574         properties => {
 575             node => get_standard_option('pve-node'),
 576         },
 577     },
 578     returns => { type => 'object' },
 579     code => sub {
 580         my ($param) = @_;
 581
 582         PVE::Ceph::Tools::check_ceph_inited();
 583
 584         my $rados = PVE::RADOS->new();
 585         my $status = $rados->mon_command({ prefix => 'status' });
 586         $status->{health} = $rados->mon_command({ prefix => 'health', detail => 'detail' });
 587         return $status;
 588     }});
 589
 590 __PACKAGE__->register_method ({
 591     name => 'lspools',
 592     path => 'pools',
 593     method => 'GET',
 594     description => "List all pools.",
 595     proxyto => 'node',
 596     protected => 1,
 597     permissions => {
 598         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 599     },
 600     parameters => {
 601         additionalProperties => 0,
 602         properties => {
 603             node => get_standard_option('pve-node'),
 604         },
 605     },
 606     returns => {
 607         type => 'array',
 608         items => {
 609             type => "object",
 610             properties => {
 611                 pool => { type => 'integer' },
 612                 pool_name => { type => 'string' },
 613                 size => { type => 'integer' },
 614             },
 615         },
 616         links => [ { rel => 'child', href => "{pool_name}" } ],
 617     },
 618     code => sub {
 619         my ($param) = @_;
 620
 621         PVE::Ceph::Tools::check_ceph_inited();
 622
 623         my $rados = PVE::RADOS->new();
 624
 625         my $stats = {};
 626         my $res = $rados->mon_command({ prefix => 'df' });
 627
 628         foreach my $d (@{$res->{pools}}) {
 629             next if !$d->{stats};
 630             next if !defined($d->{id});
 631             $stats->{$d->{id}} = $d->{stats};
 632         }
 633
 634         $res = $rados->mon_command({ prefix => 'osd dump' });
 635         my $rulestmp = $rados->mon_command({ prefix => 'osd crush rule dump'});
 636
 637         my $rules = {};
 638         for my $rule (@$rulestmp) {
 639             $rules->{$rule->{rule_id}} = $rule->{rule_name};
 640         }
 641
 642         my $data = [];
 643         foreach my $e (@{$res->{pools}}) {
 644             my $d = {};
 645             foreach my $attr (qw(pool pool_name size min_size pg_num crush_rule)) {
 646                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
 647             }
 648
 649             if (defined($d->{crush_rule}) && defined($rules->{$d->{crush_rule}})) {
 650                 $d->{crush_rule_name} = $rules->{$d->{crush_rule}};
 651             }
 652
 653             if (my $s = $stats->{$d->{pool}}) {
 654                 $d->{bytes_used} = $s->{bytes_used};
 655                 $d->{percent_used} = $s->{percent_used};
 656             }
 657             push @$data, $d;
 658         }
 659
 660
 661         return $data;
 662     }});
 663
 664 __PACKAGE__->register_method ({
 665     name => 'createpool',
 666     path => 'pools',
 667     method => 'POST',
 668     description => "Create POOL",
 669     proxyto => 'node',
 670     protected => 1,
 671     permissions => {
 672         check => ['perm', '/', [ 'Sys.Modify' ]],
 673     },
 674     parameters => {
 675         additionalProperties => 0,
 676         properties => {
 677             node => get_standard_option('pve-node'),
 678             name => {
 679                 description => "The name of the pool. It must be unique.",
 680                 type => 'string',
 681             },
 682             size => {
 683                 description => 'Number of replicas per object',
 684                 type => 'integer',
 685                 default => 3,
 686                 optional => 1,
 687                 minimum => 1,
 688                 maximum => 7,
 689             },
 690             min_size => {
 691                 description => 'Minimum number of replicas per object',
 692                 type => 'integer',
 693                 default => 2,
 694                 optional => 1,
 695                 minimum => 1,
 696                 maximum => 7,
 697             },
 698             pg_num => {
 699                 description => "Number of placement groups.",
 700                 type => 'integer',
 701                 default => 128,
 702                 optional => 1,
 703                 minimum => 8,
 704                 maximum => 32768,
 705             },
 706             crush_rule => {
 707                 description => "The rule to use for mapping object placement in the cluster.",
 708                 type => 'string',
 709                 optional => 1,
 710             },
 711             application => {
 712                 description => "The application of the pool, 'rbd' by default.",
 713                 type => 'string',
 714                 enum => ['rbd', 'cephfs', 'rgw'],
 715                 optional => 1,
 716             },
 717             add_storages => {
 718                 description => "Configure VM and CT storage using the new pool.",
 719                 type => 'boolean',
 720                 optional => 1,
 721             },
 722         },
 723     },
 724     returns => { type => 'string' },
 725     code => sub {
 726         my ($param) = @_;
 727
 728         PVE::Cluster::check_cfs_quorum();
 729         PVE::Ceph::Tools::check_ceph_configured();
 730
 731         my $pool = $param->{name};
 732         my $rpcenv = PVE::RPCEnvironment::get();
 733         my $user = $rpcenv->get_user();
 734
 735         if ($param->{add_storages}) {
 736             $rpcenv->check($user, '/storage', ['Datastore.Allocate']);
 737             die "pool name contains characters which are illegal for storage naming\n"
 738                 if !PVE::JSONSchema::parse_storage_id($pool);
 739         }
 740
 741         my $pg_num = $param->{pg_num} || 128;
 742         my $size = $param->{size} || 3;
 743         my $min_size = $param->{min_size} || 2;
 744         my $application = $param->{application} // 'rbd';
 745
 746         my $worker = sub {
 747
 748             PVE::Ceph::Tools::create_pool($pool, $param);
 749
 750             if ($param->{add_storages}) {
 751                 my $err;
 752                 eval { $add_storage->($pool, "${pool}"); };
 753                 if ($@) {
 754                     warn "failed to add storage: $@";
 755                     $err = 1;
 756                 }
 757                 die "adding storage for pool '$pool' failed, check log and add manually!\n"
 758                     if $err;
 759             }
 760         };
 761
 762         return $rpcenv->fork_worker('cephcreatepool', $pool,  $user, $worker);
 763     }});
 764
 765 my $possible_flags = {
 766     pause => {
 767         description => 'Pauses read and writes.',
 768         type => 'boolean',
 769         optional=> 1,
 770     },
 771     noup => {
 772         description => 'OSDs are not allowed to start.',
 773         type => 'boolean',
 774         optional=> 1,
 775     },
 776     nodown => {
 777         description => 'OSD failure reports are being ignored, such that the monitors will not mark OSDs down.',
 778         type => 'boolean',
 779         optional=> 1,
 780     },
 781     noout => {
 782         description => 'OSDs will not automatically be marked out after the configured interval.',
 783         type => 'boolean',
 784         optional=> 1,
 785     },
 786     noin => {
 787         description => 'OSDs that were previously marked out will not be marked back in when they start.',
 788         type => 'boolean',
 789         optional=> 1,
 790     },
 791     nobackfill => {
 792         description => 'Backfilling of PGs is suspended.',
 793         type => 'boolean',
 794         optional=> 1,
 795     },
 796     norebalance => {
 797         description => 'Rebalancing of PGs is suspended.',
 798         type => 'boolean',
 799         optional=> 1,
 800     },
 801     norecover => {
 802         description => 'Recovery of PGs is suspended.',
 803         type => 'boolean',
 804         optional=> 1,
 805     },
 806     noscrub => {
 807         description => 'Scrubbing is disabled.',
 808         type => 'boolean',
 809         optional=> 1,
 810     },
 811     'nodeep-scrub' => {
 812         description => 'Deep Scrubbing is disabled.',
 813         type => 'boolean',
 814         optional=> 1,
 815     },
 816     notieragent => {
 817         description => 'Cache tiering activity is suspended.',
 818         type => 'boolean',
 819         optional=> 1,
 820     },
 821 };
 822
 823 # the 'pause' flag gets always set to both 'pauserd' and 'pausewr'
 824 # so decide that the 'pause' flag is set if we detect 'pauserd'
 825 my $flagmap = {
 826     'pause' => 'pauserd',
 827 };
 828
 829 __PACKAGE__->register_method ({
 830     name => 'flags2',
 831     path => 'flags2',
 832     method => 'GET',
 833     description => "get the status of all ceph flags",
 834     proxyto => 'node',
 835     protected => 1,
 836     permissions => {
 837         check => ['perm', '/', [ 'Sys.Audit' ]],
 838     },
 839     parameters => {
 840         additionalProperties => 0,
 841         properties => {
 842             node => get_standard_option('pve-node'),
 843         },
 844     },
 845     returns => { type => 'array' },
 846     code => sub {
 847         my ($param) = @_;
 848
 849         PVE::Ceph::Tools::check_ceph_configured();
 850
 851         my $rados = PVE::RADOS->new();
 852
 853         my $stat = $rados->mon_command({ prefix => 'osd dump' });
 854         my $setflags = {};
 855         foreach my $flag (PVE::Tools::split_list($stat->{flags} // '')) {
 856             $setflags->{$flag} = 1;
 857         }
 858
 859         my $res = [];
 860         foreach my $flag (sort keys %$possible_flags) {
 861             my $el = {
 862                 name => $flag,
 863                 description => $possible_flags->{$flag}->{description},
 864                 value => 0,
 865             };
 866
 867             my $realflag = $flagmap->{$flag} // $flag;
 868             if ($setflags->{$realflag}) {
 869                 $el->{value} = 1;
 870             }
 871
 872             push @$res, $el;
 873         }
 874
 875         return $res;
 876     }});
 877
 878 __PACKAGE__->register_method ({
 879     name => 'get_flags',
 880     path => 'flags',
 881     method => 'GET',
 882     description => "get all set ceph flags",
 883     proxyto => 'node',
 884     protected => 1,
 885     permissions => {
 886         check => ['perm', '/', [ 'Sys.Audit' ]],
 887     },
 888     parameters => {
 889         additionalProperties => 0,
 890         properties => {
 891             node => get_standard_option('pve-node'),
 892         },
 893     },
 894     returns => { type => 'string' },
 895     code => sub {
 896         my ($param) = @_;
 897
 898         PVE::Ceph::Tools::check_ceph_configured();
 899
 900         my $rados = PVE::RADOS->new();
 901
 902         my $stat = $rados->mon_command({ prefix => 'osd dump' });
 903
 904         return $stat->{flags} // '';
 905     }});
 906
 907 __PACKAGE__->register_method ({
 908     name => 'set_flags',
 909     path => 'flags',
 910     method => 'PUT',
 911     description => "Set/Unset multiple ceph flags at once.",
 912     proxyto => 'node',
 913     protected => 1,
 914     permissions => {
 915         check => ['perm', '/', [ 'Sys.Modify' ]],
 916     },
 917     parameters => {
 918         additionalProperties => 0,
 919         properties => {
 920             node => get_standard_option('pve-node'),
 921             %$possible_flags,
 922         },
 923     },
 924     returns => { type => 'string' },
 925     code => sub {
 926         my ($param) = @_;
 927
 928         my $rpcenv = PVE::RPCEnvironment::get();
 929         my $user = $rpcenv->get_user();
 930         PVE::Ceph::Tools::check_ceph_configured();
 931
 932         my $rados = PVE::RADOS->new();
 933
 934         my $worker = sub {
 935             # reopen rados object
 936             my $rados = PVE::RADOS->new();
 937             my $stat = $rados->mon_command({ prefix => 'osd dump' });
 938             my $setflags = $stat->{flags} // '';
 939             $setflags = { map { $_ => 1 } PVE::Tools::split_list($setflags) };
 940             my $errors = 0;
 941             foreach my $flag (sort keys %$possible_flags) {
 942                 next if !defined($param->{$flag});
 943                 my $val = $param->{$flag};
 944                 my $realflag = $flagmap->{$flag} // $flag;
 945                 next if !$val == !$setflags->{$realflag}; # we do not set/unset flags to the same state
 946
 947                 my $prefix = $val ? 'set' : 'unset';
 948                 eval {
 949                     print "$prefix $flag\n";
 950                     $rados->mon_command({ prefix => "osd $prefix", key => $flag, });
 951                 };
 952                 if (my $err = $@) {
 953                     warn "error with $flag: '$err'\n";
 954                     $errors++;
 955                 }
 956             }
 957
 958             if ($errors) {
 959                 die "could not set/unset $errors flags\n";
 960             }
 961         };
 962
 963         return $rpcenv->fork_worker('cephsetflags', undef,  $user, $worker);
 964     }});
 965
 966 __PACKAGE__->register_method ({
 967     name => 'set_flag',
 968     path => 'flags/{flag}',
 969     method => 'POST',
 970     description => "Set a ceph flag",
 971     proxyto => 'node',
 972     protected => 1,
 973     permissions => {
 974         check => ['perm', '/', [ 'Sys.Modify' ]],
 975     },
 976     parameters => {
 977         additionalProperties => 0,
 978         properties => {
 979             node => get_standard_option('pve-node'),
 980             flag => {
 981                 description => 'The ceph flag to set',
 982                 type => 'string',
 983                 enum => [sort keys %$possible_flags],
 984             },
 985         },
 986     },
 987     returns => { type => 'null' },
 988     code => sub {
 989         my ($param) = @_;
 990
 991         PVE::Ceph::Tools::check_ceph_configured();
 992
 993         my $set = $param->{set} // !$param->{unset};
 994         my $rados = PVE::RADOS->new();
 995
 996         $rados->mon_command({
 997             prefix => "osd set",
 998             key => $param->{flag},
 999         });
1000
1001         return undef;
1002     }});
1003
1004 __PACKAGE__->register_method ({
1005     name => 'unset_flag',
1006     path => 'flags/{flag}',
1007     method => 'DELETE',
1008     description => "Unset a ceph flag",
1009     proxyto => 'node',
1010     protected => 1,
1011     permissions => {
1012         check => ['perm', '/', [ 'Sys.Modify' ]],
1013     },
1014     parameters => {
1015         additionalProperties => 0,
1016         properties => {
1017             node => get_standard_option('pve-node'),
1018             flag => {
1019                 description => 'The ceph flag to unset',
1020                 type => 'string',
1021                 enum => [sort keys %$possible_flags],
1022             },
1023         },
1024     },
1025     returns => { type => 'null' },
1026     code => sub {
1027         my ($param) = @_;
1028
1029         PVE::Ceph::Tools::check_ceph_configured();
1030
1031         my $set = $param->{set} // !$param->{unset};
1032         my $rados = PVE::RADOS->new();
1033
1034         $rados->mon_command({
1035             prefix => "osd unset",
1036             key => $param->{flag},
1037         });
1038
1039         return undef;
1040     }});
1041
1042 __PACKAGE__->register_method ({
1043     name => 'destroypool',
1044     path => 'pools/{name}',
1045     method => 'DELETE',
1046     description => "Destroy pool",
1047     proxyto => 'node',
1048     protected => 1,
1049     permissions => {
1050         check => ['perm', '/', [ 'Sys.Modify' ]],
1051     },
1052     parameters => {
1053         additionalProperties => 0,
1054         properties => {
1055             node => get_standard_option('pve-node'),
1056             name => {
1057                 description => "The name of the pool. It must be unique.",
1058                 type => 'string',
1059             },
1060             force => {
1061                 description => "If true, destroys pool even if in use",
1062                 type => 'boolean',
1063                 optional => 1,
1064                 default => 0,
1065             },
1066             remove_storages => {
1067                 description => "Remove all pveceph-managed storages configured for this pool",
1068                 type => 'boolean',
1069                 optional => 1,
1070                 default => 0,
1071             },
1072         },
1073     },
1074     returns => { type => 'string' },
1075     code => sub {
1076         my ($param) = @_;
1077
1078         PVE::Ceph::Tools::check_ceph_inited();
1079
1080         my $rpcenv = PVE::RPCEnvironment::get();
1081         my $user = $rpcenv->get_user();
1082         $rpcenv->check($user, '/storage', ['Datastore.Allocate'])
1083             if $param->{remove_storages};
1084
1085         my $pool = $param->{name};
1086
1087         my $worker = sub {
1088             my $storages = $get_storages->($pool);
1089
1090             # if not forced, destroy ceph pool only when no
1091             # vm disks are on it anymore
1092             if (!$param->{force}) {
1093                 my $storagecfg = PVE::Storage::config();
1094                 foreach my $storeid (keys %$storages) {
1095                     my $storage = $storages->{$storeid};
1096
1097                     # check if any vm disks are on the pool
1098                     print "checking storage '$storeid' for RBD images..\n";
1099                     my $res = PVE::Storage::vdisk_list($storagecfg, $storeid);
1100                     die "ceph pool '$pool' still in use by storage '$storeid'\n"
1101                         if @{$res->{$storeid}} != 0;
1102                 }
1103             }
1104
1105             PVE::Ceph::Tools::destroy_pool($pool);
1106
1107             if ($param->{remove_storages}) {
1108                 my $err;
1109                 foreach my $storeid (keys %$storages) {
1110                     # skip external clusters, not managed by pveceph
1111                     next if $storages->{$storeid}->{monhost};
1112                     eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
1113                     if ($@) {
1114                         warn "failed to remove storage '$storeid': $@\n";
1115                         $err = 1;
1116                     }
1117                 }
1118                 die "failed to remove (some) storages - check log and remove manually!\n"
1119                     if $err;
1120             }
1121         };
1122         return $rpcenv->fork_worker('cephdestroypool', $pool,  $user, $worker);
1123     }});
1124
1125
1126 __PACKAGE__->register_method ({
1127     name => 'crush',
1128     path => 'crush',
1129     method => 'GET',
1130     description => "Get OSD crush map",
1131     proxyto => 'node',
1132     protected => 1,
1133     permissions => {
1134         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1135     },
1136     parameters => {
1137         additionalProperties => 0,
1138         properties => {
1139             node => get_standard_option('pve-node'),
1140         },
1141     },
1142     returns => { type => 'string' },
1143     code => sub {
1144         my ($param) = @_;
1145
1146         PVE::Ceph::Tools::check_ceph_inited();
1147
1148         # this produces JSON (difficult to read for the user)
1149         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
1150
1151         my $txt = '';
1152
1153         my $mapfile = "/var/tmp/ceph-crush.map.$$";
1154         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
1155
1156         my $rados = PVE::RADOS->new();
1157
1158         eval {
1159             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
1160             file_set_contents($mapfile, $bindata);
1161             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
1162             $txt = file_get_contents($mapdata);
1163         };
1164         my $err = $@;
1165
1166         unlink $mapfile;
1167         unlink $mapdata;
1168
1169         die $err if $err;
1170
1171         return $txt;
1172     }});
1173
1174 __PACKAGE__->register_method({
1175     name => 'log',
1176     path => 'log',
1177     method => 'GET',
1178     description => "Read ceph log",
1179     proxyto => 'node',
1180     permissions => {
1181         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
1182     },
1183     protected => 1,
1184     parameters => {
1185         additionalProperties => 0,
1186         properties => {
1187             node => get_standard_option('pve-node'),
1188             start => {
1189                 type => 'integer',
1190                 minimum => 0,
1191                 optional => 1,
1192             },
1193             limit => {
1194                 type => 'integer',
1195                 minimum => 0,
1196                 optional => 1,
1197             },
1198         },
1199     },
1200     returns => {
1201         type => 'array',
1202         items => {
1203             type => "object",
1204             properties => {
1205                 n => {
1206                   description=>  "Line number",
1207                   type=> 'integer',
1208                 },
1209                 t => {
1210                   description=>  "Line text",
1211                   type => 'string',
1212                 }
1213             }
1214         }
1215     },
1216     code => sub {
1217         my ($param) = @_;
1218
1219         PVE::Ceph::Tools::check_ceph_inited();
1220
1221         my $rpcenv = PVE::RPCEnvironment::get();
1222         my $user = $rpcenv->get_user();
1223         my $node = $param->{node};
1224
1225         my $logfile = "/var/log/ceph/ceph.log";
1226         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
1227
1228         $rpcenv->set_result_attrib('total', $count);
1229
1230         return $lines;
1231     }});
1232
1233 __PACKAGE__->register_method ({
1234     name => 'rules',
1235     path => 'rules',
1236     method => 'GET',
1237     description => "List ceph rules.",
1238     proxyto => 'node',
1239     protected => 1,
1240     permissions => {
1241         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1242     },
1243     parameters => {
1244         additionalProperties => 0,
1245         properties => {
1246             node => get_standard_option('pve-node'),
1247         },
1248     },
1249     returns => {
1250         type => 'array',
1251         items => {
1252             type => "object",
1253             properties => {},
1254         },
1255         links => [ { rel => 'child', href => "{name}" } ],
1256     },
1257     code => sub {
1258         my ($param) = @_;
1259
1260         PVE::Ceph::Tools::check_ceph_inited();
1261
1262         my $rados = PVE::RADOS->new();
1263
1264         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
1265
1266         my $res = [];
1267
1268         foreach my $rule (@$rules) {
1269             push @$res, { name => $rule };
1270         }
1271
1272         return $res;
1273     }});
1274
1275 1;