PVE/API2/Ceph.pm

   1 package PVE::API2::Ceph;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use Net::IP;
   8 use UUID;
   9
  10 use PVE::Ceph::Tools;
  11 use PVE::Ceph::Services;
  12 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  13 use PVE::JSONSchema qw(get_standard_option);
  14 use PVE::Network;
  15 use PVE::RADOS;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::Storage;
  19 use PVE::Tools qw(run_command file_get_contents file_set_contents);
  20
  21 use PVE::API2::Ceph::OSD;
  22 use PVE::API2::Ceph::FS;
  23 use PVE::API2::Ceph::MDS;
  24 use PVE::API2::Ceph::MGR;
  25 use PVE::API2::Ceph::MON;
  26 use PVE::API2::Storage::Config;
  27
  28 use base qw(PVE::RESTHandler);
  29
  30 my $pve_osd_default_journal_size = 1024*5;
  31
  32 __PACKAGE__->register_method ({
  33     subclass => "PVE::API2::Ceph::OSD",
  34     path => 'osd',
  35 });
  36
  37 __PACKAGE__->register_method ({
  38     subclass => "PVE::API2::Ceph::MDS",
  39     path => 'mds',
  40 });
  41
  42 __PACKAGE__->register_method ({
  43     subclass => "PVE::API2::Ceph::MGR",
  44     path => 'mgr',
  45 });
  46
  47 __PACKAGE__->register_method ({
  48     subclass => "PVE::API2::Ceph::MON",
  49     path => 'mon',
  50 });
  51
  52 __PACKAGE__->register_method ({
  53     subclass => "PVE::API2::Ceph::FS",
  54     path => 'fs',
  55 });
  56
  57 __PACKAGE__->register_method ({
  58     name => 'index',
  59     path => '',
  60     method => 'GET',
  61     description => "Directory index.",
  62     permissions => { user => 'all' },
  63     permissions => {
  64         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  65     },
  66     parameters => {
  67         additionalProperties => 0,
  68         properties => {
  69             node => get_standard_option('pve-node'),
  70         },
  71     },
  72     returns => {
  73         type => 'array',
  74         items => {
  75             type => "object",
  76             properties => {},
  77         },
  78         links => [ { rel => 'child', href => "{name}" } ],
  79     },
  80     code => sub {
  81         my ($param) = @_;
  82
  83         my $result = [
  84             { name => 'init' },
  85             { name => 'mon' },
  86             { name => 'osd' },
  87             { name => 'pools' },
  88             { name => 'fs' },
  89             { name => 'mds' },
  90             { name => 'stop' },
  91             { name => 'start' },
  92             { name => 'restart' },
  93             { name => 'status' },
  94             { name => 'crush' },
  95             { name => 'config' },
  96             { name => 'log' },
  97             { name => 'disks' },
  98             { name => 'flags' },
  99             { name => 'rules' },
 100         ];
 101
 102         return $result;
 103     }});
 104
 105 __PACKAGE__->register_method ({
 106     name => 'disks',
 107     path => 'disks',
 108     method => 'GET',
 109     description => "List local disks.",
 110     proxyto => 'node',
 111     protected => 1,
 112     permissions => {
 113         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 114     },
 115     parameters => {
 116         additionalProperties => 0,
 117         properties => {
 118             node => get_standard_option('pve-node'),
 119             type => {
 120                 description => "Only list specific types of disks.",
 121                 type => 'string',
 122                 enum => ['unused', 'journal_disks'],
 123                 optional => 1,
 124             },
 125         },
 126     },
 127     returns => {
 128         type => 'array',
 129         items => {
 130             type => "object",
 131             properties => {
 132                 dev => { type => 'string' },
 133                 used => { type => 'string', optional => 1 },
 134                 gpt => { type => 'boolean' },
 135                 size => { type => 'integer' },
 136                 osdid => { type => 'integer' },
 137                 vendor =>  { type => 'string', optional => 1 },
 138                 model =>  { type => 'string', optional => 1 },
 139                 serial =>  { type => 'string', optional => 1 },
 140             },
 141         },
 142         # links => [ { rel => 'child', href => "{}" } ],
 143     },
 144     code => sub {
 145         my ($param) = @_;
 146
 147         PVE::Ceph::Tools::check_ceph_inited();
 148
 149         my $disks = PVE::Diskmanage::get_disks(undef, 1);
 150
 151         my $res = [];
 152         foreach my $dev (keys %$disks) {
 153             my $d = $disks->{$dev};
 154             if ($param->{type}) {
 155                 if ($param->{type} eq 'journal_disks') {
 156                     next if $d->{osdid} >= 0;
 157                     next if !$d->{gpt};
 158                 } elsif ($param->{type} eq 'unused') {
 159                     next if $d->{used};
 160                 } else {
 161                     die "internal error"; # should not happen
 162                 }
 163             }
 164
 165             $d->{dev} = "/dev/$dev";
 166             push @$res, $d;
 167         }
 168
 169         return $res;
 170     }});
 171
 172 __PACKAGE__->register_method ({
 173     name => 'config',
 174     path => 'config',
 175     method => 'GET',
 176     proxyto => 'node',
 177     permissions => {
 178         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 179     },
 180     description => "Get Ceph configuration.",
 181     parameters => {
 182         additionalProperties => 0,
 183         properties => {
 184             node => get_standard_option('pve-node'),
 185         },
 186     },
 187     returns => { type => 'string' },
 188     code => sub {
 189         my ($param) = @_;
 190
 191         PVE::Ceph::Tools::check_ceph_inited();
 192
 193         my $path = PVE::Ceph::Tools::get_config('pve_ceph_cfgpath');
 194         return file_get_contents($path);
 195
 196     }});
 197
 198 __PACKAGE__->register_method ({
 199     name => 'configdb',
 200     path => 'configdb',
 201     method => 'GET',
 202     proxyto => 'node',
 203     protected => 1,
 204     permissions => {
 205         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 206     },
 207     description => "Get Ceph configuration database.",
 208     parameters => {
 209         additionalProperties => 0,
 210         properties => {
 211             node => get_standard_option('pve-node'),
 212         },
 213     },
 214     returns => {
 215         type => 'array',
 216         items => {
 217             type => 'object',
 218             properties => {
 219                 section => { type => "string", },
 220                 name => { type => "string", },
 221                 value => { type => "string", },
 222                 level => { type => "string", },
 223                 'can_update_at_runtime' => { type => "boolean", },
 224                 mask => { type => "string" },
 225             },
 226         },
 227     },
 228     code => sub {
 229         my ($param) = @_;
 230
 231         PVE::Ceph::Tools::check_ceph_inited();
 232
 233         my $rados = PVE::RADOS->new();
 234         my $res = $rados->mon_command( { prefix => 'config dump', format => 'json' });
 235         foreach my $entry (@$res) {
 236             $entry->{can_update_at_runtime} = $entry->{can_update_at_runtime}? 1 : 0; # JSON::true/false -> 1/0
 237         }
 238
 239         return $res;
 240     }});
 241
 242 my $add_storage = sub {
 243     my ($pool, $storeid) = @_;
 244
 245     my $storage_params = {
 246         type => 'rbd',
 247         pool => $pool,
 248         storage => $storeid,
 249         krbd => 0,
 250         content => 'rootdir,images',
 251     };
 252
 253     PVE::API2::Storage::Config->create($storage_params);
 254 };
 255
 256 my $get_storages = sub {
 257     my ($pool) = @_;
 258
 259     my $cfg = PVE::Storage::config();
 260
 261     my $storages = $cfg->{ids};
 262     my $res = {};
 263     foreach my $storeid (keys %$storages) {
 264         my $curr = $storages->{$storeid};
 265         $res->{$storeid} = $storages->{$storeid}
 266             if $curr->{type} eq 'rbd' && $pool eq $curr->{pool};
 267     }
 268
 269     return $res;
 270 };
 271
 272 __PACKAGE__->register_method ({
 273     name => 'init',
 274     path => 'init',
 275     method => 'POST',
 276     description => "Create initial ceph default configuration and setup symlinks.",
 277     proxyto => 'node',
 278     protected => 1,
 279     permissions => {
 280         check => ['perm', '/', [ 'Sys.Modify' ]],
 281     },
 282     parameters => {
 283         additionalProperties => 0,
 284         properties => {
 285             node => get_standard_option('pve-node'),
 286             network => {
 287                 description => "Use specific network for all ceph related traffic",
 288                 type => 'string', format => 'CIDR',
 289                 optional => 1,
 290                 maxLength => 128,
 291             },
 292             'cluster-network' => {
 293                 description => "Declare a separate cluster network, OSDs will route" .
 294                     "heartbeat, object replication and recovery traffic over it",
 295                 type => 'string', format => 'CIDR',
 296                 requires => 'network',
 297                 optional => 1,
 298                 maxLength => 128,
 299             },
 300             size => {
 301                 description => 'Targeted number of replicas per object',
 302                 type => 'integer',
 303                 default => 3,
 304                 optional => 1,
 305                 minimum => 1,
 306                 maximum => 7,
 307             },
 308             min_size => {
 309                 description => 'Minimum number of available replicas per object to allow I/O',
 310                 type => 'integer',
 311                 default => 2,
 312                 optional => 1,
 313                 minimum => 1,
 314                 maximum => 7,
 315             },
 316             pg_bits => {
 317                 description => "Placement group bits, used to specify the " .
 318                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 319                     "default pg num' does not work for default pools.",
 320                 type => 'integer',
 321                 default => 6,
 322                 optional => 1,
 323                 minimum => 6,
 324                 maximum => 14,
 325             },
 326             disable_cephx => {
 327                 description => "Disable cephx authentication.\n\n" .
 328                     "WARNING: cephx is a security feature protecting against " .
 329                     "man-in-the-middle attacks. Only consider disabling cephx ".
 330                     "if your network is private!",
 331                 type => 'boolean',
 332                 optional => 1,
 333                 default => 0,
 334             },
 335         },
 336     },
 337     returns => { type => 'null' },
 338     code => sub {
 339         my ($param) = @_;
 340
 341         my $version = PVE::Ceph::Tools::get_local_version(1);
 342
 343         if (!$version || $version < 14) {
 344             die "Ceph Nautilus required - please run 'pveceph install'\n";
 345         } else {
 346             PVE::Ceph::Tools::check_ceph_installed('ceph_bin');
 347         }
 348
 349         my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 350
 351         # simply load old config if it already exists
 352         PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
 353             my $cfg = cfs_read_file('ceph.conf');
 354
 355             if (!$cfg->{global}) {
 356
 357                 my $fsid;
 358                 my $uuid;
 359
 360                 UUID::generate($uuid);
 361                 UUID::unparse($uuid, $fsid);
 362
 363                 $cfg->{global} = {
 364                     'fsid' => $fsid,
 365                     'auth cluster required' => $auth,
 366                     'auth service required' => $auth,
 367                     'auth client required' => $auth,
 368                     'osd pool default size' => $param->{size} // 3,
 369                     'osd pool default min size' => $param->{min_size} // 2,
 370                     'mon allow pool delete' => 'true',
 371                 };
 372
 373                 # this does not work for default pools
 374                 #'osd pool default pg num' => $pg_num,
 375                 #'osd pool default pgp num' => $pg_num,
 376             }
 377
 378             if ($auth eq 'cephx') {
 379                 $cfg->{client}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 380             }
 381
 382             if ($param->{pg_bits}) {
 383                 $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 384                 $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 385             }
 386
 387             if ($param->{network}) {
 388                 $cfg->{global}->{'public network'} = $param->{network};
 389                 $cfg->{global}->{'cluster network'} = $param->{network};
 390             }
 391
 392             if ($param->{'cluster-network'}) {
 393                 $cfg->{global}->{'cluster network'} = $param->{'cluster-network'};
 394             }
 395
 396             cfs_write_file('ceph.conf', $cfg);
 397
 398             if ($auth eq 'cephx') {
 399                 PVE::Ceph::Tools::get_or_create_admin_keyring();
 400             }
 401             PVE::Ceph::Tools::setup_pve_symlinks();
 402         });
 403         die $@ if $@;
 404
 405         return undef;
 406     }});
 407
 408 __PACKAGE__->register_method ({
 409     name => 'stop',
 410     path => 'stop',
 411     method => 'POST',
 412     description => "Stop ceph services.",
 413     proxyto => 'node',
 414     protected => 1,
 415     permissions => {
 416         check => ['perm', '/', [ 'Sys.Modify' ]],
 417     },
 418     parameters => {
 419         additionalProperties => 0,
 420         properties => {
 421             node => get_standard_option('pve-node'),
 422             service => {
 423                 description => 'Ceph service name.',
 424                 type => 'string',
 425                 optional => 1,
 426                 default => 'ceph.target',
 427                 pattern => '(ceph|mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
 428             },
 429         },
 430     },
 431     returns => { type => 'string' },
 432     code => sub {
 433         my ($param) = @_;
 434
 435         my $rpcenv = PVE::RPCEnvironment::get();
 436
 437         my $authuser = $rpcenv->get_user();
 438
 439         PVE::Ceph::Tools::check_ceph_inited();
 440
 441         my $cfg = cfs_read_file('ceph.conf');
 442         scalar(keys %$cfg) || die "no configuration\n";
 443
 444         my $worker = sub {
 445             my $upid = shift;
 446
 447             my $cmd = ['stop'];
 448             if ($param->{service}) {
 449                 push @$cmd, $param->{service};
 450             }
 451
 452             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 453         };
 454
 455         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
 456                                     $authuser, $worker);
 457     }});
 458
 459 __PACKAGE__->register_method ({
 460     name => 'start',
 461     path => 'start',
 462     method => 'POST',
 463     description => "Start ceph services.",
 464     proxyto => 'node',
 465     protected => 1,
 466     permissions => {
 467         check => ['perm', '/', [ 'Sys.Modify' ]],
 468     },
 469     parameters => {
 470         additionalProperties => 0,
 471         properties => {
 472             node => get_standard_option('pve-node'),
 473             service => {
 474                 description => 'Ceph service name.',
 475                 type => 'string',
 476                 optional => 1,
 477                 default => 'ceph.target',
 478                 pattern => '(ceph|mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
 479             },
 480         },
 481     },
 482     returns => { type => 'string' },
 483     code => sub {
 484         my ($param) = @_;
 485
 486         my $rpcenv = PVE::RPCEnvironment::get();
 487
 488         my $authuser = $rpcenv->get_user();
 489
 490         PVE::Ceph::Tools::check_ceph_inited();
 491
 492         my $cfg = cfs_read_file('ceph.conf');
 493         scalar(keys %$cfg) || die "no configuration\n";
 494
 495         my $worker = sub {
 496             my $upid = shift;
 497
 498             my $cmd = ['start'];
 499             if ($param->{service}) {
 500                 push @$cmd, $param->{service};
 501             }
 502
 503             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 504         };
 505
 506         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
 507                                     $authuser, $worker);
 508     }});
 509
 510 __PACKAGE__->register_method ({
 511     name => 'restart',
 512     path => 'restart',
 513     method => 'POST',
 514     description => "Restart ceph services.",
 515     proxyto => 'node',
 516     protected => 1,
 517     permissions => {
 518         check => ['perm', '/', [ 'Sys.Modify' ]],
 519     },
 520     parameters => {
 521         additionalProperties => 0,
 522         properties => {
 523             node => get_standard_option('pve-node'),
 524             service => {
 525                 description => 'Ceph service name.',
 526                 type => 'string',
 527                 optional => 1,
 528                 default => 'ceph.target',
 529                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
 530             },
 531         },
 532     },
 533     returns => { type => 'string' },
 534     code => sub {
 535         my ($param) = @_;
 536
 537         my $rpcenv = PVE::RPCEnvironment::get();
 538
 539         my $authuser = $rpcenv->get_user();
 540
 541         PVE::Ceph::Tools::check_ceph_inited();
 542
 543         my $cfg = cfs_read_file('ceph.conf');
 544         scalar(keys %$cfg) || die "no configuration\n";
 545
 546         my $worker = sub {
 547             my $upid = shift;
 548
 549             my $cmd = ['restart'];
 550             if ($param->{service}) {
 551                 push @$cmd, $param->{service};
 552             }
 553
 554             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 555         };
 556
 557         return $rpcenv->fork_worker('srvrestart', $param->{service} || 'ceph',
 558                                     $authuser, $worker);
 559     }});
 560
 561 __PACKAGE__->register_method ({
 562     name => 'status',
 563     path => 'status',
 564     method => 'GET',
 565     description => "Get ceph status.",
 566     proxyto => 'node',
 567     protected => 1,
 568     permissions => {
 569         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 570     },
 571     parameters => {
 572         additionalProperties => 0,
 573         properties => {
 574             node => get_standard_option('pve-node'),
 575         },
 576     },
 577     returns => { type => 'object' },
 578     code => sub {
 579         my ($param) = @_;
 580
 581         PVE::Ceph::Tools::check_ceph_inited();
 582
 583         my $rados = PVE::RADOS->new();
 584         my $status = $rados->mon_command({ prefix => 'status' });
 585         $status->{health} = $rados->mon_command({ prefix => 'health', detail => 'detail' });
 586         return $status;
 587     }});
 588
 589 __PACKAGE__->register_method ({
 590     name => 'lspools',
 591     path => 'pools',
 592     method => 'GET',
 593     description => "List all pools.",
 594     proxyto => 'node',
 595     protected => 1,
 596     permissions => {
 597         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 598     },
 599     parameters => {
 600         additionalProperties => 0,
 601         properties => {
 602             node => get_standard_option('pve-node'),
 603         },
 604     },
 605     returns => {
 606         type => 'array',
 607         items => {
 608             type => "object",
 609             properties => {
 610                 pool => { type => 'integer' },
 611                 pool_name => { type => 'string' },
 612                 size => { type => 'integer' },
 613             },
 614         },
 615         links => [ { rel => 'child', href => "{pool_name}" } ],
 616     },
 617     code => sub {
 618         my ($param) = @_;
 619
 620         PVE::Ceph::Tools::check_ceph_inited();
 621
 622         my $rados = PVE::RADOS->new();
 623
 624         my $stats = {};
 625         my $res = $rados->mon_command({ prefix => 'df' });
 626
 627         foreach my $d (@{$res->{pools}}) {
 628             next if !$d->{stats};
 629             next if !defined($d->{id});
 630             $stats->{$d->{id}} = $d->{stats};
 631         }
 632
 633         $res = $rados->mon_command({ prefix => 'osd dump' });
 634         my $rulestmp = $rados->mon_command({ prefix => 'osd crush rule dump'});
 635
 636         my $rules = {};
 637         for my $rule (@$rulestmp) {
 638             $rules->{$rule->{rule_id}} = $rule->{rule_name};
 639         }
 640
 641         my $data = [];
 642         foreach my $e (@{$res->{pools}}) {
 643             my $d = {};
 644             foreach my $attr (qw(pool pool_name size min_size pg_num crush_rule)) {
 645                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
 646             }
 647
 648             if (defined($d->{crush_rule}) && defined($rules->{$d->{crush_rule}})) {
 649                 $d->{crush_rule_name} = $rules->{$d->{crush_rule}};
 650             }
 651
 652             if (my $s = $stats->{$d->{pool}}) {
 653                 $d->{bytes_used} = $s->{bytes_used};
 654                 $d->{percent_used} = $s->{percent_used};
 655             }
 656             push @$data, $d;
 657         }
 658
 659
 660         return $data;
 661     }});
 662
 663 __PACKAGE__->register_method ({
 664     name => 'createpool',
 665     path => 'pools',
 666     method => 'POST',
 667     description => "Create POOL",
 668     proxyto => 'node',
 669     protected => 1,
 670     permissions => {
 671         check => ['perm', '/', [ 'Sys.Modify' ]],
 672     },
 673     parameters => {
 674         additionalProperties => 0,
 675         properties => {
 676             node => get_standard_option('pve-node'),
 677             name => {
 678                 description => "The name of the pool. It must be unique.",
 679                 type => 'string',
 680             },
 681             size => {
 682                 description => 'Number of replicas per object',
 683                 type => 'integer',
 684                 default => 3,
 685                 optional => 1,
 686                 minimum => 1,
 687                 maximum => 7,
 688             },
 689             min_size => {
 690                 description => 'Minimum number of replicas per object',
 691                 type => 'integer',
 692                 default => 2,
 693                 optional => 1,
 694                 minimum => 1,
 695                 maximum => 7,
 696             },
 697             pg_num => {
 698                 description => "Number of placement groups.",
 699                 type => 'integer',
 700                 default => 128,
 701                 optional => 1,
 702                 minimum => 8,
 703                 maximum => 32768,
 704             },
 705             crush_rule => {
 706                 description => "The rule to use for mapping object placement in the cluster.",
 707                 type => 'string',
 708                 optional => 1,
 709             },
 710             application => {
 711                 description => "The application of the pool, 'rbd' by default.",
 712                 type => 'string',
 713                 enum => ['rbd', 'cephfs', 'rgw'],
 714                 optional => 1,
 715             },
 716             add_storages => {
 717                 description => "Configure VM and CT storage using the new pool.",
 718                 type => 'boolean',
 719                 optional => 1,
 720             },
 721         },
 722     },
 723     returns => { type => 'string' },
 724     code => sub {
 725         my ($param) = @_;
 726
 727         PVE::Cluster::check_cfs_quorum();
 728         PVE::Ceph::Tools::check_ceph_inited();
 729
 730         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
 731
 732         die "not fully configured - missing '$pve_ckeyring_path'\n"
 733             if ! -f $pve_ckeyring_path;
 734
 735         my $pool = $param->{name};
 736         my $rpcenv = PVE::RPCEnvironment::get();
 737         my $user = $rpcenv->get_user();
 738
 739         if ($param->{add_storages}) {
 740             $rpcenv->check($user, '/storage', ['Datastore.Allocate']);
 741             die "pool name contains characters which are illegal for storage naming\n"
 742                 if !PVE::JSONSchema::parse_storage_id($pool);
 743         }
 744
 745         my $pg_num = $param->{pg_num} || 128;
 746         my $size = $param->{size} || 3;
 747         my $min_size = $param->{min_size} || 2;
 748         my $application = $param->{application} // 'rbd';
 749
 750         my $worker = sub {
 751
 752             PVE::Ceph::Tools::create_pool($pool, $param);
 753
 754             if ($param->{add_storages}) {
 755                 my $err;
 756                 eval { $add_storage->($pool, "${pool}"); };
 757                 if ($@) {
 758                     warn "failed to add storage: $@";
 759                     $err = 1;
 760                 }
 761                 die "adding storage for pool '$pool' failed, check log and add manually!\n"
 762                     if $err;
 763             }
 764         };
 765
 766         return $rpcenv->fork_worker('cephcreatepool', $pool,  $user, $worker);
 767     }});
 768
 769 __PACKAGE__->register_method ({
 770     name => 'get_flags',
 771     path => 'flags',
 772     method => 'GET',
 773     description => "get all set ceph flags",
 774     proxyto => 'node',
 775     protected => 1,
 776     permissions => {
 777         check => ['perm', '/', [ 'Sys.Audit' ]],
 778     },
 779     parameters => {
 780         additionalProperties => 0,
 781         properties => {
 782             node => get_standard_option('pve-node'),
 783         },
 784     },
 785     returns => { type => 'string' },
 786     code => sub {
 787         my ($param) = @_;
 788
 789         PVE::Ceph::Tools::check_ceph_inited();
 790
 791         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
 792
 793         die "not fully configured - missing '$pve_ckeyring_path'\n"
 794             if ! -f $pve_ckeyring_path;
 795
 796         my $rados = PVE::RADOS->new();
 797
 798         my $stat = $rados->mon_command({ prefix => 'osd dump' });
 799
 800         return $stat->{flags} // '';
 801     }});
 802
 803 __PACKAGE__->register_method ({
 804     name => 'set_flag',
 805     path => 'flags/{flag}',
 806     method => 'POST',
 807     description => "Set a ceph flag",
 808     proxyto => 'node',
 809     protected => 1,
 810     permissions => {
 811         check => ['perm', '/', [ 'Sys.Modify' ]],
 812     },
 813     parameters => {
 814         additionalProperties => 0,
 815         properties => {
 816             node => get_standard_option('pve-node'),
 817             flag => {
 818                 description => 'The ceph flag to set/unset',
 819                 type => 'string',
 820                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
 821             },
 822         },
 823     },
 824     returns => { type => 'null' },
 825     code => sub {
 826         my ($param) = @_;
 827
 828         PVE::Ceph::Tools::check_ceph_inited();
 829
 830         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
 831
 832         die "not fully configured - missing '$pve_ckeyring_path'\n"
 833             if ! -f $pve_ckeyring_path;
 834
 835         my $set = $param->{set} // !$param->{unset};
 836         my $rados = PVE::RADOS->new();
 837
 838         $rados->mon_command({
 839             prefix => "osd set",
 840             key => $param->{flag},
 841         });
 842
 843         return undef;
 844     }});
 845
 846 __PACKAGE__->register_method ({
 847     name => 'unset_flag',
 848     path => 'flags/{flag}',
 849     method => 'DELETE',
 850     description => "Unset a ceph flag",
 851     proxyto => 'node',
 852     protected => 1,
 853     permissions => {
 854         check => ['perm', '/', [ 'Sys.Modify' ]],
 855     },
 856     parameters => {
 857         additionalProperties => 0,
 858         properties => {
 859             node => get_standard_option('pve-node'),
 860             flag => {
 861                 description => 'The ceph flag to set/unset',
 862                 type => 'string',
 863                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
 864             },
 865         },
 866     },
 867     returns => { type => 'null' },
 868     code => sub {
 869         my ($param) = @_;
 870
 871         PVE::Ceph::Tools::check_ceph_inited();
 872
 873         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
 874
 875         die "not fully configured - missing '$pve_ckeyring_path'\n"
 876             if ! -f $pve_ckeyring_path;
 877
 878         my $set = $param->{set} // !$param->{unset};
 879         my $rados = PVE::RADOS->new();
 880
 881         $rados->mon_command({
 882             prefix => "osd unset",
 883             key => $param->{flag},
 884         });
 885
 886         return undef;
 887     }});
 888
 889 __PACKAGE__->register_method ({
 890     name => 'destroypool',
 891     path => 'pools/{name}',
 892     method => 'DELETE',
 893     description => "Destroy pool",
 894     proxyto => 'node',
 895     protected => 1,
 896     permissions => {
 897         check => ['perm', '/', [ 'Sys.Modify' ]],
 898     },
 899     parameters => {
 900         additionalProperties => 0,
 901         properties => {
 902             node => get_standard_option('pve-node'),
 903             name => {
 904                 description => "The name of the pool. It must be unique.",
 905                 type => 'string',
 906             },
 907             force => {
 908                 description => "If true, destroys pool even if in use",
 909                 type => 'boolean',
 910                 optional => 1,
 911                 default => 0,
 912             },
 913             remove_storages => {
 914                 description => "Remove all pveceph-managed storages configured for this pool",
 915                 type => 'boolean',
 916                 optional => 1,
 917                 default => 0,
 918             },
 919         },
 920     },
 921     returns => { type => 'string' },
 922     code => sub {
 923         my ($param) = @_;
 924
 925         PVE::Ceph::Tools::check_ceph_inited();
 926
 927         my $rpcenv = PVE::RPCEnvironment::get();
 928         my $user = $rpcenv->get_user();
 929         $rpcenv->check($user, '/storage', ['Datastore.Allocate'])
 930             if $param->{remove_storages};
 931
 932         my $pool = $param->{name};
 933
 934         my $worker = sub {
 935             my $storages = $get_storages->($pool);
 936
 937             # if not forced, destroy ceph pool only when no
 938             # vm disks are on it anymore
 939             if (!$param->{force}) {
 940                 my $storagecfg = PVE::Storage::config();
 941                 foreach my $storeid (keys %$storages) {
 942                     my $storage = $storages->{$storeid};
 943
 944                     # check if any vm disks are on the pool
 945                     print "checking storage '$storeid' for RBD images..\n";
 946                     my $res = PVE::Storage::vdisk_list($storagecfg, $storeid);
 947                     die "ceph pool '$pool' still in use by storage '$storeid'\n"
 948                         if @{$res->{$storeid}} != 0;
 949                 }
 950             }
 951
 952             PVE::Ceph::Tools::destroy_pool($pool);
 953
 954             if ($param->{remove_storages}) {
 955                 my $err;
 956                 foreach my $storeid (keys %$storages) {
 957                     # skip external clusters, not managed by pveceph
 958                     next if $storages->{$storeid}->{monhost};
 959                     eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
 960                     if ($@) {
 961                         warn "failed to remove storage '$storeid': $@\n";
 962                         $err = 1;
 963                     }
 964                 }
 965                 die "failed to remove (some) storages - check log and remove manually!\n"
 966                     if $err;
 967             }
 968         };
 969         return $rpcenv->fork_worker('cephdestroypool', $pool,  $user, $worker);
 970     }});
 971
 972
 973 __PACKAGE__->register_method ({
 974     name => 'crush',
 975     path => 'crush',
 976     method => 'GET',
 977     description => "Get OSD crush map",
 978     proxyto => 'node',
 979     protected => 1,
 980     permissions => {
 981         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 982     },
 983     parameters => {
 984         additionalProperties => 0,
 985         properties => {
 986             node => get_standard_option('pve-node'),
 987         },
 988     },
 989     returns => { type => 'string' },
 990     code => sub {
 991         my ($param) = @_;
 992
 993         PVE::Ceph::Tools::check_ceph_inited();
 994
 995         # this produces JSON (difficult to read for the user)
 996         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
 997
 998         my $txt = '';
 999
1000         my $mapfile = "/var/tmp/ceph-crush.map.$$";
1001         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
1002
1003         my $rados = PVE::RADOS->new();
1004
1005         eval {
1006             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
1007             file_set_contents($mapfile, $bindata);
1008             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
1009             $txt = file_get_contents($mapdata);
1010         };
1011         my $err = $@;
1012
1013         unlink $mapfile;
1014         unlink $mapdata;
1015
1016         die $err if $err;
1017
1018         return $txt;
1019     }});
1020
1021 __PACKAGE__->register_method({
1022     name => 'log',
1023     path => 'log',
1024     method => 'GET',
1025     description => "Read ceph log",
1026     proxyto => 'node',
1027     permissions => {
1028         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
1029     },
1030     protected => 1,
1031     parameters => {
1032         additionalProperties => 0,
1033         properties => {
1034             node => get_standard_option('pve-node'),
1035             start => {
1036                 type => 'integer',
1037                 minimum => 0,
1038                 optional => 1,
1039             },
1040             limit => {
1041                 type => 'integer',
1042                 minimum => 0,
1043                 optional => 1,
1044             },
1045         },
1046     },
1047     returns => {
1048         type => 'array',
1049         items => {
1050             type => "object",
1051             properties => {
1052                 n => {
1053                   description=>  "Line number",
1054                   type=> 'integer',
1055                 },
1056                 t => {
1057                   description=>  "Line text",
1058                   type => 'string',
1059                 }
1060             }
1061         }
1062     },
1063     code => sub {
1064         my ($param) = @_;
1065
1066         PVE::Ceph::Tools::check_ceph_inited();
1067
1068         my $rpcenv = PVE::RPCEnvironment::get();
1069         my $user = $rpcenv->get_user();
1070         my $node = $param->{node};
1071
1072         my $logfile = "/var/log/ceph/ceph.log";
1073         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
1074
1075         $rpcenv->set_result_attrib('total', $count);
1076
1077         return $lines;
1078     }});
1079
1080 __PACKAGE__->register_method ({
1081     name => 'rules',
1082     path => 'rules',
1083     method => 'GET',
1084     description => "List ceph rules.",
1085     proxyto => 'node',
1086     protected => 1,
1087     permissions => {
1088         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1089     },
1090     parameters => {
1091         additionalProperties => 0,
1092         properties => {
1093             node => get_standard_option('pve-node'),
1094         },
1095     },
1096     returns => {
1097         type => 'array',
1098         items => {
1099             type => "object",
1100             properties => {},
1101         },
1102         links => [ { rel => 'child', href => "{name}" } ],
1103     },
1104     code => sub {
1105         my ($param) = @_;
1106
1107         PVE::Ceph::Tools::check_ceph_inited();
1108
1109         my $rados = PVE::RADOS->new();
1110
1111         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
1112
1113         my $res = [];
1114
1115         foreach my $rule (@$rules) {
1116             push @$res, { name => $rule };
1117         }
1118
1119         return $res;
1120     }});
1121
1122 1;