PVE/API2/Ceph.pm

   1 package PVE::API2::Ceph;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use Net::IP;
   8 use UUID;
   9
  10 use PVE::Ceph::Tools;
  11 use PVE::Ceph::Services;
  12 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  13 use PVE::JSONSchema qw(get_standard_option);
  14 use PVE::Network;
  15 use PVE::RADOS;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::Storage;
  19 use PVE::Tools qw(run_command file_get_contents file_set_contents);
  20
  21 use PVE::API2::Ceph::OSD;
  22 use PVE::API2::Ceph::FS;
  23 use PVE::API2::Ceph::MDS;
  24 use PVE::API2::Ceph::MGR;
  25 use PVE::API2::Ceph::MON;
  26 use PVE::API2::Storage::Config;
  27
  28 use base qw(PVE::RESTHandler);
  29
  30 my $pve_osd_default_journal_size = 1024*5;
  31
  32 __PACKAGE__->register_method ({
  33     subclass => "PVE::API2::Ceph::OSD",
  34     path => 'osd',
  35 });
  36
  37 __PACKAGE__->register_method ({
  38     subclass => "PVE::API2::Ceph::MDS",
  39     path => 'mds',
  40 });
  41
  42 __PACKAGE__->register_method ({
  43     subclass => "PVE::API2::Ceph::MGR",
  44     path => 'mgr',
  45 });
  46
  47 __PACKAGE__->register_method ({
  48     subclass => "PVE::API2::Ceph::MON",
  49     path => 'mon',
  50 });
  51
  52 __PACKAGE__->register_method ({
  53     subclass => "PVE::API2::Ceph::FS",
  54     path => 'fs',
  55 });
  56
  57 __PACKAGE__->register_method ({
  58     name => 'index',
  59     path => '',
  60     method => 'GET',
  61     description => "Directory index.",
  62     permissions => { user => 'all' },
  63     permissions => {
  64         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  65     },
  66     parameters => {
  67         additionalProperties => 0,
  68         properties => {
  69             node => get_standard_option('pve-node'),
  70         },
  71     },
  72     returns => {
  73         type => 'array',
  74         items => {
  75             type => "object",
  76             properties => {},
  77         },
  78         links => [ { rel => 'child', href => "{name}" } ],
  79     },
  80     code => sub {
  81         my ($param) = @_;
  82
  83         my $result = [
  84             { name => 'init' },
  85             { name => 'mon' },
  86             { name => 'osd' },
  87             { name => 'pools' },
  88             { name => 'fs' },
  89             { name => 'mds' },
  90             { name => 'stop' },
  91             { name => 'start' },
  92             { name => 'status' },
  93             { name => 'crush' },
  94             { name => 'config' },
  95             { name => 'log' },
  96             { name => 'disks' },
  97             { name => 'flags' },
  98             { name => 'rules' },
  99         ];
 100
 101         return $result;
 102     }});
 103
 104 __PACKAGE__->register_method ({
 105     name => 'disks',
 106     path => 'disks',
 107     method => 'GET',
 108     description => "List local disks.",
 109     proxyto => 'node',
 110     protected => 1,
 111     permissions => {
 112         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 113     },
 114     parameters => {
 115         additionalProperties => 0,
 116         properties => {
 117             node => get_standard_option('pve-node'),
 118             type => {
 119                 description => "Only list specific types of disks.",
 120                 type => 'string',
 121                 enum => ['unused', 'journal_disks'],
 122                 optional => 1,
 123             },
 124         },
 125     },
 126     returns => {
 127         type => 'array',
 128         items => {
 129             type => "object",
 130             properties => {
 131                 dev => { type => 'string' },
 132                 used => { type => 'string', optional => 1 },
 133                 gpt => { type => 'boolean' },
 134                 size => { type => 'integer' },
 135                 osdid => { type => 'integer' },
 136                 vendor =>  { type => 'string', optional => 1 },
 137                 model =>  { type => 'string', optional => 1 },
 138                 serial =>  { type => 'string', optional => 1 },
 139             },
 140         },
 141         # links => [ { rel => 'child', href => "{}" } ],
 142     },
 143     code => sub {
 144         my ($param) = @_;
 145
 146         PVE::Ceph::Tools::check_ceph_inited();
 147
 148         my $disks = PVE::Diskmanage::get_disks(undef, 1);
 149
 150         my $res = [];
 151         foreach my $dev (keys %$disks) {
 152             my $d = $disks->{$dev};
 153             if ($param->{type}) {
 154                 if ($param->{type} eq 'journal_disks') {
 155                     next if $d->{osdid} >= 0;
 156                     next if !$d->{gpt};
 157                 } elsif ($param->{type} eq 'unused') {
 158                     next if $d->{used};
 159                 } else {
 160                     die "internal error"; # should not happen
 161                 }
 162             }
 163
 164             $d->{dev} = "/dev/$dev";
 165             push @$res, $d;
 166         }
 167
 168         return $res;
 169     }});
 170
 171 __PACKAGE__->register_method ({
 172     name => 'config',
 173     path => 'config',
 174     method => 'GET',
 175     permissions => {
 176         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 177     },
 178     description => "Get Ceph configuration.",
 179     parameters => {
 180         additionalProperties => 0,
 181         properties => {
 182             node => get_standard_option('pve-node'),
 183         },
 184     },
 185     returns => { type => 'string' },
 186     code => sub {
 187         my ($param) = @_;
 188
 189         PVE::Ceph::Tools::check_ceph_inited();
 190
 191         my $path = PVE::Ceph::Tools::get_config('pve_ceph_cfgpath');
 192         return file_get_contents($path);
 193
 194     }});
 195
 196 my $add_storage = sub {
 197     my ($pool, $storeid) = @_;
 198
 199     my $storage_params = {
 200         type => 'rbd',
 201         pool => $pool,
 202         storage => $storeid,
 203         krbd => 0,
 204         content => 'rootdir,images',
 205     };
 206
 207     PVE::API2::Storage::Config->create($storage_params);
 208 };
 209
 210 my $get_storages = sub {
 211     my ($pool) = @_;
 212
 213     my $cfg = PVE::Storage::config();
 214
 215     my $storages = $cfg->{ids};
 216     my $res = {};
 217     foreach my $storeid (keys %$storages) {
 218         my $curr = $storages->{$storeid};
 219         $res->{$storeid} = $storages->{$storeid}
 220             if $curr->{type} eq 'rbd' && $pool eq $curr->{pool};
 221     }
 222
 223     return $res;
 224 };
 225
 226 __PACKAGE__->register_method ({
 227     name => 'init',
 228     path => 'init',
 229     method => 'POST',
 230     description => "Create initial ceph default configuration and setup symlinks.",
 231     proxyto => 'node',
 232     protected => 1,
 233     permissions => {
 234         check => ['perm', '/', [ 'Sys.Modify' ]],
 235     },
 236     parameters => {
 237         additionalProperties => 0,
 238         properties => {
 239             node => get_standard_option('pve-node'),
 240             network => {
 241                 description => "Use specific network for all ceph related traffic",
 242                 type => 'string', format => 'CIDR',
 243                 optional => 1,
 244                 maxLength => 128,
 245             },
 246             'cluster-network' => {
 247                 description => "Declare a separate cluster network, OSDs will route" .
 248                     "heartbeat, object replication and recovery traffic over it",
 249                 type => 'string', format => 'CIDR',
 250                 requires => 'network',
 251                 optional => 1,
 252                 maxLength => 128,
 253             },
 254             size => {
 255                 description => 'Targeted number of replicas per object',
 256                 type => 'integer',
 257                 default => 3,
 258                 optional => 1,
 259                 minimum => 1,
 260                 maximum => 7,
 261             },
 262             min_size => {
 263                 description => 'Minimum number of available replicas per object to allow I/O',
 264                 type => 'integer',
 265                 default => 2,
 266                 optional => 1,
 267                 minimum => 1,
 268                 maximum => 7,
 269             },
 270             pg_bits => {
 271                 description => "Placement group bits, used to specify the " .
 272                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 273                     "default pg num' does not work for default pools.",
 274                 type => 'integer',
 275                 default => 6,
 276                 optional => 1,
 277                 minimum => 6,
 278                 maximum => 14,
 279             },
 280             disable_cephx => {
 281                 description => "Disable cephx authentification.\n\n" .
 282                     "WARNING: cephx is a security feature protecting against " .
 283                     "man-in-the-middle attacks. Only consider disabling cephx ".
 284                     "if your network is private!",
 285                 type => 'boolean',
 286                 optional => 1,
 287                 default => 0,
 288             },
 289         },
 290     },
 291     returns => { type => 'null' },
 292     code => sub {
 293         my ($param) = @_;
 294
 295         my $version = PVE::Ceph::Tools::get_local_version(1);
 296
 297         if (!$version || $version < 12) {
 298             die "Ceph Luminous required - please run 'pveceph install'\n";
 299         } else {
 300             PVE::Ceph::Tools::check_ceph_installed('ceph_bin');
 301         }
 302
 303         # simply load old config if it already exists
 304         my $cfg = cfs_read_file('ceph.conf');
 305
 306         if (!$cfg->{global}) {
 307
 308             my $fsid;
 309             my $uuid;
 310
 311             UUID::generate($uuid);
 312             UUID::unparse($uuid, $fsid);
 313
 314             my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 315
 316             $cfg->{global} = {
 317                 'fsid' => $fsid,
 318                 'auth cluster required' => $auth,
 319                 'auth service required' => $auth,
 320                 'auth client required' => $auth,
 321                 'osd journal size' => $pve_osd_default_journal_size,
 322                 'osd pool default size' => $param->{size} // 3,
 323                 'osd pool default min size' => $param->{min_size} // 2,
 324                 'mon allow pool delete' => 'true',
 325             };
 326
 327             # this does not work for default pools
 328             #'osd pool default pg num' => $pg_num,
 329             #'osd pool default pgp num' => $pg_num,
 330         }
 331
 332         $cfg->{global}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 333         $cfg->{osd}->{keyring} = '/var/lib/ceph/osd/ceph-$id/keyring';
 334
 335         if ($param->{pg_bits}) {
 336             $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 337             $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 338         }
 339
 340         if ($param->{network}) {
 341             $cfg->{global}->{'public network'} = $param->{network};
 342             $cfg->{global}->{'cluster network'} = $param->{network};
 343         }
 344
 345         if ($param->{'cluster-network'}) {
 346             $cfg->{global}->{'cluster network'} = $param->{'cluster-network'};
 347         }
 348
 349         cfs_write_file('ceph.conf', $cfg);
 350
 351         PVE::Ceph::Tools::setup_pve_symlinks();
 352
 353         return undef;
 354     }});
 355
 356 __PACKAGE__->register_method ({
 357     name => 'stop',
 358     path => 'stop',
 359     method => 'POST',
 360     description => "Stop ceph services.",
 361     proxyto => 'node',
 362     protected => 1,
 363     permissions => {
 364         check => ['perm', '/', [ 'Sys.Modify' ]],
 365     },
 366     parameters => {
 367         additionalProperties => 0,
 368         properties => {
 369             node => get_standard_option('pve-node'),
 370             service => {
 371                 description => 'Ceph service name.',
 372                 type => 'string',
 373                 optional => 1,
 374                 default => 'ceph.target',
 375                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
 376             },
 377         },
 378     },
 379     returns => { type => 'string' },
 380     code => sub {
 381         my ($param) = @_;
 382
 383         my $rpcenv = PVE::RPCEnvironment::get();
 384
 385         my $authuser = $rpcenv->get_user();
 386
 387         PVE::Ceph::Tools::check_ceph_inited();
 388
 389         my $cfg = cfs_read_file('ceph.conf');
 390         scalar(keys %$cfg) || die "no configuration\n";
 391
 392         my $worker = sub {
 393             my $upid = shift;
 394
 395             my $cmd = ['stop'];
 396             if ($param->{service}) {
 397                 push @$cmd, $param->{service};
 398             }
 399
 400             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 401         };
 402
 403         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
 404                                     $authuser, $worker);
 405     }});
 406
 407 __PACKAGE__->register_method ({
 408     name => 'start',
 409     path => 'start',
 410     method => 'POST',
 411     description => "Start ceph services.",
 412     proxyto => 'node',
 413     protected => 1,
 414     permissions => {
 415         check => ['perm', '/', [ 'Sys.Modify' ]],
 416     },
 417     parameters => {
 418         additionalProperties => 0,
 419         properties => {
 420             node => get_standard_option('pve-node'),
 421             service => {
 422                 description => 'Ceph service name.',
 423                 type => 'string',
 424                 optional => 1,
 425                 default => 'ceph.target',
 426                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
 427             },
 428         },
 429     },
 430     returns => { type => 'string' },
 431     code => sub {
 432         my ($param) = @_;
 433
 434         my $rpcenv = PVE::RPCEnvironment::get();
 435
 436         my $authuser = $rpcenv->get_user();
 437
 438         PVE::Ceph::Tools::check_ceph_inited();
 439
 440         my $cfg = cfs_read_file('ceph.conf');
 441         scalar(keys %$cfg) || die "no configuration\n";
 442
 443         my $worker = sub {
 444             my $upid = shift;
 445
 446             my $cmd = ['start'];
 447             if ($param->{service}) {
 448                 push @$cmd, $param->{service};
 449             }
 450
 451             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 452         };
 453
 454         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
 455                                     $authuser, $worker);
 456     }});
 457
 458 __PACKAGE__->register_method ({
 459     name => 'restart',
 460     path => 'restart',
 461     method => 'POST',
 462     description => "Restart ceph services.",
 463     proxyto => 'node',
 464     protected => 1,
 465     permissions => {
 466         check => ['perm', '/', [ 'Sys.Modify' ]],
 467     },
 468     parameters => {
 469         additionalProperties => 0,
 470         properties => {
 471             node => get_standard_option('pve-node'),
 472             service => {
 473                 description => 'Ceph service name.',
 474                 type => 'string',
 475                 optional => 1,
 476                 default => 'ceph.target',
 477                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
 478             },
 479         },
 480     },
 481     returns => { type => 'string' },
 482     code => sub {
 483         my ($param) = @_;
 484
 485         my $rpcenv = PVE::RPCEnvironment::get();
 486
 487         my $authuser = $rpcenv->get_user();
 488
 489         PVE::Ceph::Tools::check_ceph_inited();
 490
 491         my $cfg = cfs_read_file('ceph.conf');
 492         scalar(keys %$cfg) || die "no configuration\n";
 493
 494         my $worker = sub {
 495             my $upid = shift;
 496
 497             my $cmd = ['restart'];
 498             if ($param->{service}) {
 499                 push @$cmd, $param->{service};
 500             }
 501
 502             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 503         };
 504
 505         return $rpcenv->fork_worker('srvrestart', $param->{service} || 'ceph',
 506                                     $authuser, $worker);
 507     }});
 508
 509 __PACKAGE__->register_method ({
 510     name => 'status',
 511     path => 'status',
 512     method => 'GET',
 513     description => "Get ceph status.",
 514     proxyto => 'node',
 515     protected => 1,
 516     permissions => {
 517         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 518     },
 519     parameters => {
 520         additionalProperties => 0,
 521         properties => {
 522             node => get_standard_option('pve-node'),
 523         },
 524     },
 525     returns => { type => 'object' },
 526     code => sub {
 527         my ($param) = @_;
 528
 529         PVE::Ceph::Tools::check_ceph_inited();
 530
 531         my $rados = PVE::RADOS->new();
 532         my $status = $rados->mon_command({ prefix => 'status' });
 533         $status->{health} = $rados->mon_command({ prefix => 'health', detail => 'detail' });
 534         return $status;
 535     }});
 536
 537 __PACKAGE__->register_method ({
 538     name => 'lspools',
 539     path => 'pools',
 540     method => 'GET',
 541     description => "List all pools.",
 542     proxyto => 'node',
 543     protected => 1,
 544     permissions => {
 545         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 546     },
 547     parameters => {
 548         additionalProperties => 0,
 549         properties => {
 550             node => get_standard_option('pve-node'),
 551         },
 552     },
 553     returns => {
 554         type => 'array',
 555         items => {
 556             type => "object",
 557             properties => {
 558                 pool => { type => 'integer' },
 559                 pool_name => { type => 'string' },
 560                 size => { type => 'integer' },
 561             },
 562         },
 563         links => [ { rel => 'child', href => "{pool_name}" } ],
 564     },
 565     code => sub {
 566         my ($param) = @_;
 567
 568         PVE::Ceph::Tools::check_ceph_inited();
 569
 570         my $rados = PVE::RADOS->new();
 571
 572         my $stats = {};
 573         my $res = $rados->mon_command({ prefix => 'df' });
 574
 575         foreach my $d (@{$res->{pools}}) {
 576             next if !$d->{stats};
 577             next if !defined($d->{id});
 578             $stats->{$d->{id}} = $d->{stats};
 579         }
 580
 581         $res = $rados->mon_command({ prefix => 'osd dump' });
 582         my $rulestmp = $rados->mon_command({ prefix => 'osd crush rule dump'});
 583
 584         my $rules = {};
 585         for my $rule (@$rulestmp) {
 586             $rules->{$rule->{rule_id}} = $rule->{rule_name};
 587         }
 588
 589         my $data = [];
 590         foreach my $e (@{$res->{pools}}) {
 591             my $d = {};
 592             foreach my $attr (qw(pool pool_name size min_size pg_num crush_rule)) {
 593                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
 594             }
 595
 596             if (defined($d->{crush_rule}) && defined($rules->{$d->{crush_rule}})) {
 597                 $d->{crush_rule_name} = $rules->{$d->{crush_rule}};
 598             }
 599
 600             if (my $s = $stats->{$d->{pool}}) {
 601                 $d->{bytes_used} = $s->{bytes_used};
 602                 $d->{percent_used} = $s->{percent_used};
 603             }
 604             push @$data, $d;
 605         }
 606
 607
 608         return $data;
 609     }});
 610
 611 __PACKAGE__->register_method ({
 612     name => 'createpool',
 613     path => 'pools',
 614     method => 'POST',
 615     description => "Create POOL",
 616     proxyto => 'node',
 617     protected => 1,
 618     permissions => {
 619         check => ['perm', '/', [ 'Sys.Modify' ]],
 620     },
 621     parameters => {
 622         additionalProperties => 0,
 623         properties => {
 624             node => get_standard_option('pve-node'),
 625             name => {
 626                 description => "The name of the pool. It must be unique.",
 627                 type => 'string',
 628             },
 629             size => {
 630                 description => 'Number of replicas per object',
 631                 type => 'integer',
 632                 default => 3,
 633                 optional => 1,
 634                 minimum => 1,
 635                 maximum => 7,
 636             },
 637             min_size => {
 638                 description => 'Minimum number of replicas per object',
 639                 type => 'integer',
 640                 default => 2,
 641                 optional => 1,
 642                 minimum => 1,
 643                 maximum => 7,
 644             },
 645             pg_num => {
 646                 description => "Number of placement groups.",
 647                 type => 'integer',
 648                 default => 128,
 649                 optional => 1,
 650                 minimum => 8,
 651                 maximum => 32768,
 652             },
 653             crush_rule => {
 654                 description => "The rule to use for mapping object placement in the cluster.",
 655                 type => 'string',
 656                 optional => 1,
 657             },
 658             application => {
 659                 description => "The application of the pool, 'rbd' by default.",
 660                 type => 'string',
 661                 enum => ['rbd', 'cephfs', 'rgw'],
 662                 optional => 1,
 663             },
 664             add_storages => {
 665                 description => "Configure VM and CT storage using the new pool.",
 666                 type => 'boolean',
 667                 optional => 1,
 668             },
 669         },
 670     },
 671     returns => { type => 'string' },
 672     code => sub {
 673         my ($param) = @_;
 674
 675         PVE::Cluster::check_cfs_quorum();
 676         PVE::Ceph::Tools::check_ceph_inited();
 677
 678         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
 679
 680         die "not fully configured - missing '$pve_ckeyring_path'\n"
 681             if ! -f $pve_ckeyring_path;
 682
 683         my $pool = $param->{name};
 684         my $rpcenv = PVE::RPCEnvironment::get();
 685         my $user = $rpcenv->get_user();
 686
 687         if ($param->{add_storages}) {
 688             $rpcenv->check($user, '/storage', ['Datastore.Allocate']);
 689             die "pool name contains characters which are illegal for storage naming\n"
 690                 if !PVE::JSONSchema::parse_storage_id($pool);
 691         }
 692
 693         my $pg_num = $param->{pg_num} || 128;
 694         my $size = $param->{size} || 3;
 695         my $min_size = $param->{min_size} || 2;
 696         my $application = $param->{application} // 'rbd';
 697
 698         my $worker = sub {
 699
 700             PVE::Ceph::Tools::create_pool($pool, $param);
 701
 702             if ($param->{add_storages}) {
 703                 my $err;
 704                 eval { $add_storage->($pool, "${pool}"); };
 705                 if ($@) {
 706                     warn "failed to add storage: $@";
 707                     $err = 1;
 708                 }
 709                 die "adding storage for pool '$pool' failed, check log and add manually!\n"
 710                     if $err;
 711             }
 712         };
 713
 714         return $rpcenv->fork_worker('cephcreatepool', $pool,  $user, $worker);
 715     }});
 716
 717 __PACKAGE__->register_method ({
 718     name => 'get_flags',
 719     path => 'flags',
 720     method => 'GET',
 721     description => "get all set ceph flags",
 722     proxyto => 'node',
 723     protected => 1,
 724     permissions => {
 725         check => ['perm', '/', [ 'Sys.Audit' ]],
 726     },
 727     parameters => {
 728         additionalProperties => 0,
 729         properties => {
 730             node => get_standard_option('pve-node'),
 731         },
 732     },
 733     returns => { type => 'string' },
 734     code => sub {
 735         my ($param) = @_;
 736
 737         PVE::Ceph::Tools::check_ceph_inited();
 738
 739         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
 740
 741         die "not fully configured - missing '$pve_ckeyring_path'\n"
 742             if ! -f $pve_ckeyring_path;
 743
 744         my $rados = PVE::RADOS->new();
 745
 746         my $stat = $rados->mon_command({ prefix => 'osd dump' });
 747
 748         return $stat->{flags} // '';
 749     }});
 750
 751 __PACKAGE__->register_method ({
 752     name => 'set_flag',
 753     path => 'flags/{flag}',
 754     method => 'POST',
 755     description => "Set a ceph flag",
 756     proxyto => 'node',
 757     protected => 1,
 758     permissions => {
 759         check => ['perm', '/', [ 'Sys.Modify' ]],
 760     },
 761     parameters => {
 762         additionalProperties => 0,
 763         properties => {
 764             node => get_standard_option('pve-node'),
 765             flag => {
 766                 description => 'The ceph flag to set/unset',
 767                 type => 'string',
 768                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
 769             },
 770         },
 771     },
 772     returns => { type => 'null' },
 773     code => sub {
 774         my ($param) = @_;
 775
 776         PVE::Ceph::Tools::check_ceph_inited();
 777
 778         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
 779
 780         die "not fully configured - missing '$pve_ckeyring_path'\n"
 781             if ! -f $pve_ckeyring_path;
 782
 783         my $set = $param->{set} // !$param->{unset};
 784         my $rados = PVE::RADOS->new();
 785
 786         $rados->mon_command({
 787             prefix => "osd set",
 788             key => $param->{flag},
 789         });
 790
 791         return undef;
 792     }});
 793
 794 __PACKAGE__->register_method ({
 795     name => 'unset_flag',
 796     path => 'flags/{flag}',
 797     method => 'DELETE',
 798     description => "Unset a ceph flag",
 799     proxyto => 'node',
 800     protected => 1,
 801     permissions => {
 802         check => ['perm', '/', [ 'Sys.Modify' ]],
 803     },
 804     parameters => {
 805         additionalProperties => 0,
 806         properties => {
 807             node => get_standard_option('pve-node'),
 808             flag => {
 809                 description => 'The ceph flag to set/unset',
 810                 type => 'string',
 811                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
 812             },
 813         },
 814     },
 815     returns => { type => 'null' },
 816     code => sub {
 817         my ($param) = @_;
 818
 819         PVE::Ceph::Tools::check_ceph_inited();
 820
 821         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
 822
 823         die "not fully configured - missing '$pve_ckeyring_path'\n"
 824             if ! -f $pve_ckeyring_path;
 825
 826         my $set = $param->{set} // !$param->{unset};
 827         my $rados = PVE::RADOS->new();
 828
 829         $rados->mon_command({
 830             prefix => "osd unset",
 831             key => $param->{flag},
 832         });
 833
 834         return undef;
 835     }});
 836
 837 __PACKAGE__->register_method ({
 838     name => 'destroypool',
 839     path => 'pools/{name}',
 840     method => 'DELETE',
 841     description => "Destroy pool",
 842     proxyto => 'node',
 843     protected => 1,
 844     permissions => {
 845         check => ['perm', '/', [ 'Sys.Modify' ]],
 846     },
 847     parameters => {
 848         additionalProperties => 0,
 849         properties => {
 850             node => get_standard_option('pve-node'),
 851             name => {
 852                 description => "The name of the pool. It must be unique.",
 853                 type => 'string',
 854             },
 855             force => {
 856                 description => "If true, destroys pool even if in use",
 857                 type => 'boolean',
 858                 optional => 1,
 859                 default => 0,
 860             },
 861             remove_storages => {
 862                 description => "Remove all pveceph-managed storages configured for this pool",
 863                 type => 'boolean',
 864                 optional => 1,
 865                 default => 0,
 866             },
 867         },
 868     },
 869     returns => { type => 'string' },
 870     code => sub {
 871         my ($param) = @_;
 872
 873         PVE::Ceph::Tools::check_ceph_inited();
 874
 875         my $rpcenv = PVE::RPCEnvironment::get();
 876         my $user = $rpcenv->get_user();
 877         $rpcenv->check($user, '/storage', ['Datastore.Allocate'])
 878             if $param->{remove_storages};
 879
 880         my $pool = $param->{name};
 881
 882         my $worker = sub {
 883             my $storages = $get_storages->($pool);
 884
 885             # if not forced, destroy ceph pool only when no
 886             # vm disks are on it anymore
 887             if (!$param->{force}) {
 888                 my $storagecfg = PVE::Storage::config();
 889                 foreach my $storeid (keys %$storages) {
 890                     my $storage = $storages->{$storeid};
 891
 892                     # check if any vm disks are on the pool
 893                     print "checking storage '$storeid' for RBD images..\n";
 894                     my $res = PVE::Storage::vdisk_list($storagecfg, $storeid);
 895                     die "ceph pool '$pool' still in use by storage '$storeid'\n"
 896                         if @{$res->{$storeid}} != 0;
 897                 }
 898             }
 899
 900             PVE::Ceph::Tools::destroy_pool($pool);
 901
 902             if ($param->{remove_storages}) {
 903                 my $err;
 904                 foreach my $storeid (keys %$storages) {
 905                     # skip external clusters, not managed by pveceph
 906                     next if $storages->{$storeid}->{monhost};
 907                     eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
 908                     if ($@) {
 909                         warn "failed to remove storage '$storeid': $@\n";
 910                         $err = 1;
 911                     }
 912                 }
 913                 die "failed to remove (some) storages - check log and remove manually!\n"
 914                     if $err;
 915             }
 916         };
 917         return $rpcenv->fork_worker('cephdestroypool', $pool,  $user, $worker);
 918     }});
 919
 920
 921 __PACKAGE__->register_method ({
 922     name => 'crush',
 923     path => 'crush',
 924     method => 'GET',
 925     description => "Get OSD crush map",
 926     proxyto => 'node',
 927     protected => 1,
 928     permissions => {
 929         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 930     },
 931     parameters => {
 932         additionalProperties => 0,
 933         properties => {
 934             node => get_standard_option('pve-node'),
 935         },
 936     },
 937     returns => { type => 'string' },
 938     code => sub {
 939         my ($param) = @_;
 940
 941         PVE::Ceph::Tools::check_ceph_inited();
 942
 943         # this produces JSON (difficult to read for the user)
 944         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
 945
 946         my $txt = '';
 947
 948         my $mapfile = "/var/tmp/ceph-crush.map.$$";
 949         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
 950
 951         my $rados = PVE::RADOS->new();
 952
 953         eval {
 954             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
 955             file_set_contents($mapfile, $bindata);
 956             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
 957             $txt = file_get_contents($mapdata);
 958         };
 959         my $err = $@;
 960
 961         unlink $mapfile;
 962         unlink $mapdata;
 963
 964         die $err if $err;
 965
 966         return $txt;
 967     }});
 968
 969 __PACKAGE__->register_method({
 970     name => 'log',
 971     path => 'log',
 972     method => 'GET',
 973     description => "Read ceph log",
 974     proxyto => 'node',
 975     permissions => {
 976         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
 977     },
 978     protected => 1,
 979     parameters => {
 980         additionalProperties => 0,
 981         properties => {
 982             node => get_standard_option('pve-node'),
 983             start => {
 984                 type => 'integer',
 985                 minimum => 0,
 986                 optional => 1,
 987             },
 988             limit => {
 989                 type => 'integer',
 990                 minimum => 0,
 991                 optional => 1,
 992             },
 993         },
 994     },
 995     returns => {
 996         type => 'array',
 997         items => {
 998             type => "object",
 999             properties => {
1000                 n => {
1001                   description=>  "Line number",
1002                   type=> 'integer',
1003                 },
1004                 t => {
1005                   description=>  "Line text",
1006                   type => 'string',
1007                 }
1008             }
1009         }
1010     },
1011     code => sub {
1012         my ($param) = @_;
1013
1014         PVE::Ceph::Tools::check_ceph_inited();
1015
1016         my $rpcenv = PVE::RPCEnvironment::get();
1017         my $user = $rpcenv->get_user();
1018         my $node = $param->{node};
1019
1020         my $logfile = "/var/log/ceph/ceph.log";
1021         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
1022
1023         $rpcenv->set_result_attrib('total', $count);
1024
1025         return $lines;
1026     }});
1027
1028 __PACKAGE__->register_method ({
1029     name => 'rules',
1030     path => 'rules',
1031     method => 'GET',
1032     description => "List ceph rules.",
1033     proxyto => 'node',
1034     protected => 1,
1035     permissions => {
1036         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1037     },
1038     parameters => {
1039         additionalProperties => 0,
1040         properties => {
1041             node => get_standard_option('pve-node'),
1042         },
1043     },
1044     returns => {
1045         type => 'array',
1046         items => {
1047             type => "object",
1048             properties => {},
1049         },
1050         links => [ { rel => 'child', href => "{name}" } ],
1051     },
1052     code => sub {
1053         my ($param) = @_;
1054
1055         PVE::Ceph::Tools::check_ceph_inited();
1056
1057         my $rados = PVE::RADOS->new();
1058
1059         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
1060
1061         my $res = [];
1062
1063         foreach my $rule (@$rules) {
1064             push @$res, { name => $rule };
1065         }
1066
1067         return $res;
1068     }});
1069
1070 1;