PVE/API2/Ceph.pm

   1 package PVE::API2::Ceph;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use Net::IP;
   8 use UUID;
   9
  10 use PVE::Ceph::Tools;
  11 use PVE::Ceph::Services;
  12 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  13 use PVE::JSONSchema qw(get_standard_option);
  14 use PVE::Network;
  15 use PVE::RADOS;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::Storage;
  19 use PVE::Tools qw(run_command file_get_contents file_set_contents extract_param);
  20
  21 use PVE::API2::Ceph::OSD;
  22 use PVE::API2::Ceph::FS;
  23 use PVE::API2::Ceph::MDS;
  24 use PVE::API2::Ceph::MGR;
  25 use PVE::API2::Ceph::MON;
  26 use PVE::API2::Storage::Config;
  27
  28 use base qw(PVE::RESTHandler);
  29
  30 my $pve_osd_default_journal_size = 1024*5;
  31
  32 __PACKAGE__->register_method ({
  33     subclass => "PVE::API2::Ceph::OSD",
  34     path => 'osd',
  35 });
  36
  37 __PACKAGE__->register_method ({
  38     subclass => "PVE::API2::Ceph::MDS",
  39     path => 'mds',
  40 });
  41
  42 __PACKAGE__->register_method ({
  43     subclass => "PVE::API2::Ceph::MGR",
  44     path => 'mgr',
  45 });
  46
  47 __PACKAGE__->register_method ({
  48     subclass => "PVE::API2::Ceph::MON",
  49     path => 'mon',
  50 });
  51
  52 __PACKAGE__->register_method ({
  53     subclass => "PVE::API2::Ceph::FS",
  54     path => 'fs',
  55 });
  56
  57 __PACKAGE__->register_method ({
  58     name => 'index',
  59     path => '',
  60     method => 'GET',
  61     description => "Directory index.",
  62     permissions => { user => 'all' },
  63     permissions => {
  64         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  65     },
  66     parameters => {
  67         additionalProperties => 0,
  68         properties => {
  69             node => get_standard_option('pve-node'),
  70         },
  71     },
  72     returns => {
  73         type => 'array',
  74         items => {
  75             type => "object",
  76             properties => {},
  77         },
  78         links => [ { rel => 'child', href => "{name}" } ],
  79     },
  80     code => sub {
  81         my ($param) = @_;
  82
  83         my $result = [
  84             { name => 'init' },
  85             { name => 'mon' },
  86             { name => 'osd' },
  87             { name => 'pools' },
  88             { name => 'fs' },
  89             { name => 'mds' },
  90             { name => 'stop' },
  91             { name => 'start' },
  92             { name => 'restart' },
  93             { name => 'status' },
  94             { name => 'crush' },
  95             { name => 'config' },
  96             { name => 'log' },
  97             { name => 'disks' }, # FIXME: remove with 7.0
  98             { name => 'flags' }, # FIXME: remove with 7.0
  99             { name => 'rules' },
 100         ];
 101
 102         return $result;
 103     }});
 104
 105 # FIXME: Remove with PVE 7.0
 106 __PACKAGE__->register_method ({
 107     name => 'disks',
 108     path => 'disks',
 109     method => 'GET',
 110     description => "List local disks.",
 111     proxyto => 'node',
 112     protected => 1,
 113     permissions => {
 114         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 115     },
 116     parameters => {
 117         additionalProperties => 0,
 118         properties => {
 119             node => get_standard_option('pve-node'),
 120             type => {
 121                 description => "Only list specific types of disks.",
 122                 type => 'string',
 123                 enum => ['unused', 'journal_disks'],
 124                 optional => 1,
 125             },
 126         },
 127     },
 128     returns => {
 129         type => 'array',
 130         items => {
 131             type => "object",
 132             properties => {
 133                 dev => { type => 'string' },
 134                 used => { type => 'string', optional => 1 },
 135                 gpt => { type => 'boolean' },
 136                 size => { type => 'integer' },
 137                 osdid => { type => 'integer' },
 138                 vendor =>  { type => 'string', optional => 1 },
 139                 model =>  { type => 'string', optional => 1 },
 140                 serial =>  { type => 'string', optional => 1 },
 141             },
 142         },
 143         # links => [ { rel => 'child', href => "{}" } ],
 144     },
 145     code => sub {
 146         my ($param) = @_;
 147
 148         PVE::Ceph::Tools::check_ceph_inited();
 149
 150         my $disks = PVE::Diskmanage::get_disks(undef, 1);
 151
 152         my $res = [];
 153         foreach my $dev (keys %$disks) {
 154             my $d = $disks->{$dev};
 155             if ($param->{type}) {
 156                 if ($param->{type} eq 'journal_disks') {
 157                     next if $d->{osdid} >= 0;
 158                     next if !$d->{gpt};
 159                 } elsif ($param->{type} eq 'unused') {
 160                     next if $d->{used};
 161                 } else {
 162                     die "internal error"; # should not happen
 163                 }
 164             }
 165
 166             $d->{dev} = "/dev/$dev";
 167             push @$res, $d;
 168         }
 169
 170         return $res;
 171     }});
 172
 173 __PACKAGE__->register_method ({
 174     name => 'config',
 175     path => 'config',
 176     method => 'GET',
 177     proxyto => 'node',
 178     permissions => {
 179         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 180     },
 181     description => "Get Ceph configuration.",
 182     parameters => {
 183         additionalProperties => 0,
 184         properties => {
 185             node => get_standard_option('pve-node'),
 186         },
 187     },
 188     returns => { type => 'string' },
 189     code => sub {
 190         my ($param) = @_;
 191
 192         PVE::Ceph::Tools::check_ceph_inited();
 193
 194         my $path = PVE::Ceph::Tools::get_config('pve_ceph_cfgpath');
 195         return file_get_contents($path);
 196
 197     }});
 198
 199 __PACKAGE__->register_method ({
 200     name => 'configdb',
 201     path => 'configdb',
 202     method => 'GET',
 203     proxyto => 'node',
 204     protected => 1,
 205     permissions => {
 206         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 207     },
 208     description => "Get Ceph configuration database.",
 209     parameters => {
 210         additionalProperties => 0,
 211         properties => {
 212             node => get_standard_option('pve-node'),
 213         },
 214     },
 215     returns => {
 216         type => 'array',
 217         items => {
 218             type => 'object',
 219             properties => {
 220                 section => { type => "string", },
 221                 name => { type => "string", },
 222                 value => { type => "string", },
 223                 level => { type => "string", },
 224                 'can_update_at_runtime' => { type => "boolean", },
 225                 mask => { type => "string" },
 226             },
 227         },
 228     },
 229     code => sub {
 230         my ($param) = @_;
 231
 232         PVE::Ceph::Tools::check_ceph_inited();
 233
 234         my $rados = PVE::RADOS->new();
 235         my $res = $rados->mon_command( { prefix => 'config dump', format => 'json' });
 236         foreach my $entry (@$res) {
 237             $entry->{can_update_at_runtime} = $entry->{can_update_at_runtime}? 1 : 0; # JSON::true/false -> 1/0
 238         }
 239
 240         return $res;
 241     }});
 242
 243 my $add_storage = sub {
 244     my ($pool, $storeid) = @_;
 245
 246     my $storage_params = {
 247         type => 'rbd',
 248         pool => $pool,
 249         storage => $storeid,
 250         krbd => 0,
 251         content => 'rootdir,images',
 252     };
 253
 254     PVE::API2::Storage::Config->create($storage_params);
 255 };
 256
 257 my $get_storages = sub {
 258     my ($pool) = @_;
 259
 260     my $cfg = PVE::Storage::config();
 261
 262     my $storages = $cfg->{ids};
 263     my $res = {};
 264     foreach my $storeid (keys %$storages) {
 265         my $curr = $storages->{$storeid};
 266         $res->{$storeid} = $storages->{$storeid}
 267             if $curr->{type} eq 'rbd' && $pool eq $curr->{pool};
 268     }
 269
 270     return $res;
 271 };
 272
 273 __PACKAGE__->register_method ({
 274     name => 'init',
 275     path => 'init',
 276     method => 'POST',
 277     description => "Create initial ceph default configuration and setup symlinks.",
 278     proxyto => 'node',
 279     protected => 1,
 280     permissions => {
 281         check => ['perm', '/', [ 'Sys.Modify' ]],
 282     },
 283     parameters => {
 284         additionalProperties => 0,
 285         properties => {
 286             node => get_standard_option('pve-node'),
 287             network => {
 288                 description => "Use specific network for all ceph related traffic",
 289                 type => 'string', format => 'CIDR',
 290                 optional => 1,
 291                 maxLength => 128,
 292             },
 293             'cluster-network' => {
 294                 description => "Declare a separate cluster network, OSDs will route" .
 295                     "heartbeat, object replication and recovery traffic over it",
 296                 type => 'string', format => 'CIDR',
 297                 requires => 'network',
 298                 optional => 1,
 299                 maxLength => 128,
 300             },
 301             size => {
 302                 description => 'Targeted number of replicas per object',
 303                 type => 'integer',
 304                 default => 3,
 305                 optional => 1,
 306                 minimum => 1,
 307                 maximum => 7,
 308             },
 309             min_size => {
 310                 description => 'Minimum number of available replicas per object to allow I/O',
 311                 type => 'integer',
 312                 default => 2,
 313                 optional => 1,
 314                 minimum => 1,
 315                 maximum => 7,
 316             },
 317             pg_bits => {
 318                 description => "Placement group bits, used to specify the " .
 319                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 320                     "default pg num' does not work for default pools.",
 321                 type => 'integer',
 322                 default => 6,
 323                 optional => 1,
 324                 minimum => 6,
 325                 maximum => 14,
 326             },
 327             disable_cephx => {
 328                 description => "Disable cephx authentication.\n\n" .
 329                     "WARNING: cephx is a security feature protecting against " .
 330                     "man-in-the-middle attacks. Only consider disabling cephx ".
 331                     "if your network is private!",
 332                 type => 'boolean',
 333                 optional => 1,
 334                 default => 0,
 335             },
 336         },
 337     },
 338     returns => { type => 'null' },
 339     code => sub {
 340         my ($param) = @_;
 341
 342         my $version = PVE::Ceph::Tools::get_local_version(1);
 343
 344         if (!$version || $version < 14) {
 345             die "Ceph Nautilus required - please run 'pveceph install'\n";
 346         } else {
 347             PVE::Ceph::Tools::check_ceph_installed('ceph_bin');
 348         }
 349
 350         my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 351
 352         # simply load old config if it already exists
 353         PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
 354             my $cfg = cfs_read_file('ceph.conf');
 355
 356             if (!$cfg->{global}) {
 357
 358                 my $fsid;
 359                 my $uuid;
 360
 361                 UUID::generate($uuid);
 362                 UUID::unparse($uuid, $fsid);
 363
 364                 $cfg->{global} = {
 365                     'fsid' => $fsid,
 366                     'auth cluster required' => $auth,
 367                     'auth service required' => $auth,
 368                     'auth client required' => $auth,
 369                     'osd pool default size' => $param->{size} // 3,
 370                     'osd pool default min size' => $param->{min_size} // 2,
 371                     'mon allow pool delete' => 'true',
 372                 };
 373
 374                 # this does not work for default pools
 375                 #'osd pool default pg num' => $pg_num,
 376                 #'osd pool default pgp num' => $pg_num,
 377             }
 378
 379             if ($auth eq 'cephx') {
 380                 $cfg->{client}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 381             }
 382
 383             if ($param->{pg_bits}) {
 384                 $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 385                 $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 386             }
 387
 388             if ($param->{network}) {
 389                 $cfg->{global}->{'public network'} = $param->{network};
 390                 $cfg->{global}->{'cluster network'} = $param->{network};
 391             }
 392
 393             if ($param->{'cluster-network'}) {
 394                 $cfg->{global}->{'cluster network'} = $param->{'cluster-network'};
 395             }
 396
 397             cfs_write_file('ceph.conf', $cfg);
 398
 399             if ($auth eq 'cephx') {
 400                 PVE::Ceph::Tools::get_or_create_admin_keyring();
 401             }
 402             PVE::Ceph::Tools::setup_pve_symlinks();
 403         });
 404         die $@ if $@;
 405
 406         return undef;
 407     }});
 408
 409 __PACKAGE__->register_method ({
 410     name => 'stop',
 411     path => 'stop',
 412     method => 'POST',
 413     description => "Stop ceph services.",
 414     proxyto => 'node',
 415     protected => 1,
 416     permissions => {
 417         check => ['perm', '/', [ 'Sys.Modify' ]],
 418     },
 419     parameters => {
 420         additionalProperties => 0,
 421         properties => {
 422             node => get_standard_option('pve-node'),
 423             service => {
 424                 description => 'Ceph service name.',
 425                 type => 'string',
 426                 optional => 1,
 427                 default => 'ceph.target',
 428                 pattern => '(ceph|mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 429             },
 430         },
 431     },
 432     returns => { type => 'string' },
 433     code => sub {
 434         my ($param) = @_;
 435
 436         my $rpcenv = PVE::RPCEnvironment::get();
 437
 438         my $authuser = $rpcenv->get_user();
 439
 440         PVE::Ceph::Tools::check_ceph_inited();
 441
 442         my $cfg = cfs_read_file('ceph.conf');
 443         scalar(keys %$cfg) || die "no configuration\n";
 444
 445         my $worker = sub {
 446             my $upid = shift;
 447
 448             my $cmd = ['stop'];
 449             if ($param->{service}) {
 450                 push @$cmd, $param->{service};
 451             }
 452
 453             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 454         };
 455
 456         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
 457                                     $authuser, $worker);
 458     }});
 459
 460 __PACKAGE__->register_method ({
 461     name => 'start',
 462     path => 'start',
 463     method => 'POST',
 464     description => "Start ceph services.",
 465     proxyto => 'node',
 466     protected => 1,
 467     permissions => {
 468         check => ['perm', '/', [ 'Sys.Modify' ]],
 469     },
 470     parameters => {
 471         additionalProperties => 0,
 472         properties => {
 473             node => get_standard_option('pve-node'),
 474             service => {
 475                 description => 'Ceph service name.',
 476                 type => 'string',
 477                 optional => 1,
 478                 default => 'ceph.target',
 479                 pattern => '(ceph|mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 480             },
 481         },
 482     },
 483     returns => { type => 'string' },
 484     code => sub {
 485         my ($param) = @_;
 486
 487         my $rpcenv = PVE::RPCEnvironment::get();
 488
 489         my $authuser = $rpcenv->get_user();
 490
 491         PVE::Ceph::Tools::check_ceph_inited();
 492
 493         my $cfg = cfs_read_file('ceph.conf');
 494         scalar(keys %$cfg) || die "no configuration\n";
 495
 496         my $worker = sub {
 497             my $upid = shift;
 498
 499             my $cmd = ['start'];
 500             if ($param->{service}) {
 501                 push @$cmd, $param->{service};
 502             }
 503
 504             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 505         };
 506
 507         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
 508                                     $authuser, $worker);
 509     }});
 510
 511 __PACKAGE__->register_method ({
 512     name => 'restart',
 513     path => 'restart',
 514     method => 'POST',
 515     description => "Restart ceph services.",
 516     proxyto => 'node',
 517     protected => 1,
 518     permissions => {
 519         check => ['perm', '/', [ 'Sys.Modify' ]],
 520     },
 521     parameters => {
 522         additionalProperties => 0,
 523         properties => {
 524             node => get_standard_option('pve-node'),
 525             service => {
 526                 description => 'Ceph service name.',
 527                 type => 'string',
 528                 optional => 1,
 529                 default => 'ceph.target',
 530                 pattern => '(mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 531             },
 532         },
 533     },
 534     returns => { type => 'string' },
 535     code => sub {
 536         my ($param) = @_;
 537
 538         my $rpcenv = PVE::RPCEnvironment::get();
 539
 540         my $authuser = $rpcenv->get_user();
 541
 542         PVE::Ceph::Tools::check_ceph_inited();
 543
 544         my $cfg = cfs_read_file('ceph.conf');
 545         scalar(keys %$cfg) || die "no configuration\n";
 546
 547         my $worker = sub {
 548             my $upid = shift;
 549
 550             my $cmd = ['restart'];
 551             if ($param->{service}) {
 552                 push @$cmd, $param->{service};
 553             }
 554
 555             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 556         };
 557
 558         return $rpcenv->fork_worker('srvrestart', $param->{service} || 'ceph',
 559                                     $authuser, $worker);
 560     }});
 561
 562 __PACKAGE__->register_method ({
 563     name => 'status',
 564     path => 'status',
 565     method => 'GET',
 566     description => "Get ceph status.",
 567     proxyto => 'node',
 568     protected => 1,
 569     permissions => {
 570         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 571     },
 572     parameters => {
 573         additionalProperties => 0,
 574         properties => {
 575             node => get_standard_option('pve-node'),
 576         },
 577     },
 578     returns => { type => 'object' },
 579     code => sub {
 580         my ($param) = @_;
 581
 582         PVE::Ceph::Tools::check_ceph_inited();
 583
 584         return PVE::Ceph::Tools::ceph_cluster_status();
 585     }});
 586
 587 __PACKAGE__->register_method ({
 588     name => 'lspools',
 589     path => 'pools',
 590     method => 'GET',
 591     description => "List all pools.",
 592     proxyto => 'node',
 593     protected => 1,
 594     permissions => {
 595         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 596     },
 597     parameters => {
 598         additionalProperties => 0,
 599         properties => {
 600             node => get_standard_option('pve-node'),
 601         },
 602     },
 603     returns => {
 604         type => 'array',
 605         items => {
 606             type => "object",
 607             properties => {
 608                 pool => { type => 'integer', title => 'ID' },
 609                 pool_name => { type => 'string', title => 'Name' },
 610                 size => { type => 'integer', title => 'Size' },
 611                 min_size => { type => 'integer', title => 'Min Size' },
 612                 pg_num => { type => 'integer', title => 'PG Num' },
 613                 pg_autoscale_mode => { type => 'string', optional => 1, title => 'PG Autoscale Mode' },
 614                 crush_rule => { type => 'integer', title => 'Crush Rule' },
 615                 crush_rule_name => { type => 'string', title => 'Crush Rule Name' },
 616                 percent_used => { type => 'number', title => '%-Used' },
 617                 bytes_used => { type => 'integer', title => 'Used' },
 618             },
 619         },
 620         links => [ { rel => 'child', href => "{pool_name}" } ],
 621     },
 622     code => sub {
 623         my ($param) = @_;
 624
 625         PVE::Ceph::Tools::check_ceph_inited();
 626
 627         my $rados = PVE::RADOS->new();
 628
 629         my $stats = {};
 630         my $res = $rados->mon_command({ prefix => 'df' });
 631
 632         foreach my $d (@{$res->{pools}}) {
 633             next if !$d->{stats};
 634             next if !defined($d->{id});
 635             $stats->{$d->{id}} = $d->{stats};
 636         }
 637
 638         $res = $rados->mon_command({ prefix => 'osd dump' });
 639         my $rulestmp = $rados->mon_command({ prefix => 'osd crush rule dump'});
 640
 641         my $rules = {};
 642         for my $rule (@$rulestmp) {
 643             $rules->{$rule->{rule_id}} = $rule->{rule_name};
 644         }
 645
 646         my $data = [];
 647         my $attr_list = [
 648             'pool',
 649             'pool_name',
 650             'size',
 651             'min_size',
 652             'pg_num',
 653             'crush_rule',
 654             'pg_autoscale_mode',
 655         ];
 656
 657         foreach my $e (@{$res->{pools}}) {
 658             my $d = {};
 659             foreach my $attr (@$attr_list) {
 660                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
 661             }
 662
 663             if (defined($d->{crush_rule}) && defined($rules->{$d->{crush_rule}})) {
 664                 $d->{crush_rule_name} = $rules->{$d->{crush_rule}};
 665             }
 666
 667             if (my $s = $stats->{$d->{pool}}) {
 668                 $d->{bytes_used} = $s->{bytes_used};
 669                 $d->{percent_used} = $s->{percent_used};
 670             }
 671             push @$data, $d;
 672         }
 673
 674
 675         return $data;
 676     }});
 677
 678
 679 my $ceph_pool_common_options = sub {
 680     my ($nodefault) = shift;
 681     my $options = {
 682         name => {
 683             description => "The name of the pool. It must be unique.",
 684             type => 'string',
 685         },
 686         size => {
 687             description => 'Number of replicas per object',
 688             type => 'integer',
 689             default => 3,
 690             optional => 1,
 691             minimum => 1,
 692             maximum => 7,
 693         },
 694         min_size => {
 695             description => 'Minimum number of replicas per object',
 696             type => 'integer',
 697             default => 2,
 698             optional => 1,
 699             minimum => 1,
 700             maximum => 7,
 701         },
 702         pg_num => {
 703             description => "Number of placement groups.",
 704             type => 'integer',
 705             default => 128,
 706             optional => 1,
 707             minimum => 8,
 708             maximum => 32768,
 709         },
 710         crush_rule => {
 711             description => "The rule to use for mapping object placement in the cluster.",
 712             type => 'string',
 713             optional => 1,
 714         },
 715         application => {
 716             description => "The application of the pool.",
 717             default => 'rbd',
 718             type => 'string',
 719             enum => ['rbd', 'cephfs', 'rgw'],
 720             optional => 1,
 721         },
 722         pg_autoscale_mode => {
 723             description => "The automatic PG scaling mode of the pool.",
 724             type => 'string',
 725             enum => ['on', 'off', 'warn'],
 726             default => 'warn',
 727             optional => 1,
 728         },
 729     };
 730
 731     if ($nodefault) {
 732         delete $options->{$_}->{default} for keys %$options;
 733     }
 734     return $options;
 735 };
 736
 737
 738 __PACKAGE__->register_method ({
 739     name => 'createpool',
 740     path => 'pools',
 741     method => 'POST',
 742     description => "Create POOL",
 743     proxyto => 'node',
 744     protected => 1,
 745     permissions => {
 746         check => ['perm', '/', [ 'Sys.Modify' ]],
 747     },
 748     parameters => {
 749         additionalProperties => 0,
 750         properties => {
 751             node => get_standard_option('pve-node'),
 752             add_storages => {
 753                 description => "Configure VM and CT storage using the new pool.",
 754                 type => 'boolean',
 755                 optional => 1,
 756             },
 757             %{ $ceph_pool_common_options->() },
 758         },
 759     },
 760     returns => { type => 'string' },
 761     code => sub {
 762         my ($param) = @_;
 763
 764         PVE::Cluster::check_cfs_quorum();
 765         PVE::Ceph::Tools::check_ceph_configured();
 766
 767         my $pool = extract_param($param, 'name');
 768         my $node = extract_param($param, 'node');
 769         my $add_storages = extract_param($param, 'add_storages');
 770
 771         my $rpcenv = PVE::RPCEnvironment::get();
 772         my $user = $rpcenv->get_user();
 773
 774         if ($add_storages) {
 775             $rpcenv->check($user, '/storage', ['Datastore.Allocate']);
 776             die "pool name contains characters which are illegal for storage naming\n"
 777                 if !PVE::JSONSchema::parse_storage_id($pool);
 778         }
 779
 780         # pool defaults
 781         $param->{pg_num} //= 128;
 782         $param->{size} //= 3;
 783         $param->{min_size} //= 2;
 784         $param->{application} //= 'rbd';
 785         $param->{pg_autoscale_mode} //= 'warn';
 786
 787         my $worker = sub {
 788
 789             PVE::Ceph::Tools::create_pool($pool, $param);
 790
 791             if ($add_storages) {
 792                 my $err;
 793                 eval { $add_storage->($pool, "${pool}"); };
 794                 if ($@) {
 795                     warn "failed to add storage: $@";
 796                     $err = 1;
 797                 }
 798                 die "adding storage for pool '$pool' failed, check log and add manually!\n"
 799                     if $err;
 800             }
 801         };
 802
 803         return $rpcenv->fork_worker('cephcreatepool', $pool,  $user, $worker);
 804     }});
 805
 806 my $possible_flags = PVE::Ceph::Tools::get_possible_osd_flags();
 807 my $possible_flags_list = [ sort keys %$possible_flags ];
 808
 809 # FIXME: Remove with PVE 7.0
 810 __PACKAGE__->register_method ({
 811     name => 'get_flags',
 812     path => 'flags',
 813     method => 'GET',
 814     description => "get all set ceph flags",
 815     proxyto => 'node',
 816     protected => 1,
 817     permissions => {
 818         check => ['perm', '/', [ 'Sys.Audit' ]],
 819     },
 820     parameters => {
 821         additionalProperties => 0,
 822         properties => {
 823             node => get_standard_option('pve-node'),
 824         },
 825     },
 826     returns => { type => 'string' },
 827     code => sub {
 828         my ($param) = @_;
 829
 830         PVE::Ceph::Tools::check_ceph_configured();
 831
 832         my $rados = PVE::RADOS->new();
 833
 834         my $stat = $rados->mon_command({ prefix => 'osd dump' });
 835
 836         return $stat->{flags} // '';
 837     }});
 838
 839 # FIXME: Remove with PVE 7.0
 840 __PACKAGE__->register_method ({
 841     name => 'set_flag',
 842     path => 'flags/{flag}',
 843     method => 'POST',
 844     description => "Set a specific ceph flag",
 845     proxyto => 'node',
 846     protected => 1,
 847     permissions => {
 848         check => ['perm', '/', [ 'Sys.Modify' ]],
 849     },
 850     parameters => {
 851         additionalProperties => 0,
 852         properties => {
 853             node => get_standard_option('pve-node'),
 854             flag => {
 855                 description => 'The ceph flag to set',
 856                 type => 'string',
 857                 enum => $possible_flags_list,
 858             },
 859         },
 860     },
 861     returns => { type => 'null' },
 862     code => sub {
 863         my ($param) = @_;
 864
 865         PVE::Ceph::Tools::check_ceph_configured();
 866
 867         my $rados = PVE::RADOS->new();
 868
 869         $rados->mon_command({
 870             prefix => "osd set",
 871             key => $param->{flag},
 872         });
 873
 874         return undef;
 875     }});
 876
 877 __PACKAGE__->register_method ({
 878     name => 'unset_flag',
 879     path => 'flags/{flag}',
 880     method => 'DELETE',
 881     description => "Unset a ceph flag",
 882     proxyto => 'node',
 883     protected => 1,
 884     permissions => {
 885         check => ['perm', '/', [ 'Sys.Modify' ]],
 886     },
 887     parameters => {
 888         additionalProperties => 0,
 889         properties => {
 890             node => get_standard_option('pve-node'),
 891             flag => {
 892                 description => 'The ceph flag to unset',
 893                 type => 'string',
 894                 enum => $possible_flags_list,
 895             },
 896         },
 897     },
 898     returns => { type => 'null' },
 899     code => sub {
 900         my ($param) = @_;
 901
 902         PVE::Ceph::Tools::check_ceph_configured();
 903
 904         my $rados = PVE::RADOS->new();
 905
 906         $rados->mon_command({
 907             prefix => "osd unset",
 908             key => $param->{flag},
 909         });
 910
 911         return undef;
 912     }});
 913
 914 __PACKAGE__->register_method ({
 915     name => 'destroypool',
 916     path => 'pools/{name}',
 917     method => 'DELETE',
 918     description => "Destroy pool",
 919     proxyto => 'node',
 920     protected => 1,
 921     permissions => {
 922         check => ['perm', '/', [ 'Sys.Modify' ]],
 923     },
 924     parameters => {
 925         additionalProperties => 0,
 926         properties => {
 927             node => get_standard_option('pve-node'),
 928             name => {
 929                 description => "The name of the pool. It must be unique.",
 930                 type => 'string',
 931             },
 932             force => {
 933                 description => "If true, destroys pool even if in use",
 934                 type => 'boolean',
 935                 optional => 1,
 936                 default => 0,
 937             },
 938             remove_storages => {
 939                 description => "Remove all pveceph-managed storages configured for this pool",
 940                 type => 'boolean',
 941                 optional => 1,
 942                 default => 0,
 943             },
 944         },
 945     },
 946     returns => { type => 'string' },
 947     code => sub {
 948         my ($param) = @_;
 949
 950         PVE::Ceph::Tools::check_ceph_inited();
 951
 952         my $rpcenv = PVE::RPCEnvironment::get();
 953         my $user = $rpcenv->get_user();
 954         $rpcenv->check($user, '/storage', ['Datastore.Allocate'])
 955             if $param->{remove_storages};
 956
 957         my $pool = $param->{name};
 958
 959         my $worker = sub {
 960             my $storages = $get_storages->($pool);
 961
 962             # if not forced, destroy ceph pool only when no
 963             # vm disks are on it anymore
 964             if (!$param->{force}) {
 965                 my $storagecfg = PVE::Storage::config();
 966                 foreach my $storeid (keys %$storages) {
 967                     my $storage = $storages->{$storeid};
 968
 969                     # check if any vm disks are on the pool
 970                     print "checking storage '$storeid' for RBD images..\n";
 971                     my $res = PVE::Storage::vdisk_list($storagecfg, $storeid);
 972                     die "ceph pool '$pool' still in use by storage '$storeid'\n"
 973                         if @{$res->{$storeid}} != 0;
 974                 }
 975             }
 976
 977             PVE::Ceph::Tools::destroy_pool($pool);
 978
 979             if ($param->{remove_storages}) {
 980                 my $err;
 981                 foreach my $storeid (keys %$storages) {
 982                     # skip external clusters, not managed by pveceph
 983                     next if $storages->{$storeid}->{monhost};
 984                     eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
 985                     if ($@) {
 986                         warn "failed to remove storage '$storeid': $@\n";
 987                         $err = 1;
 988                     }
 989                 }
 990                 die "failed to remove (some) storages - check log and remove manually!\n"
 991                     if $err;
 992             }
 993         };
 994         return $rpcenv->fork_worker('cephdestroypool', $pool,  $user, $worker);
 995     }});
 996
 997
 998 __PACKAGE__->register_method ({
 999     name => 'setpool',
1000     path => 'pools/{name}',
1001     method => 'PUT',
1002     description => "Change POOL settings",
1003     proxyto => 'node',
1004     protected => 1,
1005     permissions => {
1006         check => ['perm', '/', [ 'Sys.Modify' ]],
1007     },
1008     parameters => {
1009         additionalProperties => 0,
1010         properties => {
1011             node => get_standard_option('pve-node'),
1012             %{ $ceph_pool_common_options->('nodefault') },
1013         },
1014     },
1015     returns => { type => 'string' },
1016     code => sub {
1017         my ($param) = @_;
1018
1019         PVE::Ceph::Tools::check_ceph_configured();
1020
1021         my $rpcenv = PVE::RPCEnvironment::get();
1022         my $authuser = $rpcenv->get_user();
1023
1024         my $pool = $param->{name};
1025         my $ceph_param = \%$param;
1026         for my $item ('name', 'node') {
1027             # not ceph parameters
1028             delete $ceph_param->{$item};
1029         }
1030
1031         my $worker = sub {
1032             PVE::Ceph::Tools::set_pool($pool, $ceph_param);
1033         };
1034
1035         return $rpcenv->fork_worker('cephsetpool', $pool,  $authuser, $worker);
1036     }});
1037
1038
1039 __PACKAGE__->register_method ({
1040     name => 'crush',
1041     path => 'crush',
1042     method => 'GET',
1043     description => "Get OSD crush map",
1044     proxyto => 'node',
1045     protected => 1,
1046     permissions => {
1047         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1048     },
1049     parameters => {
1050         additionalProperties => 0,
1051         properties => {
1052             node => get_standard_option('pve-node'),
1053         },
1054     },
1055     returns => { type => 'string' },
1056     code => sub {
1057         my ($param) = @_;
1058
1059         PVE::Ceph::Tools::check_ceph_inited();
1060
1061         # this produces JSON (difficult to read for the user)
1062         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
1063
1064         my $txt = '';
1065
1066         my $mapfile = "/var/tmp/ceph-crush.map.$$";
1067         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
1068
1069         my $rados = PVE::RADOS->new();
1070
1071         eval {
1072             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
1073             file_set_contents($mapfile, $bindata);
1074             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
1075             $txt = file_get_contents($mapdata);
1076         };
1077         my $err = $@;
1078
1079         unlink $mapfile;
1080         unlink $mapdata;
1081
1082         die $err if $err;
1083
1084         return $txt;
1085     }});
1086
1087 __PACKAGE__->register_method({
1088     name => 'log',
1089     path => 'log',
1090     method => 'GET',
1091     description => "Read ceph log",
1092     proxyto => 'node',
1093     permissions => {
1094         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
1095     },
1096     protected => 1,
1097     parameters => {
1098         additionalProperties => 0,
1099         properties => {
1100             node => get_standard_option('pve-node'),
1101             start => {
1102                 type => 'integer',
1103                 minimum => 0,
1104                 optional => 1,
1105             },
1106             limit => {
1107                 type => 'integer',
1108                 minimum => 0,
1109                 optional => 1,
1110             },
1111         },
1112     },
1113     returns => {
1114         type => 'array',
1115         items => {
1116             type => "object",
1117             properties => {
1118                 n => {
1119                   description=>  "Line number",
1120                   type=> 'integer',
1121                 },
1122                 t => {
1123                   description=>  "Line text",
1124                   type => 'string',
1125                 }
1126             }
1127         }
1128     },
1129     code => sub {
1130         my ($param) = @_;
1131
1132         PVE::Ceph::Tools::check_ceph_inited();
1133
1134         my $rpcenv = PVE::RPCEnvironment::get();
1135         my $user = $rpcenv->get_user();
1136         my $node = $param->{node};
1137
1138         my $logfile = "/var/log/ceph/ceph.log";
1139         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
1140
1141         $rpcenv->set_result_attrib('total', $count);
1142
1143         return $lines;
1144     }});
1145
1146 __PACKAGE__->register_method ({
1147     name => 'rules',
1148     path => 'rules',
1149     method => 'GET',
1150     description => "List ceph rules.",
1151     proxyto => 'node',
1152     protected => 1,
1153     permissions => {
1154         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1155     },
1156     parameters => {
1157         additionalProperties => 0,
1158         properties => {
1159             node => get_standard_option('pve-node'),
1160         },
1161     },
1162     returns => {
1163         type => 'array',
1164         items => {
1165             type => "object",
1166             properties => {},
1167         },
1168         links => [ { rel => 'child', href => "{name}" } ],
1169     },
1170     code => sub {
1171         my ($param) = @_;
1172
1173         PVE::Ceph::Tools::check_ceph_inited();
1174
1175         my $rados = PVE::RADOS->new();
1176
1177         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
1178
1179         my $res = [];
1180
1181         foreach my $rule (@$rules) {
1182             push @$res, { name => $rule };
1183         }
1184
1185         return $res;
1186     }});
1187
1188 1;