PVE/API2/Ceph.pm

   1 package PVE::API2::Ceph;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use Net::IP;
   8 use UUID;
   9
  10 use PVE::Ceph::Tools;
  11 use PVE::Ceph::Services;
  12 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  13 use PVE::JSONSchema qw(get_standard_option);
  14 use PVE::Network;
  15 use PVE::RADOS;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::Storage;
  19 use PVE::Tools qw(run_command file_get_contents file_set_contents);
  20
  21 use PVE::API2::Ceph::OSD;
  22 use PVE::API2::Ceph::FS;
  23 use PVE::API2::Ceph::MDS;
  24 use PVE::API2::Ceph::MGR;
  25 use PVE::API2::Ceph::MON;
  26 use PVE::API2::Storage::Config;
  27
  28 use base qw(PVE::RESTHandler);
  29
  30 my $pve_osd_default_journal_size = 1024*5;
  31
  32 __PACKAGE__->register_method ({
  33     subclass => "PVE::API2::Ceph::OSD",
  34     path => 'osd',
  35 });
  36
  37 __PACKAGE__->register_method ({
  38     subclass => "PVE::API2::Ceph::MDS",
  39     path => 'mds',
  40 });
  41
  42 __PACKAGE__->register_method ({
  43     subclass => "PVE::API2::Ceph::MGR",
  44     path => 'mgr',
  45 });
  46
  47 __PACKAGE__->register_method ({
  48     subclass => "PVE::API2::Ceph::MON",
  49     path => 'mon',
  50 });
  51
  52 __PACKAGE__->register_method ({
  53     subclass => "PVE::API2::Ceph::FS",
  54     path => 'fs',
  55 });
  56
  57 __PACKAGE__->register_method ({
  58     name => 'index',
  59     path => '',
  60     method => 'GET',
  61     description => "Directory index.",
  62     permissions => { user => 'all' },
  63     permissions => {
  64         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  65     },
  66     parameters => {
  67         additionalProperties => 0,
  68         properties => {
  69             node => get_standard_option('pve-node'),
  70         },
  71     },
  72     returns => {
  73         type => 'array',
  74         items => {
  75             type => "object",
  76             properties => {},
  77         },
  78         links => [ { rel => 'child', href => "{name}" } ],
  79     },
  80     code => sub {
  81         my ($param) = @_;
  82
  83         my $result = [
  84             { name => 'init' },
  85             { name => 'mon' },
  86             { name => 'osd' },
  87             { name => 'pools' },
  88             { name => 'fs' },
  89             { name => 'mds' },
  90             { name => 'stop' },
  91             { name => 'start' },
  92             { name => 'restart' },
  93             { name => 'status' },
  94             { name => 'crush' },
  95             { name => 'config' },
  96             { name => 'log' },
  97             { name => 'disks' },
  98             { name => 'flags' },
  99             { name => 'rules' },
 100         ];
 101
 102         return $result;
 103     }});
 104
 105 __PACKAGE__->register_method ({
 106     name => 'disks',
 107     path => 'disks',
 108     method => 'GET',
 109     description => "List local disks.",
 110     proxyto => 'node',
 111     protected => 1,
 112     permissions => {
 113         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 114     },
 115     parameters => {
 116         additionalProperties => 0,
 117         properties => {
 118             node => get_standard_option('pve-node'),
 119             type => {
 120                 description => "Only list specific types of disks.",
 121                 type => 'string',
 122                 enum => ['unused', 'journal_disks'],
 123                 optional => 1,
 124             },
 125         },
 126     },
 127     returns => {
 128         type => 'array',
 129         items => {
 130             type => "object",
 131             properties => {
 132                 dev => { type => 'string' },
 133                 used => { type => 'string', optional => 1 },
 134                 gpt => { type => 'boolean' },
 135                 size => { type => 'integer' },
 136                 osdid => { type => 'integer' },
 137                 vendor =>  { type => 'string', optional => 1 },
 138                 model =>  { type => 'string', optional => 1 },
 139                 serial =>  { type => 'string', optional => 1 },
 140             },
 141         },
 142         # links => [ { rel => 'child', href => "{}" } ],
 143     },
 144     code => sub {
 145         my ($param) = @_;
 146
 147         PVE::Ceph::Tools::check_ceph_inited();
 148
 149         my $disks = PVE::Diskmanage::get_disks(undef, 1);
 150
 151         my $res = [];
 152         foreach my $dev (keys %$disks) {
 153             my $d = $disks->{$dev};
 154             if ($param->{type}) {
 155                 if ($param->{type} eq 'journal_disks') {
 156                     next if $d->{osdid} >= 0;
 157                     next if !$d->{gpt};
 158                 } elsif ($param->{type} eq 'unused') {
 159                     next if $d->{used};
 160                 } else {
 161                     die "internal error"; # should not happen
 162                 }
 163             }
 164
 165             $d->{dev} = "/dev/$dev";
 166             push @$res, $d;
 167         }
 168
 169         return $res;
 170     }});
 171
 172 __PACKAGE__->register_method ({
 173     name => 'config',
 174     path => 'config',
 175     method => 'GET',
 176     proxyto => 'node',
 177     permissions => {
 178         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 179     },
 180     description => "Get Ceph configuration.",
 181     parameters => {
 182         additionalProperties => 0,
 183         properties => {
 184             node => get_standard_option('pve-node'),
 185         },
 186     },
 187     returns => { type => 'string' },
 188     code => sub {
 189         my ($param) = @_;
 190
 191         PVE::Ceph::Tools::check_ceph_inited();
 192
 193         my $path = PVE::Ceph::Tools::get_config('pve_ceph_cfgpath');
 194         return file_get_contents($path);
 195
 196     }});
 197
 198 my $add_storage = sub {
 199     my ($pool, $storeid) = @_;
 200
 201     my $storage_params = {
 202         type => 'rbd',
 203         pool => $pool,
 204         storage => $storeid,
 205         krbd => 0,
 206         content => 'rootdir,images',
 207     };
 208
 209     PVE::API2::Storage::Config->create($storage_params);
 210 };
 211
 212 my $get_storages = sub {
 213     my ($pool) = @_;
 214
 215     my $cfg = PVE::Storage::config();
 216
 217     my $storages = $cfg->{ids};
 218     my $res = {};
 219     foreach my $storeid (keys %$storages) {
 220         my $curr = $storages->{$storeid};
 221         $res->{$storeid} = $storages->{$storeid}
 222             if $curr->{type} eq 'rbd' && $pool eq $curr->{pool};
 223     }
 224
 225     return $res;
 226 };
 227
 228 __PACKAGE__->register_method ({
 229     name => 'init',
 230     path => 'init',
 231     method => 'POST',
 232     description => "Create initial ceph default configuration and setup symlinks.",
 233     proxyto => 'node',
 234     protected => 1,
 235     permissions => {
 236         check => ['perm', '/', [ 'Sys.Modify' ]],
 237     },
 238     parameters => {
 239         additionalProperties => 0,
 240         properties => {
 241             node => get_standard_option('pve-node'),
 242             network => {
 243                 description => "Use specific network for all ceph related traffic",
 244                 type => 'string', format => 'CIDR',
 245                 optional => 1,
 246                 maxLength => 128,
 247             },
 248             'cluster-network' => {
 249                 description => "Declare a separate cluster network, OSDs will route" .
 250                     "heartbeat, object replication and recovery traffic over it",
 251                 type => 'string', format => 'CIDR',
 252                 requires => 'network',
 253                 optional => 1,
 254                 maxLength => 128,
 255             },
 256             size => {
 257                 description => 'Targeted number of replicas per object',
 258                 type => 'integer',
 259                 default => 3,
 260                 optional => 1,
 261                 minimum => 1,
 262                 maximum => 7,
 263             },
 264             min_size => {
 265                 description => 'Minimum number of available replicas per object to allow I/O',
 266                 type => 'integer',
 267                 default => 2,
 268                 optional => 1,
 269                 minimum => 1,
 270                 maximum => 7,
 271             },
 272             pg_bits => {
 273                 description => "Placement group bits, used to specify the " .
 274                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 275                     "default pg num' does not work for default pools.",
 276                 type => 'integer',
 277                 default => 6,
 278                 optional => 1,
 279                 minimum => 6,
 280                 maximum => 14,
 281             },
 282             disable_cephx => {
 283                 description => "Disable cephx authentication.\n\n" .
 284                     "WARNING: cephx is a security feature protecting against " .
 285                     "man-in-the-middle attacks. Only consider disabling cephx ".
 286                     "if your network is private!",
 287                 type => 'boolean',
 288                 optional => 1,
 289                 default => 0,
 290             },
 291         },
 292     },
 293     returns => { type => 'null' },
 294     code => sub {
 295         my ($param) = @_;
 296
 297         my $version = PVE::Ceph::Tools::get_local_version(1);
 298
 299         if (!$version || $version < 14) {
 300             die "Ceph Nautilus required - please run 'pveceph install'\n";
 301         } else {
 302             PVE::Ceph::Tools::check_ceph_installed('ceph_bin');
 303         }
 304
 305         # simply load old config if it already exists
 306         PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
 307             my $cfg = cfs_read_file('ceph.conf');
 308
 309             if (!$cfg->{global}) {
 310
 311                 my $fsid;
 312                 my $uuid;
 313
 314                 UUID::generate($uuid);
 315                 UUID::unparse($uuid, $fsid);
 316
 317                 my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 318
 319                 $cfg->{global} = {
 320                     'fsid' => $fsid,
 321                     'auth cluster required' => $auth,
 322                     'auth service required' => $auth,
 323                     'auth client required' => $auth,
 324                     'osd pool default size' => $param->{size} // 3,
 325                     'osd pool default min size' => $param->{min_size} // 2,
 326                     'mon allow pool delete' => 'true',
 327                 };
 328
 329                 # this does not work for default pools
 330                 #'osd pool default pg num' => $pg_num,
 331                 #'osd pool default pgp num' => $pg_num,
 332             }
 333
 334             if ($auth eq 'cephx') {
 335                 $cfg->{client}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 336             }
 337
 338             if ($param->{pg_bits}) {
 339                 $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 340                 $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 341             }
 342
 343             if ($param->{network}) {
 344                 $cfg->{global}->{'public network'} = $param->{network};
 345                 $cfg->{global}->{'cluster network'} = $param->{network};
 346             }
 347
 348             if ($param->{'cluster-network'}) {
 349                 $cfg->{global}->{'cluster network'} = $param->{'cluster-network'};
 350             }
 351
 352             cfs_write_file('ceph.conf', $cfg);
 353
 354             if ($auth eq 'cephx') {
 355                 PVE::Ceph::Tools::get_or_create_admin_keyring();
 356             }
 357             PVE::Ceph::Tools::setup_pve_symlinks();
 358         });
 359
 360         return undef;
 361     }});
 362
 363 __PACKAGE__->register_method ({
 364     name => 'stop',
 365     path => 'stop',
 366     method => 'POST',
 367     description => "Stop ceph services.",
 368     proxyto => 'node',
 369     protected => 1,
 370     permissions => {
 371         check => ['perm', '/', [ 'Sys.Modify' ]],
 372     },
 373     parameters => {
 374         additionalProperties => 0,
 375         properties => {
 376             node => get_standard_option('pve-node'),
 377             service => {
 378                 description => 'Ceph service name.',
 379                 type => 'string',
 380                 optional => 1,
 381                 default => 'ceph.target',
 382                 pattern => '(ceph|mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
 383             },
 384         },
 385     },
 386     returns => { type => 'string' },
 387     code => sub {
 388         my ($param) = @_;
 389
 390         my $rpcenv = PVE::RPCEnvironment::get();
 391
 392         my $authuser = $rpcenv->get_user();
 393
 394         PVE::Ceph::Tools::check_ceph_inited();
 395
 396         my $cfg = cfs_read_file('ceph.conf');
 397         scalar(keys %$cfg) || die "no configuration\n";
 398
 399         my $worker = sub {
 400             my $upid = shift;
 401
 402             my $cmd = ['stop'];
 403             if ($param->{service}) {
 404                 push @$cmd, $param->{service};
 405             }
 406
 407             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 408         };
 409
 410         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
 411                                     $authuser, $worker);
 412     }});
 413
 414 __PACKAGE__->register_method ({
 415     name => 'start',
 416     path => 'start',
 417     method => 'POST',
 418     description => "Start ceph services.",
 419     proxyto => 'node',
 420     protected => 1,
 421     permissions => {
 422         check => ['perm', '/', [ 'Sys.Modify' ]],
 423     },
 424     parameters => {
 425         additionalProperties => 0,
 426         properties => {
 427             node => get_standard_option('pve-node'),
 428             service => {
 429                 description => 'Ceph service name.',
 430                 type => 'string',
 431                 optional => 1,
 432                 default => 'ceph.target',
 433                 pattern => '(ceph|mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
 434             },
 435         },
 436     },
 437     returns => { type => 'string' },
 438     code => sub {
 439         my ($param) = @_;
 440
 441         my $rpcenv = PVE::RPCEnvironment::get();
 442
 443         my $authuser = $rpcenv->get_user();
 444
 445         PVE::Ceph::Tools::check_ceph_inited();
 446
 447         my $cfg = cfs_read_file('ceph.conf');
 448         scalar(keys %$cfg) || die "no configuration\n";
 449
 450         my $worker = sub {
 451             my $upid = shift;
 452
 453             my $cmd = ['start'];
 454             if ($param->{service}) {
 455                 push @$cmd, $param->{service};
 456             }
 457
 458             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 459         };
 460
 461         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
 462                                     $authuser, $worker);
 463     }});
 464
 465 __PACKAGE__->register_method ({
 466     name => 'restart',
 467     path => 'restart',
 468     method => 'POST',
 469     description => "Restart ceph services.",
 470     proxyto => 'node',
 471     protected => 1,
 472     permissions => {
 473         check => ['perm', '/', [ 'Sys.Modify' ]],
 474     },
 475     parameters => {
 476         additionalProperties => 0,
 477         properties => {
 478             node => get_standard_option('pve-node'),
 479             service => {
 480                 description => 'Ceph service name.',
 481                 type => 'string',
 482                 optional => 1,
 483                 default => 'ceph.target',
 484                 pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9\-]{1,32}',
 485             },
 486         },
 487     },
 488     returns => { type => 'string' },
 489     code => sub {
 490         my ($param) = @_;
 491
 492         my $rpcenv = PVE::RPCEnvironment::get();
 493
 494         my $authuser = $rpcenv->get_user();
 495
 496         PVE::Ceph::Tools::check_ceph_inited();
 497
 498         my $cfg = cfs_read_file('ceph.conf');
 499         scalar(keys %$cfg) || die "no configuration\n";
 500
 501         my $worker = sub {
 502             my $upid = shift;
 503
 504             my $cmd = ['restart'];
 505             if ($param->{service}) {
 506                 push @$cmd, $param->{service};
 507             }
 508
 509             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 510         };
 511
 512         return $rpcenv->fork_worker('srvrestart', $param->{service} || 'ceph',
 513                                     $authuser, $worker);
 514     }});
 515
 516 __PACKAGE__->register_method ({
 517     name => 'status',
 518     path => 'status',
 519     method => 'GET',
 520     description => "Get ceph status.",
 521     proxyto => 'node',
 522     protected => 1,
 523     permissions => {
 524         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 525     },
 526     parameters => {
 527         additionalProperties => 0,
 528         properties => {
 529             node => get_standard_option('pve-node'),
 530         },
 531     },
 532     returns => { type => 'object' },
 533     code => sub {
 534         my ($param) = @_;
 535
 536         PVE::Ceph::Tools::check_ceph_inited();
 537
 538         my $rados = PVE::RADOS->new();
 539         my $status = $rados->mon_command({ prefix => 'status' });
 540         $status->{health} = $rados->mon_command({ prefix => 'health', detail => 'detail' });
 541         return $status;
 542     }});
 543
 544 __PACKAGE__->register_method ({
 545     name => 'lspools',
 546     path => 'pools',
 547     method => 'GET',
 548     description => "List all pools.",
 549     proxyto => 'node',
 550     protected => 1,
 551     permissions => {
 552         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 553     },
 554     parameters => {
 555         additionalProperties => 0,
 556         properties => {
 557             node => get_standard_option('pve-node'),
 558         },
 559     },
 560     returns => {
 561         type => 'array',
 562         items => {
 563             type => "object",
 564             properties => {
 565                 pool => { type => 'integer' },
 566                 pool_name => { type => 'string' },
 567                 size => { type => 'integer' },
 568             },
 569         },
 570         links => [ { rel => 'child', href => "{pool_name}" } ],
 571     },
 572     code => sub {
 573         my ($param) = @_;
 574
 575         PVE::Ceph::Tools::check_ceph_inited();
 576
 577         my $rados = PVE::RADOS->new();
 578
 579         my $stats = {};
 580         my $res = $rados->mon_command({ prefix => 'df' });
 581
 582         foreach my $d (@{$res->{pools}}) {
 583             next if !$d->{stats};
 584             next if !defined($d->{id});
 585             $stats->{$d->{id}} = $d->{stats};
 586         }
 587
 588         $res = $rados->mon_command({ prefix => 'osd dump' });
 589         my $rulestmp = $rados->mon_command({ prefix => 'osd crush rule dump'});
 590
 591         my $rules = {};
 592         for my $rule (@$rulestmp) {
 593             $rules->{$rule->{rule_id}} = $rule->{rule_name};
 594         }
 595
 596         my $data = [];
 597         foreach my $e (@{$res->{pools}}) {
 598             my $d = {};
 599             foreach my $attr (qw(pool pool_name size min_size pg_num crush_rule)) {
 600                 $d->{$attr} = $e->{$attr} if defined($e->{$attr});
 601             }
 602
 603             if (defined($d->{crush_rule}) && defined($rules->{$d->{crush_rule}})) {
 604                 $d->{crush_rule_name} = $rules->{$d->{crush_rule}};
 605             }
 606
 607             if (my $s = $stats->{$d->{pool}}) {
 608                 $d->{bytes_used} = $s->{bytes_used};
 609                 $d->{percent_used} = $s->{percent_used};
 610             }
 611             push @$data, $d;
 612         }
 613
 614
 615         return $data;
 616     }});
 617
 618 __PACKAGE__->register_method ({
 619     name => 'createpool',
 620     path => 'pools',
 621     method => 'POST',
 622     description => "Create POOL",
 623     proxyto => 'node',
 624     protected => 1,
 625     permissions => {
 626         check => ['perm', '/', [ 'Sys.Modify' ]],
 627     },
 628     parameters => {
 629         additionalProperties => 0,
 630         properties => {
 631             node => get_standard_option('pve-node'),
 632             name => {
 633                 description => "The name of the pool. It must be unique.",
 634                 type => 'string',
 635             },
 636             size => {
 637                 description => 'Number of replicas per object',
 638                 type => 'integer',
 639                 default => 3,
 640                 optional => 1,
 641                 minimum => 1,
 642                 maximum => 7,
 643             },
 644             min_size => {
 645                 description => 'Minimum number of replicas per object',
 646                 type => 'integer',
 647                 default => 2,
 648                 optional => 1,
 649                 minimum => 1,
 650                 maximum => 7,
 651             },
 652             pg_num => {
 653                 description => "Number of placement groups.",
 654                 type => 'integer',
 655                 default => 128,
 656                 optional => 1,
 657                 minimum => 8,
 658                 maximum => 32768,
 659             },
 660             crush_rule => {
 661                 description => "The rule to use for mapping object placement in the cluster.",
 662                 type => 'string',
 663                 optional => 1,
 664             },
 665             application => {
 666                 description => "The application of the pool, 'rbd' by default.",
 667                 type => 'string',
 668                 enum => ['rbd', 'cephfs', 'rgw'],
 669                 optional => 1,
 670             },
 671             add_storages => {
 672                 description => "Configure VM and CT storage using the new pool.",
 673                 type => 'boolean',
 674                 optional => 1,
 675             },
 676         },
 677     },
 678     returns => { type => 'string' },
 679     code => sub {
 680         my ($param) = @_;
 681
 682         PVE::Cluster::check_cfs_quorum();
 683         PVE::Ceph::Tools::check_ceph_inited();
 684
 685         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
 686
 687         die "not fully configured - missing '$pve_ckeyring_path'\n"
 688             if ! -f $pve_ckeyring_path;
 689
 690         my $pool = $param->{name};
 691         my $rpcenv = PVE::RPCEnvironment::get();
 692         my $user = $rpcenv->get_user();
 693
 694         if ($param->{add_storages}) {
 695             $rpcenv->check($user, '/storage', ['Datastore.Allocate']);
 696             die "pool name contains characters which are illegal for storage naming\n"
 697                 if !PVE::JSONSchema::parse_storage_id($pool);
 698         }
 699
 700         my $pg_num = $param->{pg_num} || 128;
 701         my $size = $param->{size} || 3;
 702         my $min_size = $param->{min_size} || 2;
 703         my $application = $param->{application} // 'rbd';
 704
 705         my $worker = sub {
 706
 707             PVE::Ceph::Tools::create_pool($pool, $param);
 708
 709             if ($param->{add_storages}) {
 710                 my $err;
 711                 eval { $add_storage->($pool, "${pool}"); };
 712                 if ($@) {
 713                     warn "failed to add storage: $@";
 714                     $err = 1;
 715                 }
 716                 die "adding storage for pool '$pool' failed, check log and add manually!\n"
 717                     if $err;
 718             }
 719         };
 720
 721         return $rpcenv->fork_worker('cephcreatepool', $pool,  $user, $worker);
 722     }});
 723
 724 __PACKAGE__->register_method ({
 725     name => 'get_flags',
 726     path => 'flags',
 727     method => 'GET',
 728     description => "get all set ceph flags",
 729     proxyto => 'node',
 730     protected => 1,
 731     permissions => {
 732         check => ['perm', '/', [ 'Sys.Audit' ]],
 733     },
 734     parameters => {
 735         additionalProperties => 0,
 736         properties => {
 737             node => get_standard_option('pve-node'),
 738         },
 739     },
 740     returns => { type => 'string' },
 741     code => sub {
 742         my ($param) = @_;
 743
 744         PVE::Ceph::Tools::check_ceph_inited();
 745
 746         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
 747
 748         die "not fully configured - missing '$pve_ckeyring_path'\n"
 749             if ! -f $pve_ckeyring_path;
 750
 751         my $rados = PVE::RADOS->new();
 752
 753         my $stat = $rados->mon_command({ prefix => 'osd dump' });
 754
 755         return $stat->{flags} // '';
 756     }});
 757
 758 __PACKAGE__->register_method ({
 759     name => 'set_flag',
 760     path => 'flags/{flag}',
 761     method => 'POST',
 762     description => "Set a ceph flag",
 763     proxyto => 'node',
 764     protected => 1,
 765     permissions => {
 766         check => ['perm', '/', [ 'Sys.Modify' ]],
 767     },
 768     parameters => {
 769         additionalProperties => 0,
 770         properties => {
 771             node => get_standard_option('pve-node'),
 772             flag => {
 773                 description => 'The ceph flag to set/unset',
 774                 type => 'string',
 775                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
 776             },
 777         },
 778     },
 779     returns => { type => 'null' },
 780     code => sub {
 781         my ($param) = @_;
 782
 783         PVE::Ceph::Tools::check_ceph_inited();
 784
 785         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
 786
 787         die "not fully configured - missing '$pve_ckeyring_path'\n"
 788             if ! -f $pve_ckeyring_path;
 789
 790         my $set = $param->{set} // !$param->{unset};
 791         my $rados = PVE::RADOS->new();
 792
 793         $rados->mon_command({
 794             prefix => "osd set",
 795             key => $param->{flag},
 796         });
 797
 798         return undef;
 799     }});
 800
 801 __PACKAGE__->register_method ({
 802     name => 'unset_flag',
 803     path => 'flags/{flag}',
 804     method => 'DELETE',
 805     description => "Unset a ceph flag",
 806     proxyto => 'node',
 807     protected => 1,
 808     permissions => {
 809         check => ['perm', '/', [ 'Sys.Modify' ]],
 810     },
 811     parameters => {
 812         additionalProperties => 0,
 813         properties => {
 814             node => get_standard_option('pve-node'),
 815             flag => {
 816                 description => 'The ceph flag to set/unset',
 817                 type => 'string',
 818                 enum => [ 'full', 'pause', 'noup', 'nodown', 'noout', 'noin', 'nobackfill', 'norebalance', 'norecover', 'noscrub', 'nodeep-scrub', 'notieragent'],
 819             },
 820         },
 821     },
 822     returns => { type => 'null' },
 823     code => sub {
 824         my ($param) = @_;
 825
 826         PVE::Ceph::Tools::check_ceph_inited();
 827
 828         my $pve_ckeyring_path = PVE::Ceph::Tools::get_config('pve_ckeyring_path');
 829
 830         die "not fully configured - missing '$pve_ckeyring_path'\n"
 831             if ! -f $pve_ckeyring_path;
 832
 833         my $set = $param->{set} // !$param->{unset};
 834         my $rados = PVE::RADOS->new();
 835
 836         $rados->mon_command({
 837             prefix => "osd unset",
 838             key => $param->{flag},
 839         });
 840
 841         return undef;
 842     }});
 843
 844 __PACKAGE__->register_method ({
 845     name => 'destroypool',
 846     path => 'pools/{name}',
 847     method => 'DELETE',
 848     description => "Destroy pool",
 849     proxyto => 'node',
 850     protected => 1,
 851     permissions => {
 852         check => ['perm', '/', [ 'Sys.Modify' ]],
 853     },
 854     parameters => {
 855         additionalProperties => 0,
 856         properties => {
 857             node => get_standard_option('pve-node'),
 858             name => {
 859                 description => "The name of the pool. It must be unique.",
 860                 type => 'string',
 861             },
 862             force => {
 863                 description => "If true, destroys pool even if in use",
 864                 type => 'boolean',
 865                 optional => 1,
 866                 default => 0,
 867             },
 868             remove_storages => {
 869                 description => "Remove all pveceph-managed storages configured for this pool",
 870                 type => 'boolean',
 871                 optional => 1,
 872                 default => 0,
 873             },
 874         },
 875     },
 876     returns => { type => 'string' },
 877     code => sub {
 878         my ($param) = @_;
 879
 880         PVE::Ceph::Tools::check_ceph_inited();
 881
 882         my $rpcenv = PVE::RPCEnvironment::get();
 883         my $user = $rpcenv->get_user();
 884         $rpcenv->check($user, '/storage', ['Datastore.Allocate'])
 885             if $param->{remove_storages};
 886
 887         my $pool = $param->{name};
 888
 889         my $worker = sub {
 890             my $storages = $get_storages->($pool);
 891
 892             # if not forced, destroy ceph pool only when no
 893             # vm disks are on it anymore
 894             if (!$param->{force}) {
 895                 my $storagecfg = PVE::Storage::config();
 896                 foreach my $storeid (keys %$storages) {
 897                     my $storage = $storages->{$storeid};
 898
 899                     # check if any vm disks are on the pool
 900                     print "checking storage '$storeid' for RBD images..\n";
 901                     my $res = PVE::Storage::vdisk_list($storagecfg, $storeid);
 902                     die "ceph pool '$pool' still in use by storage '$storeid'\n"
 903                         if @{$res->{$storeid}} != 0;
 904                 }
 905             }
 906
 907             PVE::Ceph::Tools::destroy_pool($pool);
 908
 909             if ($param->{remove_storages}) {
 910                 my $err;
 911                 foreach my $storeid (keys %$storages) {
 912                     # skip external clusters, not managed by pveceph
 913                     next if $storages->{$storeid}->{monhost};
 914                     eval { PVE::API2::Storage::Config->delete({storage => $storeid}) };
 915                     if ($@) {
 916                         warn "failed to remove storage '$storeid': $@\n";
 917                         $err = 1;
 918                     }
 919                 }
 920                 die "failed to remove (some) storages - check log and remove manually!\n"
 921                     if $err;
 922             }
 923         };
 924         return $rpcenv->fork_worker('cephdestroypool', $pool,  $user, $worker);
 925     }});
 926
 927
 928 __PACKAGE__->register_method ({
 929     name => 'crush',
 930     path => 'crush',
 931     method => 'GET',
 932     description => "Get OSD crush map",
 933     proxyto => 'node',
 934     protected => 1,
 935     permissions => {
 936         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 937     },
 938     parameters => {
 939         additionalProperties => 0,
 940         properties => {
 941             node => get_standard_option('pve-node'),
 942         },
 943     },
 944     returns => { type => 'string' },
 945     code => sub {
 946         my ($param) = @_;
 947
 948         PVE::Ceph::Tools::check_ceph_inited();
 949
 950         # this produces JSON (difficult to read for the user)
 951         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
 952
 953         my $txt = '';
 954
 955         my $mapfile = "/var/tmp/ceph-crush.map.$$";
 956         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
 957
 958         my $rados = PVE::RADOS->new();
 959
 960         eval {
 961             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
 962             file_set_contents($mapfile, $bindata);
 963             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
 964             $txt = file_get_contents($mapdata);
 965         };
 966         my $err = $@;
 967
 968         unlink $mapfile;
 969         unlink $mapdata;
 970
 971         die $err if $err;
 972
 973         return $txt;
 974     }});
 975
 976 __PACKAGE__->register_method({
 977     name => 'log',
 978     path => 'log',
 979     method => 'GET',
 980     description => "Read ceph log",
 981     proxyto => 'node',
 982     permissions => {
 983         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
 984     },
 985     protected => 1,
 986     parameters => {
 987         additionalProperties => 0,
 988         properties => {
 989             node => get_standard_option('pve-node'),
 990             start => {
 991                 type => 'integer',
 992                 minimum => 0,
 993                 optional => 1,
 994             },
 995             limit => {
 996                 type => 'integer',
 997                 minimum => 0,
 998                 optional => 1,
 999             },
1000         },
1001     },
1002     returns => {
1003         type => 'array',
1004         items => {
1005             type => "object",
1006             properties => {
1007                 n => {
1008                   description=>  "Line number",
1009                   type=> 'integer',
1010                 },
1011                 t => {
1012                   description=>  "Line text",
1013                   type => 'string',
1014                 }
1015             }
1016         }
1017     },
1018     code => sub {
1019         my ($param) = @_;
1020
1021         PVE::Ceph::Tools::check_ceph_inited();
1022
1023         my $rpcenv = PVE::RPCEnvironment::get();
1024         my $user = $rpcenv->get_user();
1025         my $node = $param->{node};
1026
1027         my $logfile = "/var/log/ceph/ceph.log";
1028         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
1029
1030         $rpcenv->set_result_attrib('total', $count);
1031
1032         return $lines;
1033     }});
1034
1035 __PACKAGE__->register_method ({
1036     name => 'rules',
1037     path => 'rules',
1038     method => 'GET',
1039     description => "List ceph rules.",
1040     proxyto => 'node',
1041     protected => 1,
1042     permissions => {
1043         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
1044     },
1045     parameters => {
1046         additionalProperties => 0,
1047         properties => {
1048             node => get_standard_option('pve-node'),
1049         },
1050     },
1051     returns => {
1052         type => 'array',
1053         items => {
1054             type => "object",
1055             properties => {},
1056         },
1057         links => [ { rel => 'child', href => "{name}" } ],
1058     },
1059     code => sub {
1060         my ($param) = @_;
1061
1062         PVE::Ceph::Tools::check_ceph_inited();
1063
1064         my $rados = PVE::RADOS->new();
1065
1066         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
1067
1068         my $res = [];
1069
1070         foreach my $rule (@$rules) {
1071             push @$res, { name => $rule };
1072         }
1073
1074         return $res;
1075     }});
1076
1077 1;