PVE/API2/Ceph.pm

   1 package PVE::API2::Ceph;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use Net::IP;
   8 use UUID;
   9
  10 use PVE::Ceph::Tools;
  11 use PVE::Ceph::Services;
  12 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  13 use PVE::JSONSchema qw(get_standard_option);
  14 use PVE::Network;
  15 use PVE::RADOS;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::Storage;
  19 use PVE::Tools qw(run_command file_get_contents file_set_contents extract_param);
  20
  21 use PVE::API2::Ceph::OSD;
  22 use PVE::API2::Ceph::FS;
  23 use PVE::API2::Ceph::MDS;
  24 use PVE::API2::Ceph::MGR;
  25 use PVE::API2::Ceph::MON;
  26 use PVE::API2::Ceph::Pool;
  27 use PVE::API2::Ceph::Pools;
  28 use PVE::API2::Storage::Config;
  29
  30 use base qw(PVE::RESTHandler);
  31
  32 my $pve_osd_default_journal_size = 1024*5;
  33
  34 __PACKAGE__->register_method ({
  35     subclass => "PVE::API2::Ceph::OSD",
  36     path => 'osd',
  37 });
  38
  39 __PACKAGE__->register_method ({
  40     subclass => "PVE::API2::Ceph::MDS",
  41     path => 'mds',
  42 });
  43
  44 __PACKAGE__->register_method ({
  45     subclass => "PVE::API2::Ceph::MGR",
  46     path => 'mgr',
  47 });
  48
  49 __PACKAGE__->register_method ({
  50     subclass => "PVE::API2::Ceph::MON",
  51     path => 'mon',
  52 });
  53
  54 __PACKAGE__->register_method ({
  55     subclass => "PVE::API2::Ceph::FS",
  56     path => 'fs',
  57 });
  58
  59 __PACKAGE__->register_method ({
  60     subclass => "PVE::API2::Ceph::Pool",
  61     path => 'pool',
  62 });
  63
  64 # TODO: deprecrated, remove with PVE 8
  65 __PACKAGE__->register_method ({
  66     subclass => "PVE::API2::Ceph::Pools",
  67     path => 'pools',
  68 });
  69
  70 __PACKAGE__->register_method ({
  71     name => 'index',
  72     path => '',
  73     method => 'GET',
  74     description => "Directory index.",
  75     permissions => { user => 'all' },
  76     permissions => {
  77         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  78     },
  79     parameters => {
  80         additionalProperties => 0,
  81         properties => {
  82             node => get_standard_option('pve-node'),
  83         },
  84     },
  85     returns => {
  86         type => 'array',
  87         items => {
  88             type => "object",
  89             properties => {},
  90         },
  91         links => [ { rel => 'child', href => "{name}" } ],
  92     },
  93     code => sub {
  94         my ($param) = @_;
  95
  96         my $result = [
  97             { name => 'cmd-safety' },
  98             { name => 'config' },
  99             { name => 'configdb' },
 100             { name => 'crush' },
 101             { name => 'fs' },
 102             { name => 'init' },
 103             { name => 'log' },
 104             { name => 'mds' },
 105             { name => 'mgr' },
 106             { name => 'mon' },
 107             { name => 'osd' },
 108             { name => 'pools' },
 109             { name => 'restart' },
 110             { name => 'rules' },
 111             { name => 'start' },
 112             { name => 'status' },
 113             { name => 'stop' },
 114         ];
 115
 116         return $result;
 117     }});
 118
 119 __PACKAGE__->register_method ({
 120     name => 'config',
 121     path => 'config',
 122     method => 'GET',
 123     proxyto => 'node',
 124     permissions => {
 125         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 126     },
 127     description => "Get the Ceph configuration file.",
 128     parameters => {
 129         additionalProperties => 0,
 130         properties => {
 131             node => get_standard_option('pve-node'),
 132         },
 133     },
 134     returns => { type => 'string' },
 135     code => sub {
 136         my ($param) = @_;
 137
 138         PVE::Ceph::Tools::check_ceph_inited();
 139
 140         my $path = PVE::Ceph::Tools::get_config('pve_ceph_cfgpath');
 141         return file_get_contents($path);
 142
 143     }});
 144
 145 __PACKAGE__->register_method ({
 146     name => 'configdb',
 147     path => 'configdb',
 148     method => 'GET',
 149     proxyto => 'node',
 150     protected => 1,
 151     permissions => {
 152         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 153     },
 154     description => "Get the Ceph configuration database.",
 155     parameters => {
 156         additionalProperties => 0,
 157         properties => {
 158             node => get_standard_option('pve-node'),
 159         },
 160     },
 161     returns => {
 162         type => 'array',
 163         items => {
 164             type => 'object',
 165             properties => {
 166                 section => { type => "string", },
 167                 name => { type => "string", },
 168                 value => { type => "string", },
 169                 level => { type => "string", },
 170                 'can_update_at_runtime' => { type => "boolean", },
 171                 mask => { type => "string" },
 172             },
 173         },
 174     },
 175     code => sub {
 176         my ($param) = @_;
 177
 178         PVE::Ceph::Tools::check_ceph_inited();
 179
 180         my $rados = PVE::RADOS->new();
 181         my $res = $rados->mon_command( { prefix => 'config dump', format => 'json' });
 182         foreach my $entry (@$res) {
 183             $entry->{can_update_at_runtime} = $entry->{can_update_at_runtime}? 1 : 0; # JSON::true/false -> 1/0
 184         }
 185
 186         return $res;
 187     }});
 188
 189
 190 __PACKAGE__->register_method ({
 191     name => 'init',
 192     path => 'init',
 193     method => 'POST',
 194     description => "Create initial ceph default configuration and setup symlinks.",
 195     proxyto => 'node',
 196     protected => 1,
 197     permissions => {
 198         check => ['perm', '/', [ 'Sys.Modify' ]],
 199     },
 200     parameters => {
 201         additionalProperties => 0,
 202         properties => {
 203             node => get_standard_option('pve-node'),
 204             network => {
 205                 description => "Use specific network for all ceph related traffic",
 206                 type => 'string', format => 'CIDR',
 207                 optional => 1,
 208                 maxLength => 128,
 209             },
 210             'cluster-network' => {
 211                 description => "Declare a separate cluster network, OSDs will route" .
 212                     "heartbeat, object replication and recovery traffic over it",
 213                 type => 'string', format => 'CIDR',
 214                 requires => 'network',
 215                 optional => 1,
 216                 maxLength => 128,
 217             },
 218             size => {
 219                 description => 'Targeted number of replicas per object',
 220                 type => 'integer',
 221                 default => 3,
 222                 optional => 1,
 223                 minimum => 1,
 224                 maximum => 7,
 225             },
 226             min_size => {
 227                 description => 'Minimum number of available replicas per object to allow I/O',
 228                 type => 'integer',
 229                 default => 2,
 230                 optional => 1,
 231                 minimum => 1,
 232                 maximum => 7,
 233             },
 234             pg_bits => {
 235                 description => "Placement group bits, used to specify the " .
 236                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 237                     "default pg num' does not work for default pools.",
 238                 type => 'integer',
 239                 default => 6,
 240                 optional => 1,
 241                 minimum => 6,
 242                 maximum => 14,
 243             },
 244             disable_cephx => {
 245                 description => "Disable cephx authentication.\n\n" .
 246                     "WARNING: cephx is a security feature protecting against " .
 247                     "man-in-the-middle attacks. Only consider disabling cephx ".
 248                     "if your network is private!",
 249                 type => 'boolean',
 250                 optional => 1,
 251                 default => 0,
 252             },
 253         },
 254     },
 255     returns => { type => 'null' },
 256     code => sub {
 257         my ($param) = @_;
 258
 259         my $version = PVE::Ceph::Tools::get_local_version(1);
 260
 261         if (!$version || $version < 14) {
 262             die "Ceph Nautilus required - please run 'pveceph install'\n";
 263         } else {
 264             PVE::Ceph::Tools::check_ceph_installed('ceph_bin');
 265         }
 266
 267         my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 268
 269         # simply load old config if it already exists
 270         PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
 271             my $cfg = cfs_read_file('ceph.conf');
 272
 273             if (!$cfg->{global}) {
 274
 275                 my $fsid;
 276                 my $uuid;
 277
 278                 UUID::generate($uuid);
 279                 UUID::unparse($uuid, $fsid);
 280
 281                 $cfg->{global} = {
 282                     'fsid' => $fsid,
 283                     'auth cluster required' => $auth,
 284                     'auth service required' => $auth,
 285                     'auth client required' => $auth,
 286                     'osd pool default size' => $param->{size} // 3,
 287                     'osd pool default min size' => $param->{min_size} // 2,
 288                     'mon allow pool delete' => 'true',
 289                 };
 290
 291                 # this does not work for default pools
 292                 #'osd pool default pg num' => $pg_num,
 293                 #'osd pool default pgp num' => $pg_num,
 294             }
 295
 296             if ($auth eq 'cephx') {
 297                 $cfg->{client}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 298             }
 299
 300             if ($param->{pg_bits}) {
 301                 $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 302                 $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 303             }
 304
 305             if ($param->{network}) {
 306                 $cfg->{global}->{'public network'} = $param->{network};
 307                 $cfg->{global}->{'cluster network'} = $param->{network};
 308             }
 309
 310             if ($param->{'cluster-network'}) {
 311                 $cfg->{global}->{'cluster network'} = $param->{'cluster-network'};
 312             }
 313
 314             cfs_write_file('ceph.conf', $cfg);
 315
 316             if ($auth eq 'cephx') {
 317                 PVE::Ceph::Tools::get_or_create_admin_keyring();
 318             }
 319             PVE::Ceph::Tools::setup_pve_symlinks();
 320         });
 321         die $@ if $@;
 322
 323         return undef;
 324     }});
 325
 326 __PACKAGE__->register_method ({
 327     name => 'stop',
 328     path => 'stop',
 329     method => 'POST',
 330     description => "Stop ceph services.",
 331     proxyto => 'node',
 332     protected => 1,
 333     permissions => {
 334         check => ['perm', '/', [ 'Sys.Modify' ]],
 335     },
 336     parameters => {
 337         additionalProperties => 0,
 338         properties => {
 339             node => get_standard_option('pve-node'),
 340             service => {
 341                 description => 'Ceph service name.',
 342                 type => 'string',
 343                 optional => 1,
 344                 default => 'ceph.target',
 345                 pattern => '(ceph|mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 346             },
 347         },
 348     },
 349     returns => { type => 'string' },
 350     code => sub {
 351         my ($param) = @_;
 352
 353         my $rpcenv = PVE::RPCEnvironment::get();
 354
 355         my $authuser = $rpcenv->get_user();
 356
 357         PVE::Ceph::Tools::check_ceph_inited();
 358
 359         my $cfg = cfs_read_file('ceph.conf');
 360         scalar(keys %$cfg) || die "no configuration\n";
 361
 362         my $worker = sub {
 363             my $upid = shift;
 364
 365             my $cmd = ['stop'];
 366             if ($param->{service}) {
 367                 push @$cmd, $param->{service};
 368             }
 369
 370             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 371         };
 372
 373         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
 374                                     $authuser, $worker);
 375     }});
 376
 377 __PACKAGE__->register_method ({
 378     name => 'start',
 379     path => 'start',
 380     method => 'POST',
 381     description => "Start ceph services.",
 382     proxyto => 'node',
 383     protected => 1,
 384     permissions => {
 385         check => ['perm', '/', [ 'Sys.Modify' ]],
 386     },
 387     parameters => {
 388         additionalProperties => 0,
 389         properties => {
 390             node => get_standard_option('pve-node'),
 391             service => {
 392                 description => 'Ceph service name.',
 393                 type => 'string',
 394                 optional => 1,
 395                 default => 'ceph.target',
 396                 pattern => '(ceph|mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 397             },
 398         },
 399     },
 400     returns => { type => 'string' },
 401     code => sub {
 402         my ($param) = @_;
 403
 404         my $rpcenv = PVE::RPCEnvironment::get();
 405
 406         my $authuser = $rpcenv->get_user();
 407
 408         PVE::Ceph::Tools::check_ceph_inited();
 409
 410         my $cfg = cfs_read_file('ceph.conf');
 411         scalar(keys %$cfg) || die "no configuration\n";
 412
 413         my $worker = sub {
 414             my $upid = shift;
 415
 416             my $cmd = ['start'];
 417             if ($param->{service}) {
 418                 push @$cmd, $param->{service};
 419             }
 420
 421             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 422         };
 423
 424         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
 425                                     $authuser, $worker);
 426     }});
 427
 428 __PACKAGE__->register_method ({
 429     name => 'restart',
 430     path => 'restart',
 431     method => 'POST',
 432     description => "Restart ceph services.",
 433     proxyto => 'node',
 434     protected => 1,
 435     permissions => {
 436         check => ['perm', '/', [ 'Sys.Modify' ]],
 437     },
 438     parameters => {
 439         additionalProperties => 0,
 440         properties => {
 441             node => get_standard_option('pve-node'),
 442             service => {
 443                 description => 'Ceph service name.',
 444                 type => 'string',
 445                 optional => 1,
 446                 default => 'ceph.target',
 447                 pattern => '(mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 448             },
 449         },
 450     },
 451     returns => { type => 'string' },
 452     code => sub {
 453         my ($param) = @_;
 454
 455         my $rpcenv = PVE::RPCEnvironment::get();
 456
 457         my $authuser = $rpcenv->get_user();
 458
 459         PVE::Ceph::Tools::check_ceph_inited();
 460
 461         my $cfg = cfs_read_file('ceph.conf');
 462         scalar(keys %$cfg) || die "no configuration\n";
 463
 464         my $worker = sub {
 465             my $upid = shift;
 466
 467             my $cmd = ['restart'];
 468             if ($param->{service}) {
 469                 push @$cmd, $param->{service};
 470             }
 471
 472             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 473         };
 474
 475         return $rpcenv->fork_worker('srvrestart', $param->{service} || 'ceph',
 476                                     $authuser, $worker);
 477     }});
 478
 479 __PACKAGE__->register_method ({
 480     name => 'status',
 481     path => 'status',
 482     method => 'GET',
 483     description => "Get ceph status.",
 484     proxyto => 'node',
 485     protected => 1,
 486     permissions => {
 487         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 488     },
 489     parameters => {
 490         additionalProperties => 0,
 491         properties => {
 492             node => get_standard_option('pve-node'),
 493         },
 494     },
 495     returns => { type => 'object' },
 496     code => sub {
 497         my ($param) = @_;
 498
 499         PVE::Ceph::Tools::check_ceph_inited();
 500
 501         return PVE::Ceph::Tools::ceph_cluster_status();
 502     }});
 503
 504
 505 __PACKAGE__->register_method ({
 506     name => 'crush',
 507     path => 'crush',
 508     method => 'GET',
 509     description => "Get OSD crush map",
 510     proxyto => 'node',
 511     protected => 1,
 512     permissions => {
 513         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 514     },
 515     parameters => {
 516         additionalProperties => 0,
 517         properties => {
 518             node => get_standard_option('pve-node'),
 519         },
 520     },
 521     returns => { type => 'string' },
 522     code => sub {
 523         my ($param) = @_;
 524
 525         PVE::Ceph::Tools::check_ceph_inited();
 526
 527         # this produces JSON (difficult to read for the user)
 528         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
 529
 530         my $txt = '';
 531
 532         my $mapfile = "/var/tmp/ceph-crush.map.$$";
 533         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
 534
 535         my $rados = PVE::RADOS->new();
 536
 537         eval {
 538             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
 539             file_set_contents($mapfile, $bindata);
 540             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
 541             $txt = file_get_contents($mapdata);
 542         };
 543         my $err = $@;
 544
 545         unlink $mapfile;
 546         unlink $mapdata;
 547
 548         die $err if $err;
 549
 550         return $txt;
 551     }});
 552
 553 __PACKAGE__->register_method({
 554     name => 'log',
 555     path => 'log',
 556     method => 'GET',
 557     description => "Read ceph log",
 558     proxyto => 'node',
 559     permissions => {
 560         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
 561     },
 562     protected => 1,
 563     parameters => {
 564         additionalProperties => 0,
 565         properties => {
 566             node => get_standard_option('pve-node'),
 567             start => {
 568                 type => 'integer',
 569                 minimum => 0,
 570                 optional => 1,
 571             },
 572             limit => {
 573                 type => 'integer',
 574                 minimum => 0,
 575                 optional => 1,
 576             },
 577         },
 578     },
 579     returns => {
 580         type => 'array',
 581         items => {
 582             type => "object",
 583             properties => {
 584                 n => {
 585                   description=>  "Line number",
 586                   type=> 'integer',
 587                 },
 588                 t => {
 589                   description=>  "Line text",
 590                   type => 'string',
 591                 }
 592             }
 593         }
 594     },
 595     code => sub {
 596         my ($param) = @_;
 597
 598         PVE::Ceph::Tools::check_ceph_inited();
 599
 600         my $rpcenv = PVE::RPCEnvironment::get();
 601         my $user = $rpcenv->get_user();
 602         my $node = $param->{node};
 603
 604         my $logfile = "/var/log/ceph/ceph.log";
 605         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
 606
 607         $rpcenv->set_result_attrib('total', $count);
 608
 609         return $lines;
 610     }});
 611
 612 __PACKAGE__->register_method ({
 613     name => 'rules',
 614     path => 'rules',
 615     method => 'GET',
 616     description => "List ceph rules.",
 617     proxyto => 'node',
 618     protected => 1,
 619     permissions => {
 620         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 621     },
 622     parameters => {
 623         additionalProperties => 0,
 624         properties => {
 625             node => get_standard_option('pve-node'),
 626         },
 627     },
 628     returns => {
 629         type => 'array',
 630         items => {
 631             type => "object",
 632             properties => {
 633                 name => {
 634                     description => "Name of the CRUSH rule.",
 635                     type => "string",
 636                 }
 637             },
 638         },
 639         links => [ { rel => 'child', href => "{name}" } ],
 640     },
 641     code => sub {
 642         my ($param) = @_;
 643
 644         PVE::Ceph::Tools::check_ceph_inited();
 645
 646         my $rados = PVE::RADOS->new();
 647
 648         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
 649
 650         my $res = [];
 651
 652         foreach my $rule (@$rules) {
 653             push @$res, { name => $rule };
 654         }
 655
 656         return $res;
 657     }});
 658
 659 __PACKAGE__->register_method ({
 660     name => 'cmd_safety',
 661     path => 'cmd-safety',
 662     method => 'GET',
 663     description => "Heuristical check if it is safe to perform an action.",
 664     proxyto => 'node',
 665     protected => 1,
 666     permissions => {
 667         check => ['perm', '/', [ 'Sys.Audit' ]],
 668     },
 669     parameters => {
 670         additionalProperties => 0,
 671         properties => {
 672             node => get_standard_option('pve-node'),
 673             service => {
 674                 description => 'Service type',
 675                 type => 'string',
 676                 enum => ['osd', 'mon', 'mds'],
 677             },
 678             id => {
 679                 description => 'ID of the service',
 680                 type => 'string',
 681             },
 682             action => {
 683                 description => 'Action to check',
 684                 type => 'string',
 685                 enum => ['stop', 'destroy'],
 686             },
 687         },
 688     },
 689     returns => {
 690         type => 'object',
 691         properties => {
 692            safe  => {
 693                 type => 'boolean',
 694                 description => 'If it is safe to run the command.',
 695             },
 696             status => {
 697                 type => 'string',
 698                 optional => 1,
 699                 description => 'Status message given by Ceph.'
 700             },
 701         },
 702     },
 703     code => sub {
 704         my ($param) = @_;
 705
 706         PVE::Ceph::Tools::check_ceph_inited();
 707
 708         my $id = $param->{id};
 709         my $service = $param->{service};
 710         my $action = $param->{action};
 711
 712         my $rados = PVE::RADOS->new();
 713
 714         my $supported_actions = {
 715             osd => {
 716                 stop => 'ok-to-stop',
 717                 destroy => 'safe-to-destroy',
 718             },
 719             mon => {
 720                 stop => 'ok-to-stop',
 721                 destroy => 'ok-to-rm',
 722             },
 723             mds => {
 724                 stop => 'ok-to-stop',
 725             },
 726         };
 727
 728         die "Service does not support this action: ${service}: ${action}\n"
 729             if !$supported_actions->{$service}->{$action};
 730
 731         my $result = {
 732             safe => 0,
 733             status => '',
 734         };
 735
 736         my $params = {
 737             prefix => "${service} $supported_actions->{$service}->{$action}",
 738             format => 'plain',
 739         };
 740         if ($service eq 'mon' && $action eq 'destroy') {
 741             $params->{id} = $id;
 742         } else {
 743             $params->{ids} = [ $id ];
 744         }
 745
 746         $result = $rados->mon_cmd($params, 1);
 747         die $@ if $@;
 748
 749         $result->{safe} = $result->{return_code} == 0 ? 1 : 0;
 750         $result->{status} = $result->{status_message};
 751
 752         return $result;
 753     }});
 754
 755 1;