PVE/API2/Ceph.pm

   1 package PVE::API2::Ceph;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use Net::IP;
   8 use UUID;
   9
  10 use PVE::Ceph::Tools;
  11 use PVE::Ceph::Services;
  12 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  13 use PVE::JSONSchema qw(get_standard_option);
  14 use PVE::Network;
  15 use PVE::RADOS;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::Storage;
  19 use PVE::Tools qw(run_command file_get_contents file_set_contents extract_param);
  20
  21 use PVE::API2::Ceph::OSD;
  22 use PVE::API2::Ceph::FS;
  23 use PVE::API2::Ceph::MDS;
  24 use PVE::API2::Ceph::MGR;
  25 use PVE::API2::Ceph::MON;
  26 use PVE::API2::Ceph::Pools;
  27 use PVE::API2::Storage::Config;
  28
  29 use base qw(PVE::RESTHandler);
  30
  31 my $pve_osd_default_journal_size = 1024*5;
  32
  33 __PACKAGE__->register_method ({
  34     subclass => "PVE::API2::Ceph::OSD",
  35     path => 'osd',
  36 });
  37
  38 __PACKAGE__->register_method ({
  39     subclass => "PVE::API2::Ceph::MDS",
  40     path => 'mds',
  41 });
  42
  43 __PACKAGE__->register_method ({
  44     subclass => "PVE::API2::Ceph::MGR",
  45     path => 'mgr',
  46 });
  47
  48 __PACKAGE__->register_method ({
  49     subclass => "PVE::API2::Ceph::MON",
  50     path => 'mon',
  51 });
  52
  53 __PACKAGE__->register_method ({
  54     subclass => "PVE::API2::Ceph::FS",
  55     path => 'fs',
  56 });
  57
  58 __PACKAGE__->register_method ({
  59     subclass => "PVE::API2::Ceph::Pools",
  60     path => 'pools',
  61 });
  62
  63 __PACKAGE__->register_method ({
  64     name => 'index',
  65     path => '',
  66     method => 'GET',
  67     description => "Directory index.",
  68     permissions => { user => 'all' },
  69     permissions => {
  70         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  71     },
  72     parameters => {
  73         additionalProperties => 0,
  74         properties => {
  75             node => get_standard_option('pve-node'),
  76         },
  77     },
  78     returns => {
  79         type => 'array',
  80         items => {
  81             type => "object",
  82             properties => {},
  83         },
  84         links => [ { rel => 'child', href => "{name}" } ],
  85     },
  86     code => sub {
  87         my ($param) = @_;
  88
  89         my $result = [
  90             { name => 'cmd-safety' },
  91             { name => 'config' },
  92             { name => 'configdb' },
  93             { name => 'crush' },
  94             { name => 'fs' },
  95             { name => 'init' },
  96             { name => 'log' },
  97             { name => 'mds' },
  98             { name => 'mgr' },
  99             { name => 'mon' },
 100             { name => 'osd' },
 101             { name => 'pools' },
 102             { name => 'restart' },
 103             { name => 'rules' },
 104             { name => 'start' },
 105             { name => 'status' },
 106             { name => 'stop' },
 107         ];
 108
 109         return $result;
 110     }});
 111
 112 __PACKAGE__->register_method ({
 113     name => 'config',
 114     path => 'config',
 115     method => 'GET',
 116     proxyto => 'node',
 117     permissions => {
 118         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 119     },
 120     description => "Get the Ceph configuration file.",
 121     parameters => {
 122         additionalProperties => 0,
 123         properties => {
 124             node => get_standard_option('pve-node'),
 125         },
 126     },
 127     returns => { type => 'string' },
 128     code => sub {
 129         my ($param) = @_;
 130
 131         PVE::Ceph::Tools::check_ceph_inited();
 132
 133         my $path = PVE::Ceph::Tools::get_config('pve_ceph_cfgpath');
 134         return file_get_contents($path);
 135
 136     }});
 137
 138 __PACKAGE__->register_method ({
 139     name => 'configdb',
 140     path => 'configdb',
 141     method => 'GET',
 142     proxyto => 'node',
 143     protected => 1,
 144     permissions => {
 145         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 146     },
 147     description => "Get the Ceph configuration database.",
 148     parameters => {
 149         additionalProperties => 0,
 150         properties => {
 151             node => get_standard_option('pve-node'),
 152         },
 153     },
 154     returns => {
 155         type => 'array',
 156         items => {
 157             type => 'object',
 158             properties => {
 159                 section => { type => "string", },
 160                 name => { type => "string", },
 161                 value => { type => "string", },
 162                 level => { type => "string", },
 163                 'can_update_at_runtime' => { type => "boolean", },
 164                 mask => { type => "string" },
 165             },
 166         },
 167     },
 168     code => sub {
 169         my ($param) = @_;
 170
 171         PVE::Ceph::Tools::check_ceph_inited();
 172
 173         my $rados = PVE::RADOS->new();
 174         my $res = $rados->mon_command( { prefix => 'config dump', format => 'json' });
 175         foreach my $entry (@$res) {
 176             $entry->{can_update_at_runtime} = $entry->{can_update_at_runtime}? 1 : 0; # JSON::true/false -> 1/0
 177         }
 178
 179         return $res;
 180     }});
 181
 182
 183 __PACKAGE__->register_method ({
 184     name => 'init',
 185     path => 'init',
 186     method => 'POST',
 187     description => "Create initial ceph default configuration and setup symlinks.",
 188     proxyto => 'node',
 189     protected => 1,
 190     permissions => {
 191         check => ['perm', '/', [ 'Sys.Modify' ]],
 192     },
 193     parameters => {
 194         additionalProperties => 0,
 195         properties => {
 196             node => get_standard_option('pve-node'),
 197             network => {
 198                 description => "Use specific network for all ceph related traffic",
 199                 type => 'string', format => 'CIDR',
 200                 optional => 1,
 201                 maxLength => 128,
 202             },
 203             'cluster-network' => {
 204                 description => "Declare a separate cluster network, OSDs will route" .
 205                     "heartbeat, object replication and recovery traffic over it",
 206                 type => 'string', format => 'CIDR',
 207                 requires => 'network',
 208                 optional => 1,
 209                 maxLength => 128,
 210             },
 211             size => {
 212                 description => 'Targeted number of replicas per object',
 213                 type => 'integer',
 214                 default => 3,
 215                 optional => 1,
 216                 minimum => 1,
 217                 maximum => 7,
 218             },
 219             min_size => {
 220                 description => 'Minimum number of available replicas per object to allow I/O',
 221                 type => 'integer',
 222                 default => 2,
 223                 optional => 1,
 224                 minimum => 1,
 225                 maximum => 7,
 226             },
 227             pg_bits => {
 228                 description => "Placement group bits, used to specify the " .
 229                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 230                     "default pg num' does not work for default pools.",
 231                 type => 'integer',
 232                 default => 6,
 233                 optional => 1,
 234                 minimum => 6,
 235                 maximum => 14,
 236             },
 237             disable_cephx => {
 238                 description => "Disable cephx authentication.\n\n" .
 239                     "WARNING: cephx is a security feature protecting against " .
 240                     "man-in-the-middle attacks. Only consider disabling cephx ".
 241                     "if your network is private!",
 242                 type => 'boolean',
 243                 optional => 1,
 244                 default => 0,
 245             },
 246         },
 247     },
 248     returns => { type => 'null' },
 249     code => sub {
 250         my ($param) = @_;
 251
 252         my $version = PVE::Ceph::Tools::get_local_version(1);
 253
 254         if (!$version || $version < 14) {
 255             die "Ceph Nautilus required - please run 'pveceph install'\n";
 256         } else {
 257             PVE::Ceph::Tools::check_ceph_installed('ceph_bin');
 258         }
 259
 260         my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 261
 262         # simply load old config if it already exists
 263         PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
 264             my $cfg = cfs_read_file('ceph.conf');
 265
 266             if (!$cfg->{global}) {
 267
 268                 my $fsid;
 269                 my $uuid;
 270
 271                 UUID::generate($uuid);
 272                 UUID::unparse($uuid, $fsid);
 273
 274                 $cfg->{global} = {
 275                     'fsid' => $fsid,
 276                     'auth cluster required' => $auth,
 277                     'auth service required' => $auth,
 278                     'auth client required' => $auth,
 279                     'osd pool default size' => $param->{size} // 3,
 280                     'osd pool default min size' => $param->{min_size} // 2,
 281                     'mon allow pool delete' => 'true',
 282                 };
 283
 284                 # this does not work for default pools
 285                 #'osd pool default pg num' => $pg_num,
 286                 #'osd pool default pgp num' => $pg_num,
 287             }
 288
 289             if ($auth eq 'cephx') {
 290                 $cfg->{client}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 291             }
 292
 293             if ($param->{pg_bits}) {
 294                 $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 295                 $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 296             }
 297
 298             if ($param->{network}) {
 299                 $cfg->{global}->{'public network'} = $param->{network};
 300                 $cfg->{global}->{'cluster network'} = $param->{network};
 301             }
 302
 303             if ($param->{'cluster-network'}) {
 304                 $cfg->{global}->{'cluster network'} = $param->{'cluster-network'};
 305             }
 306
 307             cfs_write_file('ceph.conf', $cfg);
 308
 309             if ($auth eq 'cephx') {
 310                 PVE::Ceph::Tools::get_or_create_admin_keyring();
 311             }
 312             PVE::Ceph::Tools::setup_pve_symlinks();
 313         });
 314         die $@ if $@;
 315
 316         return undef;
 317     }});
 318
 319 __PACKAGE__->register_method ({
 320     name => 'stop',
 321     path => 'stop',
 322     method => 'POST',
 323     description => "Stop ceph services.",
 324     proxyto => 'node',
 325     protected => 1,
 326     permissions => {
 327         check => ['perm', '/', [ 'Sys.Modify' ]],
 328     },
 329     parameters => {
 330         additionalProperties => 0,
 331         properties => {
 332             node => get_standard_option('pve-node'),
 333             service => {
 334                 description => 'Ceph service name.',
 335                 type => 'string',
 336                 optional => 1,
 337                 default => 'ceph.target',
 338                 pattern => '(ceph|mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 339             },
 340         },
 341     },
 342     returns => { type => 'string' },
 343     code => sub {
 344         my ($param) = @_;
 345
 346         my $rpcenv = PVE::RPCEnvironment::get();
 347
 348         my $authuser = $rpcenv->get_user();
 349
 350         PVE::Ceph::Tools::check_ceph_inited();
 351
 352         my $cfg = cfs_read_file('ceph.conf');
 353         scalar(keys %$cfg) || die "no configuration\n";
 354
 355         my $worker = sub {
 356             my $upid = shift;
 357
 358             my $cmd = ['stop'];
 359             if ($param->{service}) {
 360                 push @$cmd, $param->{service};
 361             }
 362
 363             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 364         };
 365
 366         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
 367                                     $authuser, $worker);
 368     }});
 369
 370 __PACKAGE__->register_method ({
 371     name => 'start',
 372     path => 'start',
 373     method => 'POST',
 374     description => "Start ceph services.",
 375     proxyto => 'node',
 376     protected => 1,
 377     permissions => {
 378         check => ['perm', '/', [ 'Sys.Modify' ]],
 379     },
 380     parameters => {
 381         additionalProperties => 0,
 382         properties => {
 383             node => get_standard_option('pve-node'),
 384             service => {
 385                 description => 'Ceph service name.',
 386                 type => 'string',
 387                 optional => 1,
 388                 default => 'ceph.target',
 389                 pattern => '(ceph|mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 390             },
 391         },
 392     },
 393     returns => { type => 'string' },
 394     code => sub {
 395         my ($param) = @_;
 396
 397         my $rpcenv = PVE::RPCEnvironment::get();
 398
 399         my $authuser = $rpcenv->get_user();
 400
 401         PVE::Ceph::Tools::check_ceph_inited();
 402
 403         my $cfg = cfs_read_file('ceph.conf');
 404         scalar(keys %$cfg) || die "no configuration\n";
 405
 406         my $worker = sub {
 407             my $upid = shift;
 408
 409             my $cmd = ['start'];
 410             if ($param->{service}) {
 411                 push @$cmd, $param->{service};
 412             }
 413
 414             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 415         };
 416
 417         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
 418                                     $authuser, $worker);
 419     }});
 420
 421 __PACKAGE__->register_method ({
 422     name => 'restart',
 423     path => 'restart',
 424     method => 'POST',
 425     description => "Restart ceph services.",
 426     proxyto => 'node',
 427     protected => 1,
 428     permissions => {
 429         check => ['perm', '/', [ 'Sys.Modify' ]],
 430     },
 431     parameters => {
 432         additionalProperties => 0,
 433         properties => {
 434             node => get_standard_option('pve-node'),
 435             service => {
 436                 description => 'Ceph service name.',
 437                 type => 'string',
 438                 optional => 1,
 439                 default => 'ceph.target',
 440                 pattern => '(mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 441             },
 442         },
 443     },
 444     returns => { type => 'string' },
 445     code => sub {
 446         my ($param) = @_;
 447
 448         my $rpcenv = PVE::RPCEnvironment::get();
 449
 450         my $authuser = $rpcenv->get_user();
 451
 452         PVE::Ceph::Tools::check_ceph_inited();
 453
 454         my $cfg = cfs_read_file('ceph.conf');
 455         scalar(keys %$cfg) || die "no configuration\n";
 456
 457         my $worker = sub {
 458             my $upid = shift;
 459
 460             my $cmd = ['restart'];
 461             if ($param->{service}) {
 462                 push @$cmd, $param->{service};
 463             }
 464
 465             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 466         };
 467
 468         return $rpcenv->fork_worker('srvrestart', $param->{service} || 'ceph',
 469                                     $authuser, $worker);
 470     }});
 471
 472 __PACKAGE__->register_method ({
 473     name => 'status',
 474     path => 'status',
 475     method => 'GET',
 476     description => "Get ceph status.",
 477     proxyto => 'node',
 478     protected => 1,
 479     permissions => {
 480         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 481     },
 482     parameters => {
 483         additionalProperties => 0,
 484         properties => {
 485             node => get_standard_option('pve-node'),
 486         },
 487     },
 488     returns => { type => 'object' },
 489     code => sub {
 490         my ($param) = @_;
 491
 492         PVE::Ceph::Tools::check_ceph_inited();
 493
 494         return PVE::Ceph::Tools::ceph_cluster_status();
 495     }});
 496
 497
 498 __PACKAGE__->register_method ({
 499     name => 'crush',
 500     path => 'crush',
 501     method => 'GET',
 502     description => "Get OSD crush map",
 503     proxyto => 'node',
 504     protected => 1,
 505     permissions => {
 506         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 507     },
 508     parameters => {
 509         additionalProperties => 0,
 510         properties => {
 511             node => get_standard_option('pve-node'),
 512         },
 513     },
 514     returns => { type => 'string' },
 515     code => sub {
 516         my ($param) = @_;
 517
 518         PVE::Ceph::Tools::check_ceph_inited();
 519
 520         # this produces JSON (difficult to read for the user)
 521         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
 522
 523         my $txt = '';
 524
 525         my $mapfile = "/var/tmp/ceph-crush.map.$$";
 526         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
 527
 528         my $rados = PVE::RADOS->new();
 529
 530         eval {
 531             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
 532             file_set_contents($mapfile, $bindata);
 533             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
 534             $txt = file_get_contents($mapdata);
 535         };
 536         my $err = $@;
 537
 538         unlink $mapfile;
 539         unlink $mapdata;
 540
 541         die $err if $err;
 542
 543         return $txt;
 544     }});
 545
 546 __PACKAGE__->register_method({
 547     name => 'log',
 548     path => 'log',
 549     method => 'GET',
 550     description => "Read ceph log",
 551     proxyto => 'node',
 552     permissions => {
 553         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
 554     },
 555     protected => 1,
 556     parameters => {
 557         additionalProperties => 0,
 558         properties => {
 559             node => get_standard_option('pve-node'),
 560             start => {
 561                 type => 'integer',
 562                 minimum => 0,
 563                 optional => 1,
 564             },
 565             limit => {
 566                 type => 'integer',
 567                 minimum => 0,
 568                 optional => 1,
 569             },
 570         },
 571     },
 572     returns => {
 573         type => 'array',
 574         items => {
 575             type => "object",
 576             properties => {
 577                 n => {
 578                   description=>  "Line number",
 579                   type=> 'integer',
 580                 },
 581                 t => {
 582                   description=>  "Line text",
 583                   type => 'string',
 584                 }
 585             }
 586         }
 587     },
 588     code => sub {
 589         my ($param) = @_;
 590
 591         PVE::Ceph::Tools::check_ceph_inited();
 592
 593         my $rpcenv = PVE::RPCEnvironment::get();
 594         my $user = $rpcenv->get_user();
 595         my $node = $param->{node};
 596
 597         my $logfile = "/var/log/ceph/ceph.log";
 598         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
 599
 600         $rpcenv->set_result_attrib('total', $count);
 601
 602         return $lines;
 603     }});
 604
 605 __PACKAGE__->register_method ({
 606     name => 'rules',
 607     path => 'rules',
 608     method => 'GET',
 609     description => "List ceph rules.",
 610     proxyto => 'node',
 611     protected => 1,
 612     permissions => {
 613         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 614     },
 615     parameters => {
 616         additionalProperties => 0,
 617         properties => {
 618             node => get_standard_option('pve-node'),
 619         },
 620     },
 621     returns => {
 622         type => 'array',
 623         items => {
 624             type => "object",
 625             properties => {},
 626         },
 627         links => [ { rel => 'child', href => "{name}" } ],
 628     },
 629     code => sub {
 630         my ($param) = @_;
 631
 632         PVE::Ceph::Tools::check_ceph_inited();
 633
 634         my $rados = PVE::RADOS->new();
 635
 636         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
 637
 638         my $res = [];
 639
 640         foreach my $rule (@$rules) {
 641             push @$res, { name => $rule };
 642         }
 643
 644         return $res;
 645     }});
 646
 647 __PACKAGE__->register_method ({
 648     name => 'cmd_safety',
 649     path => 'cmd-safety',
 650     method => 'GET',
 651     description => "Heuristical check if it is safe to perform an action.",
 652     proxyto => 'node',
 653     protected => 1,
 654     permissions => {
 655         check => ['perm', '/', [ 'Sys.Audit' ]],
 656     },
 657     parameters => {
 658         additionalProperties => 0,
 659         properties => {
 660             node => get_standard_option('pve-node'),
 661             service => {
 662                 description => 'Service type',
 663                 type => 'string',
 664                 enum => ['osd', 'mon', 'mds'],
 665             },
 666             id => {
 667                 description => 'ID of the service',
 668                 type => 'string',
 669             },
 670             action => {
 671                 description => 'Action to check',
 672                 type => 'string',
 673                 enum => ['stop', 'destroy'],
 674             },
 675         },
 676     },
 677     returns => {
 678         type => 'object',
 679         properties => {
 680            safe  => {
 681                 type => 'boolean',
 682                 description => 'If it is safe to run the command.',
 683             },
 684             status => {
 685                 type => 'string',
 686                 optional => 1,
 687                 description => 'Status message given by Ceph.'
 688             },
 689         },
 690     },
 691     code => sub {
 692         my ($param) = @_;
 693
 694         PVE::Ceph::Tools::check_ceph_inited();
 695
 696         my $id = $param->{id};
 697         my $service = $param->{service};
 698         my $action = $param->{action};
 699
 700         my $rados = PVE::RADOS->new();
 701
 702         my $supported_actions = {
 703             osd => {
 704                 stop => 'ok-to-stop',
 705                 destroy => 'safe-to-destroy',
 706             },
 707             mon => {
 708                 stop => 'ok-to-stop',
 709                 destroy => 'ok-to-rm',
 710             },
 711             mds => {
 712                 stop => 'ok-to-stop',
 713             },
 714         };
 715
 716         die "Service does not support this action: ${service}: ${action}\n"
 717             if !$supported_actions->{$service}->{$action};
 718
 719         my $result = {
 720             safe => 0,
 721             status => '',
 722         };
 723
 724         my $params = {
 725             prefix => "${service} $supported_actions->{$service}->{$action}",
 726             format => 'plain',
 727         };
 728         if ($service eq 'mon' && $action eq 'destroy') {
 729             $params->{id} = $id;
 730         } else {
 731             $params->{ids} = [ $id ];
 732         }
 733
 734         $result = $rados->mon_cmd($params, 1);
 735         die $@ if $@;
 736
 737         $result->{safe} = $result->{return_code} == 0 ? 1 : 0;
 738         $result->{status} = $result->{status_message};
 739
 740         return $result;
 741     }});
 742
 743 1;