PVE/API2/Ceph.pm

   1 package PVE::API2::Ceph;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use Net::IP;
   8 use UUID;
   9
  10 use PVE::Ceph::Tools;
  11 use PVE::Ceph::Services;
  12 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  13 use PVE::JSONSchema qw(get_standard_option);
  14 use PVE::Network;
  15 use PVE::RADOS;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::Storage;
  19 use PVE::Tools qw(run_command file_get_contents file_set_contents extract_param);
  20
  21 use PVE::API2::Ceph::OSD;
  22 use PVE::API2::Ceph::FS;
  23 use PVE::API2::Ceph::MDS;
  24 use PVE::API2::Ceph::MGR;
  25 use PVE::API2::Ceph::MON;
  26 use PVE::API2::Ceph::Pools;
  27 use PVE::API2::Storage::Config;
  28
  29 use base qw(PVE::RESTHandler);
  30
  31 my $pve_osd_default_journal_size = 1024*5;
  32
  33 __PACKAGE__->register_method ({
  34     subclass => "PVE::API2::Ceph::OSD",
  35     path => 'osd',
  36 });
  37
  38 __PACKAGE__->register_method ({
  39     subclass => "PVE::API2::Ceph::MDS",
  40     path => 'mds',
  41 });
  42
  43 __PACKAGE__->register_method ({
  44     subclass => "PVE::API2::Ceph::MGR",
  45     path => 'mgr',
  46 });
  47
  48 __PACKAGE__->register_method ({
  49     subclass => "PVE::API2::Ceph::MON",
  50     path => 'mon',
  51 });
  52
  53 __PACKAGE__->register_method ({
  54     subclass => "PVE::API2::Ceph::FS",
  55     path => 'fs',
  56 });
  57
  58 __PACKAGE__->register_method ({
  59     subclass => "PVE::API2::Ceph::Pools",
  60     path => 'pools',
  61 });
  62
  63 __PACKAGE__->register_method ({
  64     name => 'index',
  65     path => '',
  66     method => 'GET',
  67     description => "Directory index.",
  68     permissions => { user => 'all' },
  69     permissions => {
  70         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  71     },
  72     parameters => {
  73         additionalProperties => 0,
  74         properties => {
  75             node => get_standard_option('pve-node'),
  76         },
  77     },
  78     returns => {
  79         type => 'array',
  80         items => {
  81             type => "object",
  82             properties => {},
  83         },
  84         links => [ { rel => 'child', href => "{name}" } ],
  85     },
  86     code => sub {
  87         my ($param) = @_;
  88
  89         my $result = [
  90             { name => 'init' },
  91             { name => 'mon' },
  92             { name => 'osd' },
  93             { name => 'pools' },
  94             { name => 'fs' },
  95             { name => 'mds' },
  96             { name => 'stop' },
  97             { name => 'start' },
  98             { name => 'restart' },
  99             { name => 'status' },
 100             { name => 'crush' },
 101             { name => 'config' },
 102             { name => 'log' },
 103             { name => 'rules' },
 104         ];
 105
 106         return $result;
 107     }});
 108
 109 __PACKAGE__->register_method ({
 110     name => 'config',
 111     path => 'config',
 112     method => 'GET',
 113     proxyto => 'node',
 114     permissions => {
 115         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 116     },
 117     description => "Get Ceph configuration.",
 118     parameters => {
 119         additionalProperties => 0,
 120         properties => {
 121             node => get_standard_option('pve-node'),
 122         },
 123     },
 124     returns => { type => 'string' },
 125     code => sub {
 126         my ($param) = @_;
 127
 128         PVE::Ceph::Tools::check_ceph_inited();
 129
 130         my $path = PVE::Ceph::Tools::get_config('pve_ceph_cfgpath');
 131         return file_get_contents($path);
 132
 133     }});
 134
 135 __PACKAGE__->register_method ({
 136     name => 'configdb',
 137     path => 'configdb',
 138     method => 'GET',
 139     proxyto => 'node',
 140     protected => 1,
 141     permissions => {
 142         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 143     },
 144     description => "Get Ceph configuration database.",
 145     parameters => {
 146         additionalProperties => 0,
 147         properties => {
 148             node => get_standard_option('pve-node'),
 149         },
 150     },
 151     returns => {
 152         type => 'array',
 153         items => {
 154             type => 'object',
 155             properties => {
 156                 section => { type => "string", },
 157                 name => { type => "string", },
 158                 value => { type => "string", },
 159                 level => { type => "string", },
 160                 'can_update_at_runtime' => { type => "boolean", },
 161                 mask => { type => "string" },
 162             },
 163         },
 164     },
 165     code => sub {
 166         my ($param) = @_;
 167
 168         PVE::Ceph::Tools::check_ceph_inited();
 169
 170         my $rados = PVE::RADOS->new();
 171         my $res = $rados->mon_command( { prefix => 'config dump', format => 'json' });
 172         foreach my $entry (@$res) {
 173             $entry->{can_update_at_runtime} = $entry->{can_update_at_runtime}? 1 : 0; # JSON::true/false -> 1/0
 174         }
 175
 176         return $res;
 177     }});
 178
 179
 180 __PACKAGE__->register_method ({
 181     name => 'init',
 182     path => 'init',
 183     method => 'POST',
 184     description => "Create initial ceph default configuration and setup symlinks.",
 185     proxyto => 'node',
 186     protected => 1,
 187     permissions => {
 188         check => ['perm', '/', [ 'Sys.Modify' ]],
 189     },
 190     parameters => {
 191         additionalProperties => 0,
 192         properties => {
 193             node => get_standard_option('pve-node'),
 194             network => {
 195                 description => "Use specific network for all ceph related traffic",
 196                 type => 'string', format => 'CIDR',
 197                 optional => 1,
 198                 maxLength => 128,
 199             },
 200             'cluster-network' => {
 201                 description => "Declare a separate cluster network, OSDs will route" .
 202                     "heartbeat, object replication and recovery traffic over it",
 203                 type => 'string', format => 'CIDR',
 204                 requires => 'network',
 205                 optional => 1,
 206                 maxLength => 128,
 207             },
 208             size => {
 209                 description => 'Targeted number of replicas per object',
 210                 type => 'integer',
 211                 default => 3,
 212                 optional => 1,
 213                 minimum => 1,
 214                 maximum => 7,
 215             },
 216             min_size => {
 217                 description => 'Minimum number of available replicas per object to allow I/O',
 218                 type => 'integer',
 219                 default => 2,
 220                 optional => 1,
 221                 minimum => 1,
 222                 maximum => 7,
 223             },
 224             pg_bits => {
 225                 description => "Placement group bits, used to specify the " .
 226                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 227                     "default pg num' does not work for default pools.",
 228                 type => 'integer',
 229                 default => 6,
 230                 optional => 1,
 231                 minimum => 6,
 232                 maximum => 14,
 233             },
 234             disable_cephx => {
 235                 description => "Disable cephx authentication.\n\n" .
 236                     "WARNING: cephx is a security feature protecting against " .
 237                     "man-in-the-middle attacks. Only consider disabling cephx ".
 238                     "if your network is private!",
 239                 type => 'boolean',
 240                 optional => 1,
 241                 default => 0,
 242             },
 243         },
 244     },
 245     returns => { type => 'null' },
 246     code => sub {
 247         my ($param) = @_;
 248
 249         my $version = PVE::Ceph::Tools::get_local_version(1);
 250
 251         if (!$version || $version < 14) {
 252             die "Ceph Nautilus required - please run 'pveceph install'\n";
 253         } else {
 254             PVE::Ceph::Tools::check_ceph_installed('ceph_bin');
 255         }
 256
 257         my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 258
 259         # simply load old config if it already exists
 260         PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
 261             my $cfg = cfs_read_file('ceph.conf');
 262
 263             if (!$cfg->{global}) {
 264
 265                 my $fsid;
 266                 my $uuid;
 267
 268                 UUID::generate($uuid);
 269                 UUID::unparse($uuid, $fsid);
 270
 271                 $cfg->{global} = {
 272                     'fsid' => $fsid,
 273                     'auth cluster required' => $auth,
 274                     'auth service required' => $auth,
 275                     'auth client required' => $auth,
 276                     'osd pool default size' => $param->{size} // 3,
 277                     'osd pool default min size' => $param->{min_size} // 2,
 278                     'mon allow pool delete' => 'true',
 279                 };
 280
 281                 # this does not work for default pools
 282                 #'osd pool default pg num' => $pg_num,
 283                 #'osd pool default pgp num' => $pg_num,
 284             }
 285
 286             if ($auth eq 'cephx') {
 287                 $cfg->{client}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 288             }
 289
 290             if ($param->{pg_bits}) {
 291                 $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 292                 $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 293             }
 294
 295             if ($param->{network}) {
 296                 $cfg->{global}->{'public network'} = $param->{network};
 297                 $cfg->{global}->{'cluster network'} = $param->{network};
 298             }
 299
 300             if ($param->{'cluster-network'}) {
 301                 $cfg->{global}->{'cluster network'} = $param->{'cluster-network'};
 302             }
 303
 304             cfs_write_file('ceph.conf', $cfg);
 305
 306             if ($auth eq 'cephx') {
 307                 PVE::Ceph::Tools::get_or_create_admin_keyring();
 308             }
 309             PVE::Ceph::Tools::setup_pve_symlinks();
 310         });
 311         die $@ if $@;
 312
 313         return undef;
 314     }});
 315
 316 __PACKAGE__->register_method ({
 317     name => 'stop',
 318     path => 'stop',
 319     method => 'POST',
 320     description => "Stop ceph services.",
 321     proxyto => 'node',
 322     protected => 1,
 323     permissions => {
 324         check => ['perm', '/', [ 'Sys.Modify' ]],
 325     },
 326     parameters => {
 327         additionalProperties => 0,
 328         properties => {
 329             node => get_standard_option('pve-node'),
 330             service => {
 331                 description => 'Ceph service name.',
 332                 type => 'string',
 333                 optional => 1,
 334                 default => 'ceph.target',
 335                 pattern => '(ceph|mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 336             },
 337         },
 338     },
 339     returns => { type => 'string' },
 340     code => sub {
 341         my ($param) = @_;
 342
 343         my $rpcenv = PVE::RPCEnvironment::get();
 344
 345         my $authuser = $rpcenv->get_user();
 346
 347         PVE::Ceph::Tools::check_ceph_inited();
 348
 349         my $cfg = cfs_read_file('ceph.conf');
 350         scalar(keys %$cfg) || die "no configuration\n";
 351
 352         my $worker = sub {
 353             my $upid = shift;
 354
 355             my $cmd = ['stop'];
 356             if ($param->{service}) {
 357                 push @$cmd, $param->{service};
 358             }
 359
 360             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 361         };
 362
 363         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
 364                                     $authuser, $worker);
 365     }});
 366
 367 __PACKAGE__->register_method ({
 368     name => 'start',
 369     path => 'start',
 370     method => 'POST',
 371     description => "Start ceph services.",
 372     proxyto => 'node',
 373     protected => 1,
 374     permissions => {
 375         check => ['perm', '/', [ 'Sys.Modify' ]],
 376     },
 377     parameters => {
 378         additionalProperties => 0,
 379         properties => {
 380             node => get_standard_option('pve-node'),
 381             service => {
 382                 description => 'Ceph service name.',
 383                 type => 'string',
 384                 optional => 1,
 385                 default => 'ceph.target',
 386                 pattern => '(ceph|mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 387             },
 388         },
 389     },
 390     returns => { type => 'string' },
 391     code => sub {
 392         my ($param) = @_;
 393
 394         my $rpcenv = PVE::RPCEnvironment::get();
 395
 396         my $authuser = $rpcenv->get_user();
 397
 398         PVE::Ceph::Tools::check_ceph_inited();
 399
 400         my $cfg = cfs_read_file('ceph.conf');
 401         scalar(keys %$cfg) || die "no configuration\n";
 402
 403         my $worker = sub {
 404             my $upid = shift;
 405
 406             my $cmd = ['start'];
 407             if ($param->{service}) {
 408                 push @$cmd, $param->{service};
 409             }
 410
 411             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 412         };
 413
 414         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
 415                                     $authuser, $worker);
 416     }});
 417
 418 __PACKAGE__->register_method ({
 419     name => 'restart',
 420     path => 'restart',
 421     method => 'POST',
 422     description => "Restart ceph services.",
 423     proxyto => 'node',
 424     protected => 1,
 425     permissions => {
 426         check => ['perm', '/', [ 'Sys.Modify' ]],
 427     },
 428     parameters => {
 429         additionalProperties => 0,
 430         properties => {
 431             node => get_standard_option('pve-node'),
 432             service => {
 433                 description => 'Ceph service name.',
 434                 type => 'string',
 435                 optional => 1,
 436                 default => 'ceph.target',
 437                 pattern => '(mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 438             },
 439         },
 440     },
 441     returns => { type => 'string' },
 442     code => sub {
 443         my ($param) = @_;
 444
 445         my $rpcenv = PVE::RPCEnvironment::get();
 446
 447         my $authuser = $rpcenv->get_user();
 448
 449         PVE::Ceph::Tools::check_ceph_inited();
 450
 451         my $cfg = cfs_read_file('ceph.conf');
 452         scalar(keys %$cfg) || die "no configuration\n";
 453
 454         my $worker = sub {
 455             my $upid = shift;
 456
 457             my $cmd = ['restart'];
 458             if ($param->{service}) {
 459                 push @$cmd, $param->{service};
 460             }
 461
 462             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 463         };
 464
 465         return $rpcenv->fork_worker('srvrestart', $param->{service} || 'ceph',
 466                                     $authuser, $worker);
 467     }});
 468
 469 __PACKAGE__->register_method ({
 470     name => 'status',
 471     path => 'status',
 472     method => 'GET',
 473     description => "Get ceph status.",
 474     proxyto => 'node',
 475     protected => 1,
 476     permissions => {
 477         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 478     },
 479     parameters => {
 480         additionalProperties => 0,
 481         properties => {
 482             node => get_standard_option('pve-node'),
 483         },
 484     },
 485     returns => { type => 'object' },
 486     code => sub {
 487         my ($param) = @_;
 488
 489         PVE::Ceph::Tools::check_ceph_inited();
 490
 491         return PVE::Ceph::Tools::ceph_cluster_status();
 492     }});
 493
 494
 495 __PACKAGE__->register_method ({
 496     name => 'crush',
 497     path => 'crush',
 498     method => 'GET',
 499     description => "Get OSD crush map",
 500     proxyto => 'node',
 501     protected => 1,
 502     permissions => {
 503         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 504     },
 505     parameters => {
 506         additionalProperties => 0,
 507         properties => {
 508             node => get_standard_option('pve-node'),
 509         },
 510     },
 511     returns => { type => 'string' },
 512     code => sub {
 513         my ($param) = @_;
 514
 515         PVE::Ceph::Tools::check_ceph_inited();
 516
 517         # this produces JSON (difficult to read for the user)
 518         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
 519
 520         my $txt = '';
 521
 522         my $mapfile = "/var/tmp/ceph-crush.map.$$";
 523         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
 524
 525         my $rados = PVE::RADOS->new();
 526
 527         eval {
 528             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
 529             file_set_contents($mapfile, $bindata);
 530             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
 531             $txt = file_get_contents($mapdata);
 532         };
 533         my $err = $@;
 534
 535         unlink $mapfile;
 536         unlink $mapdata;
 537
 538         die $err if $err;
 539
 540         return $txt;
 541     }});
 542
 543 __PACKAGE__->register_method({
 544     name => 'log',
 545     path => 'log',
 546     method => 'GET',
 547     description => "Read ceph log",
 548     proxyto => 'node',
 549     permissions => {
 550         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
 551     },
 552     protected => 1,
 553     parameters => {
 554         additionalProperties => 0,
 555         properties => {
 556             node => get_standard_option('pve-node'),
 557             start => {
 558                 type => 'integer',
 559                 minimum => 0,
 560                 optional => 1,
 561             },
 562             limit => {
 563                 type => 'integer',
 564                 minimum => 0,
 565                 optional => 1,
 566             },
 567         },
 568     },
 569     returns => {
 570         type => 'array',
 571         items => {
 572             type => "object",
 573             properties => {
 574                 n => {
 575                   description=>  "Line number",
 576                   type=> 'integer',
 577                 },
 578                 t => {
 579                   description=>  "Line text",
 580                   type => 'string',
 581                 }
 582             }
 583         }
 584     },
 585     code => sub {
 586         my ($param) = @_;
 587
 588         PVE::Ceph::Tools::check_ceph_inited();
 589
 590         my $rpcenv = PVE::RPCEnvironment::get();
 591         my $user = $rpcenv->get_user();
 592         my $node = $param->{node};
 593
 594         my $logfile = "/var/log/ceph/ceph.log";
 595         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
 596
 597         $rpcenv->set_result_attrib('total', $count);
 598
 599         return $lines;
 600     }});
 601
 602 __PACKAGE__->register_method ({
 603     name => 'rules',
 604     path => 'rules',
 605     method => 'GET',
 606     description => "List ceph rules.",
 607     proxyto => 'node',
 608     protected => 1,
 609     permissions => {
 610         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 611     },
 612     parameters => {
 613         additionalProperties => 0,
 614         properties => {
 615             node => get_standard_option('pve-node'),
 616         },
 617     },
 618     returns => {
 619         type => 'array',
 620         items => {
 621             type => "object",
 622             properties => {},
 623         },
 624         links => [ { rel => 'child', href => "{name}" } ],
 625     },
 626     code => sub {
 627         my ($param) = @_;
 628
 629         PVE::Ceph::Tools::check_ceph_inited();
 630
 631         my $rados = PVE::RADOS->new();
 632
 633         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
 634
 635         my $res = [];
 636
 637         foreach my $rule (@$rules) {
 638             push @$res, { name => $rule };
 639         }
 640
 641         return $res;
 642     }});
 643
 644 __PACKAGE__->register_method ({
 645     name => 'cmd_safety',
 646     path => 'cmd-safety',
 647     method => 'GET',
 648     description => "Heuristical check if it is safe to perform an action.",
 649     proxyto => 'node',
 650     protected => 1,
 651     permissions => {
 652         check => ['perm', '/', [ 'Sys.audit' ]],
 653     },
 654     parameters => {
 655         additionalProperties => 0,
 656         properties => {
 657             node => get_standard_option('pve-node'),
 658             service => {
 659                 description => 'Service type',
 660                 type => 'string',
 661                 enum => ['osd', 'mon', 'mds'],
 662             },
 663             id => {
 664                 description => 'ID of the service',
 665                 type => 'string',
 666             },
 667             action => {
 668                 description => 'Action to check',
 669                 type => 'string',
 670                 enum => ['stop', 'destroy'],
 671             },
 672         },
 673     },
 674     returns => {
 675         type => 'object',
 676         properties => {
 677            safe  => {
 678                 type => 'boolean',
 679                 description => 'If it is safe to run the command.',
 680             },
 681             status => {
 682                 type => 'string',
 683                 optional => 1,
 684                 description => 'Status message given by Ceph.'
 685             },
 686         },
 687     },
 688     code => sub {
 689         my ($param) = @_;
 690
 691         PVE::Ceph::Tools::check_ceph_inited();
 692
 693         my $id = $param->{id};
 694         my $service = $param->{service};
 695         my $action = $param->{action};
 696
 697         my $rados = PVE::RADOS->new();
 698
 699         my $supported_actions = {
 700             osd => {
 701                 stop => 'ok-to-stop',
 702                 destroy => 'safe-to-destroy',
 703             },
 704             mon => {
 705                 stop => 'ok-to-stop',
 706                 destroy => 'ok-to-rm',
 707             },
 708             mds => {
 709                 stop => 'ok-to-stop',
 710             },
 711         };
 712
 713         die "Service does not support this action: ${service}: ${action}\n"
 714             if !$supported_actions->{$service}->{$action};
 715
 716         my $result = {
 717             safe => 0,
 718             status => '',
 719         };
 720
 721         my $params = {
 722             prefix => "${service} $supported_actions->{$service}->{$action}",
 723             format => 'plain',
 724         };
 725         if ($service eq 'mon' && $action eq 'destroy') {
 726             $params->{id} = $id;
 727         } else {
 728             $params->{ids} = [ $id ];
 729         }
 730
 731         $result = $rados->mon_cmd($params, 1);
 732         die $@ if $@;
 733
 734         $result->{safe} = $result->{return_code} == 0 ? 1 : 0;
 735         $result->{status} = $result->{status_message};
 736
 737         return $result;
 738     }});
 739
 740 1;