PVE/API2/Ceph.pm

   1 package PVE::API2::Ceph;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use Net::IP;
   8 use UUID;
   9
  10 use PVE::Ceph::Tools;
  11 use PVE::Ceph::Services;
  12 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  13 use PVE::JSONSchema qw(get_standard_option);
  14 use PVE::Network;
  15 use PVE::RADOS;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::Storage;
  19 use PVE::Tools qw(run_command file_get_contents file_set_contents extract_param);
  20
  21 use PVE::API2::Ceph::Cfg;
  22 use PVE::API2::Ceph::OSD;
  23 use PVE::API2::Ceph::FS;
  24 use PVE::API2::Ceph::MDS;
  25 use PVE::API2::Ceph::MGR;
  26 use PVE::API2::Ceph::MON;
  27 use PVE::API2::Ceph::Pool;
  28 use PVE::API2::Storage::Config;
  29
  30 use base qw(PVE::RESTHandler);
  31
  32 my $pve_osd_default_journal_size = 1024*5;
  33
  34 __PACKAGE__->register_method ({
  35     subclass => "PVE::API2::Ceph::Cfg",
  36     path => 'cfg',
  37 });
  38
  39 __PACKAGE__->register_method ({
  40     subclass => "PVE::API2::Ceph::OSD",
  41     path => 'osd',
  42 });
  43
  44 __PACKAGE__->register_method ({
  45     subclass => "PVE::API2::Ceph::MDS",
  46     path => 'mds',
  47 });
  48
  49 __PACKAGE__->register_method ({
  50     subclass => "PVE::API2::Ceph::MGR",
  51     path => 'mgr',
  52 });
  53
  54 __PACKAGE__->register_method ({
  55     subclass => "PVE::API2::Ceph::MON",
  56     path => 'mon',
  57 });
  58
  59 __PACKAGE__->register_method ({
  60     subclass => "PVE::API2::Ceph::FS",
  61     path => 'fs',
  62 });
  63
  64 __PACKAGE__->register_method ({
  65     subclass => "PVE::API2::Ceph::Pool",
  66     path => 'pool',
  67 });
  68
  69 __PACKAGE__->register_method ({
  70     name => 'index',
  71     path => '',
  72     method => 'GET',
  73     description => "Directory index.",
  74     permissions => { user => 'all' },
  75     permissions => {
  76         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  77     },
  78     parameters => {
  79         additionalProperties => 0,
  80         properties => {
  81             node => get_standard_option('pve-node'),
  82         },
  83     },
  84     returns => {
  85         type => 'array',
  86         items => {
  87             type => "object",
  88             properties => {},
  89         },
  90         links => [ { rel => 'child', href => "{name}" } ],
  91     },
  92     code => sub {
  93         my ($param) = @_;
  94
  95         my $result = [
  96             { name => 'cmd-safety' },
  97             { name => 'cfg' },
  98             { name => 'crush' },
  99             { name => 'fs' },
 100             { name => 'init' },
 101             { name => 'log' },
 102             { name => 'mds' },
 103             { name => 'mgr' },
 104             { name => 'mon' },
 105             { name => 'osd' },
 106             { name => 'pool' },
 107             { name => 'restart' },
 108             { name => 'rules' },
 109             { name => 'start' },
 110             { name => 'status' },
 111             { name => 'stop' },
 112         ];
 113
 114         return $result;
 115     }});
 116
 117 __PACKAGE__->register_method ({
 118     name => 'init',
 119     path => 'init',
 120     method => 'POST',
 121     description => "Create initial ceph default configuration and setup symlinks.",
 122     proxyto => 'node',
 123     protected => 1,
 124     permissions => {
 125         check => ['perm', '/', [ 'Sys.Modify' ]],
 126     },
 127     parameters => {
 128         additionalProperties => 0,
 129         properties => {
 130             node => get_standard_option('pve-node'),
 131             network => {
 132                 description => "Use specific network for all ceph related traffic",
 133                 type => 'string', format => 'CIDR',
 134                 optional => 1,
 135                 maxLength => 128,
 136             },
 137             'cluster-network' => {
 138                 description => "Declare a separate cluster network, OSDs will route" .
 139                     "heartbeat, object replication and recovery traffic over it",
 140                 type => 'string', format => 'CIDR',
 141                 requires => 'network',
 142                 optional => 1,
 143                 maxLength => 128,
 144             },
 145             size => {
 146                 description => 'Targeted number of replicas per object',
 147                 type => 'integer',
 148                 default => 3,
 149                 optional => 1,
 150                 minimum => 1,
 151                 maximum => 7,
 152             },
 153             min_size => {
 154                 description => 'Minimum number of available replicas per object to allow I/O',
 155                 type => 'integer',
 156                 default => 2,
 157                 optional => 1,
 158                 minimum => 1,
 159                 maximum => 7,
 160             },
 161             # TODO: deprecrated, remove with PVE 9
 162             pg_bits => {
 163                 description => "Placement group bits, used to specify the " .
 164                     "default number of placement groups.\n\nDepreacted. This " .
 165                     "setting was deprecated in recent Ceph versions.",
 166                 type => 'integer',
 167                 default => 6,
 168                 optional => 1,
 169                 minimum => 6,
 170                 maximum => 14,
 171             },
 172             disable_cephx => {
 173                 description => "Disable cephx authentication.\n\n" .
 174                     "WARNING: cephx is a security feature protecting against " .
 175                     "man-in-the-middle attacks. Only consider disabling cephx ".
 176                     "if your network is private!",
 177                 type => 'boolean',
 178                 optional => 1,
 179                 default => 0,
 180             },
 181         },
 182     },
 183     returns => { type => 'null' },
 184     code => sub {
 185         my ($param) = @_;
 186
 187         my $version = PVE::Ceph::Tools::get_local_version(1);
 188
 189         if (!$version || $version < 14) {
 190             die "Ceph Nautilus required - please run 'pveceph install'\n";
 191         } else {
 192             PVE::Ceph::Tools::check_ceph_installed('ceph_bin');
 193         }
 194
 195         my $pve_ceph_cfgdir = PVE::Ceph::Tools::get_config('pve_ceph_cfgdir');
 196         if (! -d $pve_ceph_cfgdir) {
 197             File::Path::make_path($pve_ceph_cfgdir);
 198         }
 199
 200         my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 201
 202         # simply load old config if it already exists
 203         PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
 204             my $cfg = cfs_read_file('ceph.conf');
 205
 206             if (!$cfg->{global}) {
 207
 208                 my $fsid;
 209                 my $uuid;
 210
 211                 UUID::generate($uuid);
 212                 UUID::unparse($uuid, $fsid);
 213
 214                 $cfg->{global} = {
 215                     'fsid' => $fsid,
 216                     'auth_cluster_required' => $auth,
 217                     'auth_service_required' => $auth,
 218                     'auth_client_required' => $auth,
 219                     'osd_pool_default_size' => $param->{size} // 3,
 220                     'osd_pool_default_min_size' => $param->{min_size} // 2,
 221                     'mon_allow_pool_delete' => 'true',
 222                 };
 223
 224                 # this does not work for default pools
 225                 #'osd pool default pg num' => $pg_num,
 226                 #'osd pool default pgp num' => $pg_num,
 227             }
 228
 229             if ($auth eq 'cephx') {
 230                 $cfg->{client}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 231             }
 232
 233             if ($param->{network}) {
 234                 $cfg->{global}->{'public_network'} = $param->{network};
 235                 $cfg->{global}->{'cluster_network'} = $param->{network};
 236             }
 237
 238             if ($param->{'cluster-network'}) {
 239                 $cfg->{global}->{'cluster_network'} = $param->{'cluster-network'};
 240             }
 241
 242             cfs_write_file('ceph.conf', $cfg);
 243
 244             if ($auth eq 'cephx') {
 245                 PVE::Ceph::Tools::get_or_create_admin_keyring();
 246             }
 247             PVE::Ceph::Tools::setup_pve_symlinks();
 248         });
 249         die $@ if $@;
 250
 251         return undef;
 252     }});
 253
 254 __PACKAGE__->register_method ({
 255     name => 'stop',
 256     path => 'stop',
 257     method => 'POST',
 258     description => "Stop ceph services.",
 259     proxyto => 'node',
 260     protected => 1,
 261     permissions => {
 262         check => ['perm', '/', [ 'Sys.Modify' ]],
 263     },
 264     parameters => {
 265         additionalProperties => 0,
 266         properties => {
 267             node => get_standard_option('pve-node'),
 268             service => {
 269                 description => 'Ceph service name.',
 270                 type => 'string',
 271                 optional => 1,
 272                 default => 'ceph.target',
 273                 pattern => '(ceph|mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 274             },
 275         },
 276     },
 277     returns => { type => 'string' },
 278     code => sub {
 279         my ($param) = @_;
 280
 281         my $rpcenv = PVE::RPCEnvironment::get();
 282
 283         my $authuser = $rpcenv->get_user();
 284
 285         PVE::Ceph::Tools::check_ceph_inited();
 286
 287         my $cfg = cfs_read_file('ceph.conf');
 288         scalar(keys %$cfg) || die "no configuration\n";
 289
 290         my $worker = sub {
 291             my $upid = shift;
 292
 293             my $cmd = ['stop'];
 294             if ($param->{service}) {
 295                 push @$cmd, $param->{service};
 296             }
 297
 298             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 299         };
 300
 301         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
 302                                     $authuser, $worker);
 303     }});
 304
 305 __PACKAGE__->register_method ({
 306     name => 'start',
 307     path => 'start',
 308     method => 'POST',
 309     description => "Start ceph services.",
 310     proxyto => 'node',
 311     protected => 1,
 312     permissions => {
 313         check => ['perm', '/', [ 'Sys.Modify' ]],
 314     },
 315     parameters => {
 316         additionalProperties => 0,
 317         properties => {
 318             node => get_standard_option('pve-node'),
 319             service => {
 320                 description => 'Ceph service name.',
 321                 type => 'string',
 322                 optional => 1,
 323                 default => 'ceph.target',
 324                 pattern => '(ceph|mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 325             },
 326         },
 327     },
 328     returns => { type => 'string' },
 329     code => sub {
 330         my ($param) = @_;
 331
 332         my $rpcenv = PVE::RPCEnvironment::get();
 333
 334         my $authuser = $rpcenv->get_user();
 335
 336         PVE::Ceph::Tools::check_ceph_inited();
 337
 338         my $cfg = cfs_read_file('ceph.conf');
 339         scalar(keys %$cfg) || die "no configuration\n";
 340
 341         my $worker = sub {
 342             my $upid = shift;
 343
 344             my $cmd = ['start'];
 345             if ($param->{service}) {
 346                 push @$cmd, $param->{service};
 347             }
 348
 349             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 350         };
 351
 352         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
 353                                     $authuser, $worker);
 354     }});
 355
 356 __PACKAGE__->register_method ({
 357     name => 'restart',
 358     path => 'restart',
 359     method => 'POST',
 360     description => "Restart ceph services.",
 361     proxyto => 'node',
 362     protected => 1,
 363     permissions => {
 364         check => ['perm', '/', [ 'Sys.Modify' ]],
 365     },
 366     parameters => {
 367         additionalProperties => 0,
 368         properties => {
 369             node => get_standard_option('pve-node'),
 370             service => {
 371                 description => 'Ceph service name.',
 372                 type => 'string',
 373                 optional => 1,
 374                 default => 'ceph.target',
 375                 pattern => '(mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 376             },
 377         },
 378     },
 379     returns => { type => 'string' },
 380     code => sub {
 381         my ($param) = @_;
 382
 383         my $rpcenv = PVE::RPCEnvironment::get();
 384
 385         my $authuser = $rpcenv->get_user();
 386
 387         PVE::Ceph::Tools::check_ceph_inited();
 388
 389         my $cfg = cfs_read_file('ceph.conf');
 390         scalar(keys %$cfg) || die "no configuration\n";
 391
 392         my $worker = sub {
 393             my $upid = shift;
 394
 395             my $cmd = ['restart'];
 396             if ($param->{service}) {
 397                 push @$cmd, $param->{service};
 398             }
 399
 400             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 401         };
 402
 403         return $rpcenv->fork_worker('srvrestart', $param->{service} || 'ceph',
 404                                     $authuser, $worker);
 405     }});
 406
 407 __PACKAGE__->register_method ({
 408     name => 'status',
 409     path => 'status',
 410     method => 'GET',
 411     description => "Get ceph status.",
 412     proxyto => 'node',
 413     protected => 1,
 414     permissions => {
 415         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 416     },
 417     parameters => {
 418         additionalProperties => 0,
 419         properties => {
 420             node => get_standard_option('pve-node'),
 421         },
 422     },
 423     returns => { type => 'object' },
 424     code => sub {
 425         my ($param) = @_;
 426
 427         PVE::Ceph::Tools::check_ceph_inited();
 428
 429         return PVE::Ceph::Tools::ceph_cluster_status();
 430     }});
 431
 432
 433 __PACKAGE__->register_method ({
 434     name => 'crush',
 435     path => 'crush',
 436     method => 'GET',
 437     description => "Get OSD crush map",
 438     proxyto => 'node',
 439     protected => 1,
 440     permissions => {
 441         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 442     },
 443     parameters => {
 444         additionalProperties => 0,
 445         properties => {
 446             node => get_standard_option('pve-node'),
 447         },
 448     },
 449     returns => { type => 'string' },
 450     code => sub {
 451         my ($param) = @_;
 452
 453         PVE::Ceph::Tools::check_ceph_inited();
 454
 455         # this produces JSON (difficult to read for the user)
 456         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
 457
 458         my $txt = '';
 459
 460         my $mapfile = "/var/tmp/ceph-crush.map.$$";
 461         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
 462
 463         my $rados = PVE::RADOS->new();
 464
 465         eval {
 466             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
 467             file_set_contents($mapfile, $bindata);
 468             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
 469             $txt = file_get_contents($mapdata);
 470         };
 471         my $err = $@;
 472
 473         unlink $mapfile;
 474         unlink $mapdata;
 475
 476         die $err if $err;
 477
 478         return $txt;
 479     }});
 480
 481 __PACKAGE__->register_method({
 482     name => 'log',
 483     path => 'log',
 484     method => 'GET',
 485     description => "Read ceph log",
 486     proxyto => 'node',
 487     permissions => {
 488         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
 489     },
 490     protected => 1,
 491     parameters => {
 492         additionalProperties => 0,
 493         properties => {
 494             node => get_standard_option('pve-node'),
 495             start => {
 496                 type => 'integer',
 497                 minimum => 0,
 498                 optional => 1,
 499             },
 500             limit => {
 501                 type => 'integer',
 502                 minimum => 0,
 503                 optional => 1,
 504             },
 505         },
 506     },
 507     returns => {
 508         type => 'array',
 509         items => {
 510             type => "object",
 511             properties => {
 512                 n => {
 513                   description=>  "Line number",
 514                   type=> 'integer',
 515                 },
 516                 t => {
 517                   description=>  "Line text",
 518                   type => 'string',
 519                 }
 520             }
 521         }
 522     },
 523     code => sub {
 524         my ($param) = @_;
 525
 526         PVE::Ceph::Tools::check_ceph_inited();
 527
 528         my $rpcenv = PVE::RPCEnvironment::get();
 529         my $user = $rpcenv->get_user();
 530         my $node = $param->{node};
 531
 532         my $logfile = "/var/log/ceph/ceph.log";
 533         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
 534
 535         $rpcenv->set_result_attrib('total', $count);
 536
 537         return $lines;
 538     }});
 539
 540 __PACKAGE__->register_method ({
 541     name => 'rules',
 542     path => 'rules',
 543     method => 'GET',
 544     description => "List ceph rules.",
 545     proxyto => 'node',
 546     protected => 1,
 547     permissions => {
 548         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 549     },
 550     parameters => {
 551         additionalProperties => 0,
 552         properties => {
 553             node => get_standard_option('pve-node'),
 554         },
 555     },
 556     returns => {
 557         type => 'array',
 558         items => {
 559             type => "object",
 560             properties => {
 561                 name => {
 562                     description => "Name of the CRUSH rule.",
 563                     type => "string",
 564                 }
 565             },
 566         },
 567         links => [ { rel => 'child', href => "{name}" } ],
 568     },
 569     code => sub {
 570         my ($param) = @_;
 571
 572         PVE::Ceph::Tools::check_ceph_inited();
 573
 574         my $rados = PVE::RADOS->new();
 575
 576         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
 577
 578         my $res = [];
 579
 580         foreach my $rule (@$rules) {
 581             push @$res, { name => $rule };
 582         }
 583
 584         return $res;
 585     }});
 586
 587 __PACKAGE__->register_method ({
 588     name => 'cmd_safety',
 589     path => 'cmd-safety',
 590     method => 'GET',
 591     description => "Heuristical check if it is safe to perform an action.",
 592     proxyto => 'node',
 593     protected => 1,
 594     permissions => {
 595         check => ['perm', '/', [ 'Sys.Audit' ]],
 596     },
 597     parameters => {
 598         additionalProperties => 0,
 599         properties => {
 600             node => get_standard_option('pve-node'),
 601             service => {
 602                 description => 'Service type',
 603                 type => 'string',
 604                 enum => ['osd', 'mon', 'mds'],
 605             },
 606             id => {
 607                 description => 'ID of the service',
 608                 type => 'string',
 609             },
 610             action => {
 611                 description => 'Action to check',
 612                 type => 'string',
 613                 enum => ['stop', 'destroy'],
 614             },
 615         },
 616     },
 617     returns => {
 618         type => 'object',
 619         properties => {
 620            safe  => {
 621                 type => 'boolean',
 622                 description => 'If it is safe to run the command.',
 623             },
 624             status => {
 625                 type => 'string',
 626                 optional => 1,
 627                 description => 'Status message given by Ceph.'
 628             },
 629         },
 630     },
 631     code => sub {
 632         my ($param) = @_;
 633
 634         PVE::Ceph::Tools::check_ceph_inited();
 635
 636         my $id = $param->{id};
 637         my $service = $param->{service};
 638         my $action = $param->{action};
 639
 640         my $rados = PVE::RADOS->new();
 641
 642         my $supported_actions = {
 643             osd => {
 644                 stop => 'ok-to-stop',
 645                 destroy => 'safe-to-destroy',
 646             },
 647             mon => {
 648                 stop => 'ok-to-stop',
 649                 destroy => 'ok-to-rm',
 650             },
 651             mds => {
 652                 stop => 'ok-to-stop',
 653             },
 654         };
 655
 656         die "Service does not support this action: ${service}: ${action}\n"
 657             if !$supported_actions->{$service}->{$action};
 658
 659         my $result = {
 660             safe => 0,
 661             status => '',
 662         };
 663
 664         my $params = {
 665             prefix => "${service} $supported_actions->{$service}->{$action}",
 666             format => 'plain',
 667         };
 668         if ($service eq 'mon' && $action eq 'destroy') {
 669             $params->{id} = $id;
 670         } else {
 671             $params->{ids} = [ $id ];
 672         }
 673
 674         $result = $rados->mon_cmd($params, 1);
 675         die $@ if $@;
 676
 677         $result->{safe} = $result->{return_code} == 0 ? 1 : 0;
 678         $result->{status} = $result->{status_message};
 679
 680         return $result;
 681     }});
 682
 683 1;