PVE/API2/Ceph.pm

   1 package PVE::API2::Ceph;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use Net::IP;
   8 use UUID;
   9
  10 use PVE::Ceph::Tools;
  11 use PVE::Ceph::Services;
  12 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  13 use PVE::JSONSchema qw(get_standard_option);
  14 use PVE::Network;
  15 use PVE::RADOS;
  16 use PVE::RESTHandler;
  17 use PVE::RPCEnvironment;
  18 use PVE::Storage;
  19 use PVE::Tools qw(run_command file_get_contents file_set_contents extract_param);
  20
  21 use PVE::API2::Ceph::Cfg;
  22 use PVE::API2::Ceph::OSD;
  23 use PVE::API2::Ceph::FS;
  24 use PVE::API2::Ceph::MDS;
  25 use PVE::API2::Ceph::MGR;
  26 use PVE::API2::Ceph::MON;
  27 use PVE::API2::Ceph::Pool;
  28 use PVE::API2::Storage::Config;
  29
  30 use base qw(PVE::RESTHandler);
  31
  32 my $pve_osd_default_journal_size = 1024*5;
  33
  34 __PACKAGE__->register_method ({
  35     subclass => "PVE::API2::Ceph::Cfg",
  36     path => 'cfg',
  37 });
  38
  39 __PACKAGE__->register_method ({
  40     subclass => "PVE::API2::Ceph::OSD",
  41     path => 'osd',
  42 });
  43
  44 __PACKAGE__->register_method ({
  45     subclass => "PVE::API2::Ceph::MDS",
  46     path => 'mds',
  47 });
  48
  49 __PACKAGE__->register_method ({
  50     subclass => "PVE::API2::Ceph::MGR",
  51     path => 'mgr',
  52 });
  53
  54 __PACKAGE__->register_method ({
  55     subclass => "PVE::API2::Ceph::MON",
  56     path => 'mon',
  57 });
  58
  59 __PACKAGE__->register_method ({
  60     subclass => "PVE::API2::Ceph::FS",
  61     path => 'fs',
  62 });
  63
  64 __PACKAGE__->register_method ({
  65     subclass => "PVE::API2::Ceph::Pool",
  66     path => 'pool',
  67 });
  68
  69 __PACKAGE__->register_method ({
  70     name => 'index',
  71     path => '',
  72     method => 'GET',
  73     description => "Directory index.",
  74     permissions => { user => 'all' },
  75     permissions => {
  76         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
  77     },
  78     parameters => {
  79         additionalProperties => 0,
  80         properties => {
  81             node => get_standard_option('pve-node'),
  82         },
  83     },
  84     returns => {
  85         type => 'array',
  86         items => {
  87             type => "object",
  88             properties => {},
  89         },
  90         links => [ { rel => 'child', href => "{name}" } ],
  91     },
  92     code => sub {
  93         my ($param) = @_;
  94
  95         my $result = [
  96             { name => 'cmd-safety' },
  97             { name => 'cfg' },
  98             { name => 'config' },
  99             { name => 'configdb' },
 100             { name => 'crush' },
 101             { name => 'fs' },
 102             { name => 'init' },
 103             { name => 'log' },
 104             { name => 'mds' },
 105             { name => 'mgr' },
 106             { name => 'mon' },
 107             { name => 'osd' },
 108             { name => 'pools' },
 109             { name => 'restart' },
 110             { name => 'rules' },
 111             { name => 'start' },
 112             { name => 'status' },
 113             { name => 'stop' },
 114         ];
 115
 116         return $result;
 117     }});
 118
 119 __PACKAGE__->register_method ({
 120     name => 'init',
 121     path => 'init',
 122     method => 'POST',
 123     description => "Create initial ceph default configuration and setup symlinks.",
 124     proxyto => 'node',
 125     protected => 1,
 126     permissions => {
 127         check => ['perm', '/', [ 'Sys.Modify' ]],
 128     },
 129     parameters => {
 130         additionalProperties => 0,
 131         properties => {
 132             node => get_standard_option('pve-node'),
 133             network => {
 134                 description => "Use specific network for all ceph related traffic",
 135                 type => 'string', format => 'CIDR',
 136                 optional => 1,
 137                 maxLength => 128,
 138             },
 139             'cluster-network' => {
 140                 description => "Declare a separate cluster network, OSDs will route" .
 141                     "heartbeat, object replication and recovery traffic over it",
 142                 type => 'string', format => 'CIDR',
 143                 requires => 'network',
 144                 optional => 1,
 145                 maxLength => 128,
 146             },
 147             size => {
 148                 description => 'Targeted number of replicas per object',
 149                 type => 'integer',
 150                 default => 3,
 151                 optional => 1,
 152                 minimum => 1,
 153                 maximum => 7,
 154             },
 155             min_size => {
 156                 description => 'Minimum number of available replicas per object to allow I/O',
 157                 type => 'integer',
 158                 default => 2,
 159                 optional => 1,
 160                 minimum => 1,
 161                 maximum => 7,
 162             },
 163             pg_bits => {
 164                 description => "Placement group bits, used to specify the " .
 165                     "default number of placement groups.\n\nNOTE: 'osd pool " .
 166                     "default pg num' does not work for default pools.",
 167                 type => 'integer',
 168                 default => 6,
 169                 optional => 1,
 170                 minimum => 6,
 171                 maximum => 14,
 172             },
 173             disable_cephx => {
 174                 description => "Disable cephx authentication.\n\n" .
 175                     "WARNING: cephx is a security feature protecting against " .
 176                     "man-in-the-middle attacks. Only consider disabling cephx ".
 177                     "if your network is private!",
 178                 type => 'boolean',
 179                 optional => 1,
 180                 default => 0,
 181             },
 182         },
 183     },
 184     returns => { type => 'null' },
 185     code => sub {
 186         my ($param) = @_;
 187
 188         my $version = PVE::Ceph::Tools::get_local_version(1);
 189
 190         if (!$version || $version < 14) {
 191             die "Ceph Nautilus required - please run 'pveceph install'\n";
 192         } else {
 193             PVE::Ceph::Tools::check_ceph_installed('ceph_bin');
 194         }
 195
 196         my $auth = $param->{disable_cephx} ? 'none' : 'cephx';
 197
 198         # simply load old config if it already exists
 199         PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
 200             my $cfg = cfs_read_file('ceph.conf');
 201
 202             if (!$cfg->{global}) {
 203
 204                 my $fsid;
 205                 my $uuid;
 206
 207                 UUID::generate($uuid);
 208                 UUID::unparse($uuid, $fsid);
 209
 210                 $cfg->{global} = {
 211                     'fsid' => $fsid,
 212                     'auth cluster required' => $auth,
 213                     'auth service required' => $auth,
 214                     'auth client required' => $auth,
 215                     'osd pool default size' => $param->{size} // 3,
 216                     'osd pool default min size' => $param->{min_size} // 2,
 217                     'mon allow pool delete' => 'true',
 218                 };
 219
 220                 # this does not work for default pools
 221                 #'osd pool default pg num' => $pg_num,
 222                 #'osd pool default pgp num' => $pg_num,
 223             }
 224
 225             if ($auth eq 'cephx') {
 226                 $cfg->{client}->{keyring} = '/etc/pve/priv/$cluster.$name.keyring';
 227             }
 228
 229             if ($param->{pg_bits}) {
 230                 $cfg->{global}->{'osd pg bits'} = $param->{pg_bits};
 231                 $cfg->{global}->{'osd pgp bits'} = $param->{pg_bits};
 232             }
 233
 234             if ($param->{network}) {
 235                 $cfg->{global}->{'public network'} = $param->{network};
 236                 $cfg->{global}->{'cluster network'} = $param->{network};
 237             }
 238
 239             if ($param->{'cluster-network'}) {
 240                 $cfg->{global}->{'cluster network'} = $param->{'cluster-network'};
 241             }
 242
 243             cfs_write_file('ceph.conf', $cfg);
 244
 245             if ($auth eq 'cephx') {
 246                 PVE::Ceph::Tools::get_or_create_admin_keyring();
 247             }
 248             PVE::Ceph::Tools::setup_pve_symlinks();
 249         });
 250         die $@ if $@;
 251
 252         return undef;
 253     }});
 254
 255 __PACKAGE__->register_method ({
 256     name => 'stop',
 257     path => 'stop',
 258     method => 'POST',
 259     description => "Stop ceph services.",
 260     proxyto => 'node',
 261     protected => 1,
 262     permissions => {
 263         check => ['perm', '/', [ 'Sys.Modify' ]],
 264     },
 265     parameters => {
 266         additionalProperties => 0,
 267         properties => {
 268             node => get_standard_option('pve-node'),
 269             service => {
 270                 description => 'Ceph service name.',
 271                 type => 'string',
 272                 optional => 1,
 273                 default => 'ceph.target',
 274                 pattern => '(ceph|mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 275             },
 276         },
 277     },
 278     returns => { type => 'string' },
 279     code => sub {
 280         my ($param) = @_;
 281
 282         my $rpcenv = PVE::RPCEnvironment::get();
 283
 284         my $authuser = $rpcenv->get_user();
 285
 286         PVE::Ceph::Tools::check_ceph_inited();
 287
 288         my $cfg = cfs_read_file('ceph.conf');
 289         scalar(keys %$cfg) || die "no configuration\n";
 290
 291         my $worker = sub {
 292             my $upid = shift;
 293
 294             my $cmd = ['stop'];
 295             if ($param->{service}) {
 296                 push @$cmd, $param->{service};
 297             }
 298
 299             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 300         };
 301
 302         return $rpcenv->fork_worker('srvstop', $param->{service} || 'ceph',
 303                                     $authuser, $worker);
 304     }});
 305
 306 __PACKAGE__->register_method ({
 307     name => 'start',
 308     path => 'start',
 309     method => 'POST',
 310     description => "Start ceph services.",
 311     proxyto => 'node',
 312     protected => 1,
 313     permissions => {
 314         check => ['perm', '/', [ 'Sys.Modify' ]],
 315     },
 316     parameters => {
 317         additionalProperties => 0,
 318         properties => {
 319             node => get_standard_option('pve-node'),
 320             service => {
 321                 description => 'Ceph service name.',
 322                 type => 'string',
 323                 optional => 1,
 324                 default => 'ceph.target',
 325                 pattern => '(ceph|mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 326             },
 327         },
 328     },
 329     returns => { type => 'string' },
 330     code => sub {
 331         my ($param) = @_;
 332
 333         my $rpcenv = PVE::RPCEnvironment::get();
 334
 335         my $authuser = $rpcenv->get_user();
 336
 337         PVE::Ceph::Tools::check_ceph_inited();
 338
 339         my $cfg = cfs_read_file('ceph.conf');
 340         scalar(keys %$cfg) || die "no configuration\n";
 341
 342         my $worker = sub {
 343             my $upid = shift;
 344
 345             my $cmd = ['start'];
 346             if ($param->{service}) {
 347                 push @$cmd, $param->{service};
 348             }
 349
 350             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 351         };
 352
 353         return $rpcenv->fork_worker('srvstart', $param->{service} || 'ceph',
 354                                     $authuser, $worker);
 355     }});
 356
 357 __PACKAGE__->register_method ({
 358     name => 'restart',
 359     path => 'restart',
 360     method => 'POST',
 361     description => "Restart ceph services.",
 362     proxyto => 'node',
 363     protected => 1,
 364     permissions => {
 365         check => ['perm', '/', [ 'Sys.Modify' ]],
 366     },
 367     parameters => {
 368         additionalProperties => 0,
 369         properties => {
 370             node => get_standard_option('pve-node'),
 371             service => {
 372                 description => 'Ceph service name.',
 373                 type => 'string',
 374                 optional => 1,
 375                 default => 'ceph.target',
 376                 pattern => '(mon|mds|osd|mgr)(\.'.PVE::Ceph::Services::SERVICE_REGEX.')?',
 377             },
 378         },
 379     },
 380     returns => { type => 'string' },
 381     code => sub {
 382         my ($param) = @_;
 383
 384         my $rpcenv = PVE::RPCEnvironment::get();
 385
 386         my $authuser = $rpcenv->get_user();
 387
 388         PVE::Ceph::Tools::check_ceph_inited();
 389
 390         my $cfg = cfs_read_file('ceph.conf');
 391         scalar(keys %$cfg) || die "no configuration\n";
 392
 393         my $worker = sub {
 394             my $upid = shift;
 395
 396             my $cmd = ['restart'];
 397             if ($param->{service}) {
 398                 push @$cmd, $param->{service};
 399             }
 400
 401             PVE::Ceph::Services::ceph_service_cmd(@$cmd);
 402         };
 403
 404         return $rpcenv->fork_worker('srvrestart', $param->{service} || 'ceph',
 405                                     $authuser, $worker);
 406     }});
 407
 408 __PACKAGE__->register_method ({
 409     name => 'status',
 410     path => 'status',
 411     method => 'GET',
 412     description => "Get ceph status.",
 413     proxyto => 'node',
 414     protected => 1,
 415     permissions => {
 416         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 417     },
 418     parameters => {
 419         additionalProperties => 0,
 420         properties => {
 421             node => get_standard_option('pve-node'),
 422         },
 423     },
 424     returns => { type => 'object' },
 425     code => sub {
 426         my ($param) = @_;
 427
 428         PVE::Ceph::Tools::check_ceph_inited();
 429
 430         return PVE::Ceph::Tools::ceph_cluster_status();
 431     }});
 432
 433
 434 __PACKAGE__->register_method ({
 435     name => 'crush',
 436     path => 'crush',
 437     method => 'GET',
 438     description => "Get OSD crush map",
 439     proxyto => 'node',
 440     protected => 1,
 441     permissions => {
 442         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 443     },
 444     parameters => {
 445         additionalProperties => 0,
 446         properties => {
 447             node => get_standard_option('pve-node'),
 448         },
 449     },
 450     returns => { type => 'string' },
 451     code => sub {
 452         my ($param) = @_;
 453
 454         PVE::Ceph::Tools::check_ceph_inited();
 455
 456         # this produces JSON (difficult to read for the user)
 457         # my $txt = &$run_ceph_cmd_text(['osd', 'crush', 'dump'], quiet => 1);
 458
 459         my $txt = '';
 460
 461         my $mapfile = "/var/tmp/ceph-crush.map.$$";
 462         my $mapdata = "/var/tmp/ceph-crush.txt.$$";
 463
 464         my $rados = PVE::RADOS->new();
 465
 466         eval {
 467             my $bindata = $rados->mon_command({ prefix => 'osd getcrushmap', format => 'plain' });
 468             file_set_contents($mapfile, $bindata);
 469             run_command(['crushtool', '-d', $mapfile, '-o', $mapdata]);
 470             $txt = file_get_contents($mapdata);
 471         };
 472         my $err = $@;
 473
 474         unlink $mapfile;
 475         unlink $mapdata;
 476
 477         die $err if $err;
 478
 479         return $txt;
 480     }});
 481
 482 __PACKAGE__->register_method({
 483     name => 'log',
 484     path => 'log',
 485     method => 'GET',
 486     description => "Read ceph log",
 487     proxyto => 'node',
 488     permissions => {
 489         check => ['perm', '/nodes/{node}', [ 'Sys.Syslog' ]],
 490     },
 491     protected => 1,
 492     parameters => {
 493         additionalProperties => 0,
 494         properties => {
 495             node => get_standard_option('pve-node'),
 496             start => {
 497                 type => 'integer',
 498                 minimum => 0,
 499                 optional => 1,
 500             },
 501             limit => {
 502                 type => 'integer',
 503                 minimum => 0,
 504                 optional => 1,
 505             },
 506         },
 507     },
 508     returns => {
 509         type => 'array',
 510         items => {
 511             type => "object",
 512             properties => {
 513                 n => {
 514                   description=>  "Line number",
 515                   type=> 'integer',
 516                 },
 517                 t => {
 518                   description=>  "Line text",
 519                   type => 'string',
 520                 }
 521             }
 522         }
 523     },
 524     code => sub {
 525         my ($param) = @_;
 526
 527         PVE::Ceph::Tools::check_ceph_inited();
 528
 529         my $rpcenv = PVE::RPCEnvironment::get();
 530         my $user = $rpcenv->get_user();
 531         my $node = $param->{node};
 532
 533         my $logfile = "/var/log/ceph/ceph.log";
 534         my ($count, $lines) = PVE::Tools::dump_logfile($logfile, $param->{start}, $param->{limit});
 535
 536         $rpcenv->set_result_attrib('total', $count);
 537
 538         return $lines;
 539     }});
 540
 541 __PACKAGE__->register_method ({
 542     name => 'rules',
 543     path => 'rules',
 544     method => 'GET',
 545     description => "List ceph rules.",
 546     proxyto => 'node',
 547     protected => 1,
 548     permissions => {
 549         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 550     },
 551     parameters => {
 552         additionalProperties => 0,
 553         properties => {
 554             node => get_standard_option('pve-node'),
 555         },
 556     },
 557     returns => {
 558         type => 'array',
 559         items => {
 560             type => "object",
 561             properties => {
 562                 name => {
 563                     description => "Name of the CRUSH rule.",
 564                     type => "string",
 565                 }
 566             },
 567         },
 568         links => [ { rel => 'child', href => "{name}" } ],
 569     },
 570     code => sub {
 571         my ($param) = @_;
 572
 573         PVE::Ceph::Tools::check_ceph_inited();
 574
 575         my $rados = PVE::RADOS->new();
 576
 577         my $rules = $rados->mon_command({ prefix => 'osd crush rule ls' });
 578
 579         my $res = [];
 580
 581         foreach my $rule (@$rules) {
 582             push @$res, { name => $rule };
 583         }
 584
 585         return $res;
 586     }});
 587
 588 __PACKAGE__->register_method ({
 589     name => 'cmd_safety',
 590     path => 'cmd-safety',
 591     method => 'GET',
 592     description => "Heuristical check if it is safe to perform an action.",
 593     proxyto => 'node',
 594     protected => 1,
 595     permissions => {
 596         check => ['perm', '/', [ 'Sys.Audit' ]],
 597     },
 598     parameters => {
 599         additionalProperties => 0,
 600         properties => {
 601             node => get_standard_option('pve-node'),
 602             service => {
 603                 description => 'Service type',
 604                 type => 'string',
 605                 enum => ['osd', 'mon', 'mds'],
 606             },
 607             id => {
 608                 description => 'ID of the service',
 609                 type => 'string',
 610             },
 611             action => {
 612                 description => 'Action to check',
 613                 type => 'string',
 614                 enum => ['stop', 'destroy'],
 615             },
 616         },
 617     },
 618     returns => {
 619         type => 'object',
 620         properties => {
 621            safe  => {
 622                 type => 'boolean',
 623                 description => 'If it is safe to run the command.',
 624             },
 625             status => {
 626                 type => 'string',
 627                 optional => 1,
 628                 description => 'Status message given by Ceph.'
 629             },
 630         },
 631     },
 632     code => sub {
 633         my ($param) = @_;
 634
 635         PVE::Ceph::Tools::check_ceph_inited();
 636
 637         my $id = $param->{id};
 638         my $service = $param->{service};
 639         my $action = $param->{action};
 640
 641         my $rados = PVE::RADOS->new();
 642
 643         my $supported_actions = {
 644             osd => {
 645                 stop => 'ok-to-stop',
 646                 destroy => 'safe-to-destroy',
 647             },
 648             mon => {
 649                 stop => 'ok-to-stop',
 650                 destroy => 'ok-to-rm',
 651             },
 652             mds => {
 653                 stop => 'ok-to-stop',
 654             },
 655         };
 656
 657         die "Service does not support this action: ${service}: ${action}\n"
 658             if !$supported_actions->{$service}->{$action};
 659
 660         my $result = {
 661             safe => 0,
 662             status => '',
 663         };
 664
 665         my $params = {
 666             prefix => "${service} $supported_actions->{$service}->{$action}",
 667             format => 'plain',
 668         };
 669         if ($service eq 'mon' && $action eq 'destroy') {
 670             $params->{id} = $id;
 671         } else {
 672             $params->{ids} = [ $id ];
 673         }
 674
 675         $result = $rados->mon_cmd($params, 1);
 676         die $@ if $@;
 677
 678         $result->{safe} = $result->{return_code} == 0 ? 1 : 0;
 679         $result->{status} = $result->{status_message};
 680
 681         return $result;
 682     }});
 683
 684 1;