PVE/API2/Ceph/MON.pm

   1 package PVE::API2::Ceph::MON;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Net::IP;
   7 use File::Path;
   8
   9 use PVE::Ceph::Tools;
  10 use PVE::Ceph::Services;
  11 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  12 use PVE::JSONSchema qw(get_standard_option);
  13 use PVE::Network;
  14 use PVE::RADOS;
  15 use PVE::RESTHandler;
  16 use PVE::RPCEnvironment;
  17 use PVE::Tools qw(run_command file_set_contents);
  18 use PVE::CephConfig;
  19 use PVE::API2::Ceph::MGR;
  20
  21 use base qw(PVE::RESTHandler);
  22
  23 my $find_mon_ips = sub {
  24     my ($cfg, $rados, $node, $mon_address) = @_;
  25
  26     my $overwrite_ips = [ PVE::Tools::split_list($mon_address) ];
  27     $overwrite_ips = PVE::Network::unique_ips($overwrite_ips);
  28
  29     my $pubnet;
  30     if ($rados) {
  31         $pubnet = $rados->mon_command({ prefix => "config get" , who => "mon.",
  32                 key => "public_network", format => 'plain' });
  33         # if not defined in the db, the result is empty, it is also always
  34         # followed by a newline
  35         ($pubnet) = $pubnet =~ m/^(\S+)$/;
  36     }
  37     $pubnet //= $cfg->{global}->{public_network};
  38
  39     if (!$pubnet) {
  40         if (scalar(@{$overwrite_ips})) {
  41             return $overwrite_ips;
  42         } else {
  43            # don't refactor into '[ PVE::Cluster::remote... ]' as it uses wantarray
  44            my $ip = PVE::Cluster::remote_node_ip($node);
  45            return [ $ip ];
  46         }
  47     }
  48
  49     my $public_nets = [ PVE::Tools::split_list($pubnet) ];
  50     if (scalar(@{$public_nets}) > 1) {
  51         warn "Multiple Ceph public networks detected on $node: $pubnet\n";
  52         warn "Networks must be capable of routing to each other.\n";
  53     }
  54
  55     my $res = [];
  56
  57     if (!scalar(@{$overwrite_ips})) { # auto-select one address for each public network
  58         for my $net (@{$public_nets}) {
  59             $net = PVE::JSONSchema::pve_verify_cidr($net);
  60
  61             my $allowed_ips = PVE::Network::get_local_ip_from_cidr($net);
  62             $allowed_ips = PVE::Network::unique_ips($allowed_ips);
  63
  64             die "No active IP found for the requested ceph public network '$net' on node '$node'\n"
  65                 if scalar(@$allowed_ips) < 1;
  66
  67             if (scalar(@$allowed_ips) == 1) {
  68                 push @{$res}, $allowed_ips->[0];
  69             } else {
  70                 die "Multiple IPs for ceph public network '$net' detected on $node:\n".
  71                     join("\n", @$allowed_ips) ."\nuse 'mon-address' to specify one of them.\n";
  72             }
  73         }
  74     } else { # check if overwrite IPs are active and in any of the public networks
  75         my $allowed_list = [];
  76
  77         for my $net (@{$public_nets}) {
  78             $net = PVE::JSONSchema::pve_verify_cidr($net);
  79
  80             push @{$allowed_list}, @{PVE::Network::get_local_ip_from_cidr($net)};
  81         }
  82
  83         my $allowed_ips = PVE::Network::unique_ips($allowed_list);
  84
  85         for my $overwrite_ip (@{$overwrite_ips}) {
  86             die "Specified monitor IP '$overwrite_ip' not configured or up on $node!\n"
  87                 if !grep { $_ eq $overwrite_ip } @{$allowed_ips};
  88
  89             push @{$res}, $overwrite_ip;
  90         }
  91     }
  92
  93     return $res;
  94 };
  95
  96 my $ips_from_mon_host = sub {
  97     my ($mon_host) = @_;
  98
  99     my $ips = [];
 100
 101     my @hosts = PVE::Tools::split_list($mon_host);
 102
 103     for my $host (@hosts) {
 104         $host =~ s|^\[?v\d+\:||; # remove beginning of vector
 105         $host =~ s|/\d+\]?||; # remove end of vector
 106
 107         ($host) = PVE::Tools::parse_host_and_port($host);
 108         next if !defined($host);
 109
 110         # filter out hostnames
 111         my $ip = PVE::JSONSchema::pve_verify_ip($host, 1);
 112         next if !defined($ip);
 113
 114         push @{$ips}, $ip;
 115     }
 116
 117     return $ips;
 118 };
 119
 120 my $assert_mon_prerequisites = sub {
 121     my ($cfg, $monhash, $monid, $monips) = @_;
 122
 123     my $used_ips = {};
 124
 125     my $mon_host_ips = $ips_from_mon_host->($cfg->{global}->{mon_host});
 126
 127     for my $mon_host_ip (@{$mon_host_ips}) {
 128         my $ip = PVE::Network::canonical_ip($mon_host_ip);
 129         $used_ips->{$ip} = 1;
 130     }
 131
 132     for my $mon (values %{$monhash}) {
 133         next if !defined($mon->{addr});
 134
 135         for my $ip ($ips_from_mon_host->($mon->{addr})->@*) {
 136             $ip = PVE::Network::canonical_ip($ip);
 137             $used_ips->{$ip} = 1;
 138         }
 139     }
 140
 141     for my $monip (@{$monips}) {
 142         $monip = PVE::Network::canonical_ip($monip);
 143         die "monitor address '$monip' already in use\n" if $used_ips->{$monip};
 144     }
 145
 146     if (defined($monhash->{$monid})) {
 147         die "monitor '$monid' already exists\n";
 148     }
 149 };
 150
 151 my $assert_mon_can_remove = sub {
 152     my ($monhash, $monlist, $monid, $mondir) = @_;
 153
 154     if (!(defined($monhash->{"mon.$monid"}) ||
 155           grep { $_->{name} && $_->{name} eq $monid } @$monlist))
 156     {
 157         die "no such monitor id '$monid'\n"
 158     }
 159
 160     die "monitor filesystem '$mondir' does not exist on this node\n" if ! -d $mondir;
 161     die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
 162 };
 163
 164 my $remove_addr_from_mon_host = sub {
 165     my ($monhost, $addr) = @_;
 166
 167     $addr = "[$addr]" if PVE::JSONSchema::pve_verify_ipv6($addr, 1);
 168
 169     # various replaces to remove the ip
 170     # we always match the beginning or a separator (also at the end)
 171     # so we do not accidentally remove a wrong ip
 172     # e.g. removing 10.0.0.1 should not remove 10.0.0.101 or 110.0.0.1
 173
 174     # remove vector containing this ip
 175     # format is [vX:ip:port/nonce,vY:ip:port/nonce]
 176     my $vectorpart_re = "v\\d+:\Q$addr\E:\\d+\\/\\d+";
 177     $monhost =~ s/(^|[ ,;]*)\[$vectorpart_re(?:,$vectorpart_re)*\](?:[ ,;]+|$)/$1/;
 178
 179     # ip (+ port)
 180     $monhost =~ s/(^|[ ,;]+)\Q$addr\E(?::\d+)?(?:[ ,;]+|$)/$1/;
 181
 182     # ipv6 only without brackets
 183     if ($addr =~ m/^\[?(.*?:.*?)\]?$/) {
 184         $addr = $1;
 185         $monhost =~ s/(^|[ ,;]+)\Q$addr\E(?:[ ,;]+|$)/$1/;
 186     }
 187
 188     # remove trailing separators
 189     $monhost =~ s/[ ,;]+$//;
 190
 191     return $monhost;
 192 };
 193
 194 __PACKAGE__->register_method ({
 195     name => 'listmon',
 196     path => '',
 197     method => 'GET',
 198     description => "Get Ceph monitor list.",
 199     proxyto => 'node',
 200     protected => 1,
 201     permissions => {
 202         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 203     },
 204     parameters => {
 205         additionalProperties => 0,
 206         properties => {
 207             node => get_standard_option('pve-node'),
 208         },
 209     },
 210     returns => {
 211         type => 'array',
 212         items => {
 213             type => "object",
 214             properties => {
 215                 addr => { type => 'string', optional => 1 },
 216                 ceph_version => { type => 'string', optional => 1 },
 217                 ceph_version_short => { type => 'string', optional => 1 },
 218                 direxists => { type => 'string', optional => 1 },
 219                 host => { type => 'boolean', optional => 1 },
 220                 name => { type => 'string' },
 221                 quorum => { type => 'boolean', optional => 1 },
 222                 rank => { type => 'integer', optional => 1 },
 223                 service => { type => 'integer', optional => 1 },
 224                 state => { type => 'string', optional => 1 },
 225             },
 226         },
 227         links => [ { rel => 'child', href => "{name}" } ],
 228     },
 229     code => sub {
 230         my ($param) = @_;
 231
 232         PVE::Ceph::Tools::check_ceph_inited();
 233
 234         my $res = [];
 235
 236         my $cfg = cfs_read_file('ceph.conf');
 237
 238         my $rados = eval { PVE::RADOS->new() };
 239         warn $@ if $@;
 240         my $monhash = PVE::Ceph::Services::get_services_info("mon", $cfg, $rados);
 241
 242         if ($rados) {
 243             my $monstat = $rados->mon_command({ prefix => 'quorum_status' });
 244
 245             my $mons = $monstat->{monmap}->{mons};
 246             foreach my $d (@$mons) {
 247                 next if !defined($d->{name});
 248                 my $name = $d->{name};
 249                 $monhash->{$name}->{rank} = $d->{rank};
 250                 $monhash->{$name}->{addr} = $d->{addr};
 251                 if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
 252                     $monhash->{$name}->{quorum} = 1;
 253                     $monhash->{$name}->{state} = 'running';
 254                 }
 255             }
 256
 257         } else {
 258             # we cannot check the status if we do not have a RADOS
 259             # object, so set the state to unknown
 260             foreach my $monid (sort keys %$monhash) {
 261                 $monhash->{$monid}->{state} = 'unknown';
 262             }
 263         }
 264
 265         return PVE::RESTHandler::hash_to_array($monhash, 'name');
 266     }});
 267
 268 __PACKAGE__->register_method ({
 269     name => 'createmon',
 270     path => '{monid}',
 271     method => 'POST',
 272     description => "Create Ceph Monitor and Manager",
 273     proxyto => 'node',
 274     protected => 1,
 275     permissions => {
 276         check => ['perm', '/', [ 'Sys.Modify' ]],
 277     },
 278     parameters => {
 279         additionalProperties => 0,
 280         properties => {
 281             node => get_standard_option('pve-node'),
 282             monid => {
 283                 type => 'string',
 284                 optional => 1,
 285                 pattern => PVE::Ceph::Services::SERVICE_REGEX,
 286                 maxLength => 200,
 287                 description => "The ID for the monitor, when omitted the same as the nodename",
 288             },
 289             'mon-address' => {
 290                 description => 'Overwrites autodetected monitor IP address(es). ' .
 291                                'Must be in the public network(s) of Ceph.',
 292                 type => 'string', format => 'ip-list',
 293                 optional => 1,
 294             },
 295         },
 296     },
 297     returns => { type => 'string' },
 298     code => sub {
 299         my ($param) = @_;
 300
 301         PVE::Ceph::Tools::check_ceph_installed('ceph_mon');
 302         PVE::Ceph::Tools::check_ceph_inited();
 303         PVE::Ceph::Tools::setup_pve_symlinks();
 304
 305         my $rpcenv = PVE::RPCEnvironment::get();
 306         my $authuser = $rpcenv->get_user();
 307
 308         my $cfg = cfs_read_file('ceph.conf');
 309         my $rados = eval { PVE::RADOS->new() }; # try a rados connection, fails for first monitor
 310         my $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
 311
 312         my $is_first_monitor = !(scalar(keys %$monhash) || $cfg->{global}->{mon_host});
 313
 314         if (!defined($rados) && !$is_first_monitor) {
 315             die "Could not connect to ceph cluster despite configured monitors\n";
 316         }
 317
 318         my $monid = $param->{monid} // $param->{node};
 319         my $monsection = "mon.$monid";
 320         my $ips = $find_mon_ips->($cfg, $rados, $param->{node}, $param->{'mon-address'});
 321
 322         $assert_mon_prerequisites->($cfg, $monhash, $monid, $ips);
 323
 324         my $worker = sub  {
 325             my $upid = shift;
 326
 327             PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
 328                 # update cfg content and reassert prereqs inside the lock
 329                 $cfg = cfs_read_file('ceph.conf');
 330                 # reopen with longer timeout
 331                 if (defined($rados)) {
 332                     $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
 333                 }
 334                 $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
 335                 $assert_mon_prerequisites->($cfg, $monhash, $monid, $ips);
 336
 337                 my $client_keyring = PVE::Ceph::Tools::get_or_create_admin_keyring();
 338                 my $mon_keyring = PVE::Ceph::Tools::get_config('pve_mon_key_path');
 339
 340                 if (! -f $mon_keyring) {
 341                     print "creating new monitor keyring\n";
 342                     run_command([
 343                         'ceph-authtool',
 344                         '--create-keyring',
 345                         $mon_keyring,
 346                         '--gen-key',
 347                         '-n',
 348                         'mon.',
 349                         '--cap',
 350                         'mon',
 351                         'allow *',
 352                     ]);
 353                     run_command([
 354                         'ceph-authtool',
 355                         $mon_keyring,
 356                         '--import-keyring',
 357                         $client_keyring,
 358                     ]);
 359                 }
 360
 361                 my $ccname = PVE::Ceph::Tools::get_config('ccname');
 362                 my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 363                 -d $mondir && die "monitor filesystem '$mondir' already exist\n";
 364
 365                 my $monmap = "/tmp/monmap";
 366
 367                 eval {
 368                     mkdir $mondir;
 369
 370                     run_command(['chown', 'ceph:ceph', $mondir]);
 371
 372                     my $is_first_address = !defined($rados);
 373
 374                     my $monaddrs = [];
 375
 376                     for my $ip (@{$ips}) {
 377                         if (Net::IP::ip_is_ipv6($ip)) {
 378                             $cfg->{global}->{ms_bind_ipv6} = 'true';
 379                             $cfg->{global}->{ms_bind_ipv4} = 'false' if $is_first_address;
 380                         } else {
 381                             $cfg->{global}->{ms_bind_ipv4} = 'true';
 382                             $cfg->{global}->{ms_bind_ipv6} = 'false' if $is_first_address;
 383                         }
 384
 385                         my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]" : $ip;
 386                         push @{$monaddrs}, "v2:$monaddr:3300";
 387                         push @{$monaddrs}, "v1:$monaddr:6789";
 388
 389                         $is_first_address = 0;
 390                     }
 391
 392                     my $monmaptool_cmd = [
 393                         'monmaptool',
 394                         '--clobber',
 395                         '--addv',
 396                         $monid,
 397                         "[" . join(',', @{$monaddrs}) . "]",
 398                         '--print',
 399                         $monmap,
 400                     ];
 401
 402                     if (defined($rados)) { # we can only have a RADOS object if we have a monitor
 403                         my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
 404                         file_set_contents($monmap, $mapdata);
 405                         run_command($monmaptool_cmd);
 406                     } else { # we need to create a monmap for the first monitor
 407                         push @{$monmaptool_cmd}, '--create';
 408                         run_command($monmaptool_cmd);
 409                     }
 410
 411                     run_command([
 412                         'ceph-mon',
 413                         '--mkfs',
 414                         '-i',
 415                         $monid,
 416                         '--monmap',
 417                         $monmap,
 418                         '--keyring',
 419                         $mon_keyring,
 420                     ]);
 421                     run_command(['chown', 'ceph:ceph', '-R', $mondir]);
 422                 };
 423                 my $err = $@;
 424                 unlink $monmap;
 425                 if ($err) {
 426                     File::Path::remove_tree($mondir);
 427                     die $err;
 428                 }
 429
 430                 # update ceph.conf
 431                 my $monhost = $cfg->{global}->{mon_host} // "";
 432                 # add all known monitor ips to mon_host if it does not exist
 433                 if (!defined($cfg->{global}->{mon_host})) {
 434                     for my $mon (sort keys %$monhash) {
 435                         $monhost .= " " . $monhash->{$mon}->{addr};
 436                     }
 437                 }
 438                 $monhost .= " " . join(' ', @{$ips});
 439                 $cfg->{global}->{mon_host} = $monhost;
 440                 # The IP is needed in the ceph.conf for the first boot
 441                 $cfg->{$monsection}->{public_addr} = $ips->[0];
 442
 443                 cfs_write_file('ceph.conf', $cfg);
 444
 445                 PVE::Ceph::Services::ceph_service_cmd('start', $monsection);
 446
 447                 if ($is_first_monitor) {
 448                     print "created the first monitor, assume it's safe to disable insecure global"
 449                         ." ID reclaim for new setup\n";
 450                     eval {
 451                         run_command(
 452                             ['ceph', 'config', 'set', 'mon', 'auth_allow_insecure_global_id_reclaim', 'false'],
 453                             errfunc => sub { print STDERR "$_[0]\n" },
 454                         )
 455                     };
 456                     warn "$@" if $@;
 457                 }
 458
 459                 eval { PVE::Ceph::Services::ceph_service_cmd('enable', $monsection) };
 460                 warn "Enable ceph-mon\@${monid}.service failed, do manually: $@\n" if $@;
 461
 462                 PVE::Ceph::Services::broadcast_ceph_services();
 463             });
 464             die $@ if $@;
 465             # automatically create manager after the first monitor is created
 466             if ($is_first_monitor) {
 467                 PVE::API2::Ceph::MGR->createmgr({
 468                     node => $param->{node},
 469                     id => $param->{node}
 470                 })
 471             }
 472         };
 473
 474         return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
 475     }});
 476
 477 __PACKAGE__->register_method ({
 478     name => 'destroymon',
 479     path => '{monid}',
 480     method => 'DELETE',
 481     description => "Destroy Ceph Monitor and Manager.",
 482     proxyto => 'node',
 483     protected => 1,
 484     permissions => {
 485         check => ['perm', '/', [ 'Sys.Modify' ]],
 486     },
 487     parameters => {
 488         additionalProperties => 0,
 489         properties => {
 490             node => get_standard_option('pve-node'),
 491             monid => {
 492                 description => 'Monitor ID',
 493                 type => 'string',
 494                 pattern => PVE::Ceph::Services::SERVICE_REGEX,
 495             },
 496         },
 497     },
 498     returns => { type => 'string' },
 499     code => sub {
 500         my ($param) = @_;
 501
 502         my $rpcenv = PVE::RPCEnvironment::get();
 503
 504         my $authuser = $rpcenv->get_user();
 505
 506         PVE::Ceph::Tools::check_ceph_inited();
 507
 508         my $cfg = cfs_read_file('ceph.conf');
 509
 510         my $monid = $param->{monid};
 511         my $monsection = "mon.$monid";
 512
 513         my $rados = PVE::RADOS->new();
 514         my $monstat = $rados->mon_command({ prefix => 'quorum_status' });
 515         my $monlist = $monstat->{monmap}->{mons};
 516         my $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
 517
 518         my $ccname = PVE::Ceph::Tools::get_config('ccname');
 519         my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 520
 521         $assert_mon_can_remove->($monhash, $monlist, $monid, $mondir);
 522
 523         my $worker = sub {
 524             my $upid = shift;
 525             PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
 526                 # reload info and recheck
 527                 $cfg = cfs_read_file('ceph.conf');
 528
 529                 # reopen with longer timeout
 530                 $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
 531                 $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
 532                 $monstat = $rados->mon_command({ prefix => 'quorum_status' });
 533                 $monlist = $monstat->{monmap}->{mons};
 534
 535                 my $addrs = [];
 536
 537                 my $add_addr = sub {
 538                     my ($addr) = @_;
 539
 540                     # extract the ip without port and nonce (if present)
 541                     ($addr) = $addr =~ m|^(.*):\d+(/\d+)?$|;
 542                     ($addr) = $addr =~ m|^\[?(.*?)\]?$|; # remove brackets
 543                     push @{$addrs}, $addr;
 544                 };
 545
 546                 for my $mon (@$monlist) {
 547                     if ($mon->{name} eq $monid) {
 548                         if ($mon->{public_addrs} && $mon->{public_addrs}->{addrvec}) {
 549                             my $addrvec = $mon->{public_addrs}->{addrvec};
 550                             for my $addr (@{$addrvec}) {
 551                                 $add_addr->($addr->{addr});
 552                             }
 553                         } else {
 554                             $add_addr->($mon->{public_addr} // $mon->{addr});
 555                         }
 556                         last;
 557                     }
 558                 }
 559
 560                 $assert_mon_can_remove->($monhash, $monlist, $monid, $mondir);
 561
 562                 # this also stops the service
 563                 $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });
 564
 565                 # delete section
 566                 delete $cfg->{$monsection};
 567
 568                 # delete from mon_host
 569                 if (my $monhost = $cfg->{global}->{mon_host}) {
 570                     my $mon_host_ips = $ips_from_mon_host->($cfg->{global}->{mon_host});
 571
 572                     for my $addr (@{$addrs}) {
 573                         $monhost = $remove_addr_from_mon_host->($monhost, $addr);
 574
 575                         # also remove matching IPs that differ syntactically
 576                         if (PVE::JSONSchema::pve_verify_ip($addr, 1)) {
 577                             $addr = PVE::Network::canonical_ip($addr);
 578
 579                             for my $mon_host_ip (@{$mon_host_ips}) {
 580                                 # match canonical addresses, but remove as present in mon_host
 581                                 if (PVE::Network::canonical_ip($mon_host_ip) eq $addr) {
 582                                     $monhost = $remove_addr_from_mon_host->($monhost, $mon_host_ip);
 583                                 }
 584                             }
 585                         }
 586                     }
 587                     $cfg->{global}->{mon_host} = $monhost;
 588                 }
 589
 590                 cfs_write_file('ceph.conf', $cfg);
 591                 File::Path::remove_tree($mondir);
 592                 eval { PVE::Ceph::Services::ceph_service_cmd('disable', $monsection) };
 593                 warn $@ if $@;
 594                 PVE::Ceph::Services::broadcast_ceph_services();
 595             });
 596
 597             die $@ if $@;
 598         };
 599
 600         return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
 601     }});
 602
 603 1;