PVE/API2/Ceph/MON.pm

   1 package PVE::API2::Ceph::MON;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Net::IP;
   7 use File::Path;
   8
   9 use PVE::Ceph::Tools;
  10 use PVE::Ceph::Services;
  11 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  12 use PVE::JSONSchema qw(get_standard_option);
  13 use PVE::Network;
  14 use PVE::RADOS;
  15 use PVE::RESTHandler;
  16 use PVE::RPCEnvironment;
  17 use PVE::Tools qw(run_command file_set_contents);
  18 use PVE::CephConfig;
  19 use PVE::API2::Ceph::MGR;
  20
  21 use base qw(PVE::RESTHandler);
  22
  23 my $find_mon_ips = sub {
  24     my ($cfg, $rados, $node, $mon_address) = @_;
  25
  26     my $overwrite_ips = [ PVE::Tools::split_list($mon_address) ];
  27     $overwrite_ips = PVE::Network::unique_ips($overwrite_ips);
  28
  29     my $pubnet;
  30     if ($rados) {
  31         $pubnet = $rados->mon_command({ prefix => "config get" , who => "mon.",
  32                 key => "public_network", format => 'plain' });
  33         # if not defined in the db, the result is empty, it is also always
  34         # followed by a newline
  35         ($pubnet) = $pubnet =~ m/^(\S+)$/;
  36     }
  37     $pubnet //= $cfg->{global}->{public_network};
  38
  39     if (!$pubnet) {
  40         if (scalar(@{$overwrite_ips})) {
  41             return $overwrite_ips;
  42         } else {
  43            # don't refactor into '[ PVE::Cluster::remote... ]' as it uses wantarray
  44            my $ip = PVE::Cluster::remote_node_ip($node);
  45            return [ $ip ];
  46         }
  47     }
  48
  49     my $public_nets = [ PVE::Tools::split_list($pubnet) ];
  50     if (scalar(@{$public_nets}) > 1) {
  51         warn "Multiple Ceph public networks detected on $node: $pubnet\n";
  52         warn "Networks must be capable of routing to each other.\n";
  53     }
  54
  55     my $res = [];
  56
  57     if (!scalar(@{$overwrite_ips})) { # auto-select one address for each public network
  58         for my $net (@{$public_nets}) {
  59             $net = PVE::JSONSchema::pve_verify_cidr($net);
  60
  61             my $allowed_ips = PVE::Network::get_local_ip_from_cidr($net);
  62             $allowed_ips = PVE::Network::unique_ips($allowed_ips);
  63
  64             die "No active IP found for the requested ceph public network '$net' on node '$node'\n"
  65                 if scalar(@$allowed_ips) < 1;
  66
  67             if (scalar(@$allowed_ips) == 1) {
  68                 push @{$res}, $allowed_ips->[0];
  69             } else {
  70                 die "Multiple IPs for ceph public network '$net' detected on $node:\n".
  71                     join("\n", @$allowed_ips) ."\nuse 'mon-address' to specify one of them.\n";
  72             }
  73         }
  74     } else { # check if overwrite IPs are active and in any of the public networks
  75         my $allowed_list = [];
  76
  77         for my $net (@{$public_nets}) {
  78             $net = PVE::JSONSchema::pve_verify_cidr($net);
  79
  80             push @{$allowed_list}, @{PVE::Network::get_local_ip_from_cidr($net)};
  81         }
  82
  83         my $allowed_ips = PVE::Network::unique_ips($allowed_list);
  84
  85         for my $overwrite_ip (@{$overwrite_ips}) {
  86             die "Specified monitor IP '$overwrite_ip' not configured or up on $node!\n"
  87                 if !grep { $_ eq $overwrite_ip } @{$allowed_ips};
  88
  89             push @{$res}, $overwrite_ip;
  90         }
  91     }
  92
  93     return $res;
  94 };
  95
  96 my $ips_from_mon_host = sub {
  97     my ($mon_host) = @_;
  98
  99     my $ips = [];
 100
 101     my @hosts = PVE::Tools::split_list($mon_host);
 102
 103     for my $host (@hosts) {
 104         $host =~ s|^\[?v\d+\:||; # remove beginning of vector
 105         $host =~ s|/\d+\]?||; # remove end of vector
 106
 107         ($host) = PVE::Tools::parse_host_and_port($host);
 108         next if !defined($host);
 109
 110         # filter out hostnames
 111         my $ip = PVE::JSONSchema::pve_verify_ip($host, 1);
 112         next if !defined($ip);
 113
 114         push @{$ips}, $ip;
 115     }
 116
 117     return $ips;
 118 };
 119
 120 my $assert_mon_prerequisites = sub {
 121     my ($cfg, $monhash, $monid, $monips) = @_;
 122
 123     my $used_ips = {};
 124
 125     my $mon_host_ips = $ips_from_mon_host->($cfg->{global}->{mon_host});
 126
 127     for my $mon_host_ip (@{$mon_host_ips}) {
 128         my $ip = PVE::Network::canonical_ip($mon_host_ip);
 129         $used_ips->{$ip} = 1;
 130     }
 131
 132     for my $mon (values %{$monhash}) {
 133         next if !defined($mon->{addr});
 134
 135         for my $ip ($ips_from_mon_host->($mon->{addr})->@*) {
 136             $ip = PVE::Network::canonical_ip($ip);
 137             $used_ips->{$ip} = 1;
 138         }
 139     }
 140
 141     for my $monip (@{$monips}) {
 142         $monip = PVE::Network::canonical_ip($monip);
 143         die "monitor address '$monip' already in use\n" if $used_ips->{$monip};
 144     }
 145
 146     if (defined($monhash->{$monid})) {
 147         die "monitor '$monid' already exists\n";
 148     }
 149 };
 150
 151 my $assert_mon_can_remove = sub {
 152     my ($monhash, $monlist, $monid, $mondir) = @_;
 153
 154     if (!(defined($monhash->{"mon.$monid"}) ||
 155           grep { $_->{name} && $_->{name} eq $monid } @$monlist))
 156     {
 157         die "no such monitor id '$monid'\n"
 158     }
 159
 160     die "monitor filesystem '$mondir' does not exist on this node\n" if ! -d $mondir;
 161     die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
 162 };
 163
 164 my $remove_addr_from_mon_host = sub {
 165     my ($monhost, $addr) = @_;
 166
 167     $addr = "[$addr]" if PVE::JSONSchema::pve_verify_ipv6($addr, 1);
 168
 169     # various replaces to remove the ip
 170     # we always match the beginning or a separator (also at the end)
 171     # so we do not accidentally remove a wrong ip
 172     # e.g. removing 10.0.0.1 should not remove 10.0.0.101 or 110.0.0.1
 173
 174     # remove vector containing this ip
 175     # format is [vX:ip:port/nonce,vY:ip:port/nonce]
 176     my $vectorpart_re = "v\\d+:\Q$addr\E:\\d+\\/\\d+";
 177     $monhost =~ s/(^|[ ,;]*)\[$vectorpart_re(?:,$vectorpart_re)*\](?:[ ,;]+|$)/$1/;
 178
 179     # ip (+ port)
 180     $monhost =~ s/(^|[ ,;]+)\Q$addr\E(?::\d+)?(?:[ ,;]+|$)/$1/;
 181
 182     # ipv6 only without brackets
 183     if ($addr =~ m/^\[?(.*?:.*?)\]?$/) {
 184         $addr = $1;
 185         $monhost =~ s/(^|[ ,;]+)\Q$addr\E(?:[ ,;]+|$)/$1/;
 186     }
 187
 188     # remove trailing separators
 189     $monhost =~ s/[ ,;]+$//;
 190
 191     return $monhost;
 192 };
 193
 194 __PACKAGE__->register_method ({
 195     name => 'listmon',
 196     path => '',
 197     method => 'GET',
 198     description => "Get Ceph monitor list.",
 199     proxyto => 'node',
 200     protected => 1,
 201     permissions => {
 202         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 203     },
 204     parameters => {
 205         additionalProperties => 0,
 206         properties => {
 207             node => get_standard_option('pve-node'),
 208         },
 209     },
 210     returns => {
 211         type => 'array',
 212         items => {
 213             type => "object",
 214             properties => {
 215                 name => { type => 'string' },
 216                 addr => { type => 'string', optional => 1 },
 217                 host => { type => 'string', optional => 1 },
 218             },
 219         },
 220         links => [ { rel => 'child', href => "{name}" } ],
 221     },
 222     code => sub {
 223         my ($param) = @_;
 224
 225         PVE::Ceph::Tools::check_ceph_inited();
 226
 227         my $res = [];
 228
 229         my $cfg = cfs_read_file('ceph.conf');
 230
 231         my $rados = eval { PVE::RADOS->new() };
 232         warn $@ if $@;
 233         my $monhash = PVE::Ceph::Services::get_services_info("mon", $cfg, $rados);
 234
 235         if ($rados) {
 236             my $monstat = $rados->mon_command({ prefix => 'quorum_status' });
 237
 238             my $mons = $monstat->{monmap}->{mons};
 239             foreach my $d (@$mons) {
 240                 next if !defined($d->{name});
 241                 my $name = $d->{name};
 242                 $monhash->{$name}->{rank} = $d->{rank};
 243                 $monhash->{$name}->{addr} = $d->{addr};
 244                 if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
 245                     $monhash->{$name}->{quorum} = 1;
 246                     $monhash->{$name}->{state} = 'running';
 247                 }
 248             }
 249
 250         } else {
 251             # we cannot check the status if we do not have a RADOS
 252             # object, so set the state to unknown
 253             foreach my $monid (sort keys %$monhash) {
 254                 $monhash->{$monid}->{state} = 'unknown';
 255             }
 256         }
 257
 258         return PVE::RESTHandler::hash_to_array($monhash, 'name');
 259     }});
 260
 261 __PACKAGE__->register_method ({
 262     name => 'createmon',
 263     path => '{monid}',
 264     method => 'POST',
 265     description => "Create Ceph Monitor and Manager",
 266     proxyto => 'node',
 267     protected => 1,
 268     permissions => {
 269         check => ['perm', '/', [ 'Sys.Modify' ]],
 270     },
 271     parameters => {
 272         additionalProperties => 0,
 273         properties => {
 274             node => get_standard_option('pve-node'),
 275             monid => {
 276                 type => 'string',
 277                 optional => 1,
 278                 pattern => PVE::Ceph::Services::SERVICE_REGEX,
 279                 maxLength => 200,
 280                 description => "The ID for the monitor, when omitted the same as the nodename",
 281             },
 282             'mon-address' => {
 283                 description => 'Overwrites autodetected monitor IP address(es). ' .
 284                                'Must be in the public network(s) of Ceph.',
 285                 type => 'string', format => 'ip-list',
 286                 optional => 1,
 287             },
 288         },
 289     },
 290     returns => { type => 'string' },
 291     code => sub {
 292         my ($param) = @_;
 293
 294         PVE::Ceph::Tools::check_ceph_installed('ceph_mon');
 295         PVE::Ceph::Tools::check_ceph_inited();
 296         PVE::Ceph::Tools::setup_pve_symlinks();
 297
 298         my $rpcenv = PVE::RPCEnvironment::get();
 299         my $authuser = $rpcenv->get_user();
 300
 301         my $cfg = cfs_read_file('ceph.conf');
 302         my $rados = eval { PVE::RADOS->new() }; # try a rados connection, fails for first monitor
 303         my $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
 304
 305         my $is_first_monitor = !(scalar(keys %$monhash) || $cfg->{global}->{mon_host});
 306
 307         if (!defined($rados) && !$is_first_monitor) {
 308             die "Could not connect to ceph cluster despite configured monitors\n";
 309         }
 310
 311         my $monid = $param->{monid} // $param->{node};
 312         my $monsection = "mon.$monid";
 313         my $ips = $find_mon_ips->($cfg, $rados, $param->{node}, $param->{'mon-address'});
 314
 315         $assert_mon_prerequisites->($cfg, $monhash, $monid, $ips);
 316
 317         my $worker = sub  {
 318             my $upid = shift;
 319
 320             PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
 321                 # update cfg content and reassert prereqs inside the lock
 322                 $cfg = cfs_read_file('ceph.conf');
 323                 # reopen with longer timeout
 324                 if (defined($rados)) {
 325                     $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
 326                 }
 327                 $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
 328                 $assert_mon_prerequisites->($cfg, $monhash, $monid, $ips);
 329
 330                 my $client_keyring = PVE::Ceph::Tools::get_or_create_admin_keyring();
 331                 my $mon_keyring = PVE::Ceph::Tools::get_config('pve_mon_key_path');
 332
 333                 if (! -f $mon_keyring) {
 334                     print "creating new monitor keyring\n";
 335                     run_command([
 336                         'ceph-authtool',
 337                         '--create-keyring',
 338                         $mon_keyring,
 339                         '--gen-key',
 340                         '-n',
 341                         'mon.',
 342                         '--cap',
 343                         'mon',
 344                         'allow *',
 345                     ]);
 346                     run_command([
 347                         'ceph-authtool',
 348                         $mon_keyring,
 349                         '--import-keyring',
 350                         $client_keyring,
 351                     ]);
 352                 }
 353
 354                 my $ccname = PVE::Ceph::Tools::get_config('ccname');
 355                 my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 356                 -d $mondir && die "monitor filesystem '$mondir' already exist\n";
 357
 358                 my $monmap = "/tmp/monmap";
 359
 360                 eval {
 361                     mkdir $mondir;
 362
 363                     run_command(['chown', 'ceph:ceph', $mondir]);
 364
 365                     my $is_first_address = !defined($rados);
 366
 367                     my $monaddrs = [];
 368
 369                     for my $ip (@{$ips}) {
 370                         if (Net::IP::ip_is_ipv6($ip)) {
 371                             $cfg->{global}->{ms_bind_ipv6} = 'true';
 372                             $cfg->{global}->{ms_bind_ipv4} = 'false' if $is_first_address;
 373                         } else {
 374                             $cfg->{global}->{ms_bind_ipv4} = 'true';
 375                             $cfg->{global}->{ms_bind_ipv6} = 'false' if $is_first_address;
 376                         }
 377
 378                         my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]" : $ip;
 379                         push @{$monaddrs}, "v2:$monaddr:3300";
 380                         push @{$monaddrs}, "v1:$monaddr:6789";
 381
 382                         $is_first_address = 0;
 383                     }
 384
 385                     my $monmaptool_cmd = [
 386                         'monmaptool',
 387                         '--clobber',
 388                         '--addv',
 389                         $monid,
 390                         "[" . join(',', @{$monaddrs}) . "]",
 391                         '--print',
 392                         $monmap,
 393                     ];
 394
 395                     if (defined($rados)) { # we can only have a RADOS object if we have a monitor
 396                         my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
 397                         file_set_contents($monmap, $mapdata);
 398                         run_command($monmaptool_cmd);
 399                     } else { # we need to create a monmap for the first monitor
 400                         push @{$monmaptool_cmd}, '--create';
 401                         run_command($monmaptool_cmd);
 402                     }
 403
 404                     run_command([
 405                         'ceph-mon',
 406                         '--mkfs',
 407                         '-i',
 408                         $monid,
 409                         '--monmap',
 410                         $monmap,
 411                         '--keyring',
 412                         $mon_keyring,
 413                     ]);
 414                     run_command(['chown', 'ceph:ceph', '-R', $mondir]);
 415                 };
 416                 my $err = $@;
 417                 unlink $monmap;
 418                 if ($err) {
 419                     File::Path::remove_tree($mondir);
 420                     die $err;
 421                 }
 422
 423                 # update ceph.conf
 424                 my $monhost = $cfg->{global}->{mon_host} // "";
 425                 # add all known monitor ips to mon_host if it does not exist
 426                 if (!defined($cfg->{global}->{mon_host})) {
 427                     for my $mon (sort keys %$monhash) {
 428                         $monhost .= " " . $monhash->{$mon}->{addr};
 429                     }
 430                 }
 431                 $monhost .= " " . join(' ', @{$ips});
 432                 $cfg->{global}->{mon_host} = $monhost;
 433                 # The IP is needed in the ceph.conf for the first boot
 434                 $cfg->{$monsection}->{public_addr} = $ips->[0];
 435
 436                 cfs_write_file('ceph.conf', $cfg);
 437
 438                 PVE::Ceph::Services::ceph_service_cmd('start', $monsection);
 439
 440                 if ($is_first_monitor) {
 441                     print "created the first monitor, assume it's safe to disable insecure global"
 442                         ." ID reclaim for new setup\n";
 443                     eval {
 444                         run_command(
 445                             ['ceph', 'config', 'set', 'mon', 'auth_allow_insecure_global_id_reclaim', 'false'],
 446                             errfunc => sub { print STDERR "$_[0]\n" },
 447                         )
 448                     };
 449                     warn "$@" if $@;
 450                 }
 451
 452                 eval { PVE::Ceph::Services::ceph_service_cmd('enable', $monsection) };
 453                 warn "Enable ceph-mon\@${monid}.service failed, do manually: $@\n" if $@;
 454
 455                 PVE::Ceph::Services::broadcast_ceph_services();
 456             });
 457             die $@ if $@;
 458             # automatically create manager after the first monitor is created
 459             if ($is_first_monitor) {
 460                 PVE::API2::Ceph::MGR->createmgr({
 461                     node => $param->{node},
 462                     id => $param->{node}
 463                 })
 464             }
 465         };
 466
 467         return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
 468     }});
 469
 470 __PACKAGE__->register_method ({
 471     name => 'destroymon',
 472     path => '{monid}',
 473     method => 'DELETE',
 474     description => "Destroy Ceph Monitor and Manager.",
 475     proxyto => 'node',
 476     protected => 1,
 477     permissions => {
 478         check => ['perm', '/', [ 'Sys.Modify' ]],
 479     },
 480     parameters => {
 481         additionalProperties => 0,
 482         properties => {
 483             node => get_standard_option('pve-node'),
 484             monid => {
 485                 description => 'Monitor ID',
 486                 type => 'string',
 487                 pattern => PVE::Ceph::Services::SERVICE_REGEX,
 488             },
 489         },
 490     },
 491     returns => { type => 'string' },
 492     code => sub {
 493         my ($param) = @_;
 494
 495         my $rpcenv = PVE::RPCEnvironment::get();
 496
 497         my $authuser = $rpcenv->get_user();
 498
 499         PVE::Ceph::Tools::check_ceph_inited();
 500
 501         my $cfg = cfs_read_file('ceph.conf');
 502
 503         my $monid = $param->{monid};
 504         my $monsection = "mon.$monid";
 505
 506         my $rados = PVE::RADOS->new();
 507         my $monstat = $rados->mon_command({ prefix => 'quorum_status' });
 508         my $monlist = $monstat->{monmap}->{mons};
 509         my $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
 510
 511         my $ccname = PVE::Ceph::Tools::get_config('ccname');
 512         my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 513
 514         $assert_mon_can_remove->($monhash, $monlist, $monid, $mondir);
 515
 516         my $worker = sub {
 517             my $upid = shift;
 518             PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
 519                 # reload info and recheck
 520                 $cfg = cfs_read_file('ceph.conf');
 521
 522                 # reopen with longer timeout
 523                 $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
 524                 $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
 525                 $monstat = $rados->mon_command({ prefix => 'quorum_status' });
 526                 $monlist = $monstat->{monmap}->{mons};
 527
 528                 my $addrs = [];
 529
 530                 my $add_addr = sub {
 531                     my ($addr) = @_;
 532
 533                     # extract the ip without port and nonce (if present)
 534                     ($addr) = $addr =~ m|^(.*):\d+(/\d+)?$|;
 535                     ($addr) = $addr =~ m|^\[?(.*?)\]?$|; # remove brackets
 536                     push @{$addrs}, $addr;
 537                 };
 538
 539                 for my $mon (@$monlist) {
 540                     if ($mon->{name} eq $monid) {
 541                         if ($mon->{public_addrs} && $mon->{public_addrs}->{addrvec}) {
 542                             my $addrvec = $mon->{public_addrs}->{addrvec};
 543                             for my $addr (@{$addrvec}) {
 544                                 $add_addr->($addr->{addr});
 545                             }
 546                         } else {
 547                             $add_addr->($mon->{public_addr} // $mon->{addr});
 548                         }
 549                         last;
 550                     }
 551                 }
 552
 553                 $assert_mon_can_remove->($monhash, $monlist, $monid, $mondir);
 554
 555                 # this also stops the service
 556                 $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });
 557
 558                 # delete section
 559                 delete $cfg->{$monsection};
 560
 561                 # delete from mon_host
 562                 if (my $monhost = $cfg->{global}->{mon_host}) {
 563                     my $mon_host_ips = $ips_from_mon_host->($cfg->{global}->{mon_host});
 564
 565                     for my $addr (@{$addrs}) {
 566                         $monhost = $remove_addr_from_mon_host->($monhost, $addr);
 567
 568                         # also remove matching IPs that differ syntactically
 569                         if (PVE::JSONSchema::pve_verify_ip($addr, 1)) {
 570                             $addr = PVE::Network::canonical_ip($addr);
 571
 572                             for my $mon_host_ip (@{$mon_host_ips}) {
 573                                 # match canonical addresses, but remove as present in mon_host
 574                                 if (PVE::Network::canonical_ip($mon_host_ip) eq $addr) {
 575                                     $monhost = $remove_addr_from_mon_host->($monhost, $mon_host_ip);
 576                                 }
 577                             }
 578                         }
 579                     }
 580                     $cfg->{global}->{mon_host} = $monhost;
 581                 }
 582
 583                 cfs_write_file('ceph.conf', $cfg);
 584                 File::Path::remove_tree($mondir);
 585                 eval { PVE::Ceph::Services::ceph_service_cmd('disable', $monsection) };
 586                 warn $@ if $@;
 587                 PVE::Ceph::Services::broadcast_ceph_services();
 588             });
 589
 590             die $@ if $@;
 591         };
 592
 593         return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
 594     }});
 595
 596 1;