PVE/API2/Ceph/MON.pm

   1 package PVE::API2::Ceph::MON;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Net::IP;
   7 use File::Path;
   8
   9 use PVE::Ceph::Tools;
  10 use PVE::Ceph::Services;
  11 use PVE::Cluster qw(cfs_read_file cfs_write_file);
  12 use PVE::JSONSchema qw(get_standard_option);
  13 use PVE::Network;
  14 use PVE::RADOS;
  15 use PVE::RESTHandler;
  16 use PVE::RPCEnvironment;
  17 use PVE::Tools qw(run_command file_set_contents);
  18 use PVE::CephConfig;
  19 use PVE::API2::Ceph::MGR;
  20
  21 use base qw(PVE::RESTHandler);
  22
  23 my $find_mon_ips = sub {
  24     my ($cfg, $rados, $node, $mon_address) = @_;
  25
  26     my $overwrite_ips = [ PVE::Tools::split_list($mon_address) ];
  27     $overwrite_ips = PVE::Network::unique_ips($overwrite_ips);
  28
  29     my $pubnet;
  30     if ($rados) {
  31         $pubnet = $rados->mon_command({ prefix => "config get" , who => "mon.",
  32                 key => "public_network", format => 'plain' });
  33         # if not defined in the db, the result is empty, it is also always
  34         # followed by a newline
  35         ($pubnet) = $pubnet =~ m/^(\S+)$/;
  36     }
  37     $pubnet //= $cfg->{global}->{public_network};
  38
  39     if (!$pubnet) {
  40         if (scalar(@{$overwrite_ips})) {
  41             return $overwrite_ips;
  42         } else {
  43            # don't refactor into '[ PVE::Cluster::remote... ]' as it uses wantarray
  44            my $ip = PVE::Cluster::remote_node_ip($node);
  45            return [ $ip ];
  46         }
  47     }
  48
  49     my $public_nets = [ PVE::Tools::split_list($pubnet) ];
  50     if (scalar(@{$public_nets}) > 1) {
  51         warn "Multiple Ceph public networks detected on $node: $pubnet\n";
  52         warn "Networks must be capable of routing to each other.\n";
  53     }
  54
  55     my $res = [];
  56
  57     if (!scalar(@{$overwrite_ips})) { # auto-select one address for each public network
  58         for my $net (@{$public_nets}) {
  59             my $allowed_ips = PVE::Network::get_local_ip_from_cidr($net);
  60             $allowed_ips = PVE::Network::unique_ips($allowed_ips);
  61
  62             die "No active IP found for the requested ceph public network '$net' on node '$node'\n"
  63                 if scalar(@$allowed_ips) < 1;
  64
  65             if (scalar(@$allowed_ips) == 1) {
  66                 push @{$res}, $allowed_ips->[0];
  67             } else {
  68                 die "Multiple IPs for ceph public network '$net' detected on $node:\n".
  69                     join("\n", @$allowed_ips) ."\nuse 'mon-address' to specify one of them.\n";
  70             }
  71         }
  72     } else { # check if overwrite IPs are active and in any of the public networks
  73         my $allowed_list = [];
  74
  75         for my $net (@{$public_nets}) {
  76             push @{$allowed_list}, @{PVE::Network::get_local_ip_from_cidr($net)};
  77         }
  78
  79         my $allowed_ips = PVE::Network::unique_ips($allowed_list);
  80
  81         for my $overwrite_ip (@{$overwrite_ips}) {
  82             die "Specified monitor IP '$overwrite_ip' not configured or up on $node!\n"
  83                 if !grep { $_ eq $overwrite_ip } @{$allowed_ips};
  84
  85             push @{$res}, $overwrite_ip;
  86         }
  87     }
  88
  89     return $res;
  90 };
  91
  92 my $ips_from_mon_host = sub {
  93     my ($mon_host) = @_;
  94
  95     my $ips = [];
  96
  97     my @hosts = PVE::Tools::split_list($mon_host);
  98
  99     for my $host (@hosts) {
 100         $host =~ s|^\[?v\d+\:||; # remove beginning of vector
 101         $host =~ s|/\d+\]?||; # remove end of vector
 102
 103         ($host) = PVE::Tools::parse_host_and_port($host);
 104         next if !defined($host);
 105
 106         # filter out hostnames
 107         my $ip = PVE::JSONSchema::pve_verify_ip($host, 1);
 108         next if !defined($ip);
 109
 110         push @{$ips}, $ip;
 111     }
 112
 113     return $ips;
 114 };
 115
 116 my $assert_mon_prerequisites = sub {
 117     my ($cfg, $monhash, $monid, $monips) = @_;
 118
 119     my $used_ips = {};
 120
 121     my $mon_host_ips = $ips_from_mon_host->($cfg->{global}->{mon_host});
 122
 123     for my $mon_host_ip (@{$mon_host_ips}) {
 124         my $ip = PVE::Network::canonical_ip($mon_host_ip);
 125         $used_ips->{$ip} = 1;
 126     }
 127
 128     for my $mon (values %{$monhash}) {
 129         next if !defined($mon->{addr});
 130
 131         for my $ip ($ips_from_mon_host->($mon->{addr})->@*) {
 132             $ip = PVE::Network::canonical_ip($ip);
 133             $used_ips->{$ip} = 1;
 134         }
 135     }
 136
 137     for my $monip (@{$monips}) {
 138         $monip = PVE::Network::canonical_ip($monip);
 139         die "monitor address '$monip' already in use\n" if $used_ips->{$monip};
 140     }
 141
 142     if (defined($monhash->{$monid})) {
 143         die "monitor '$monid' already exists\n";
 144     }
 145 };
 146
 147 my $assert_mon_can_remove = sub {
 148     my ($monhash, $monlist, $monid, $mondir) = @_;
 149
 150     if (!(defined($monhash->{"mon.$monid"}) ||
 151           grep { $_->{name} && $_->{name} eq $monid } @$monlist))
 152     {
 153         die "no such monitor id '$monid'\n"
 154     }
 155
 156     die "monitor filesystem '$mondir' does not exist on this node\n" if ! -d $mondir;
 157     die "can't remove last monitor\n" if scalar(@$monlist) <= 1;
 158 };
 159
 160 my $remove_addr_from_mon_host = sub {
 161     my ($monhost, $addr) = @_;
 162
 163     $addr = "[$addr]" if PVE::JSONSchema::pve_verify_ipv6($addr, 1);
 164
 165     # various replaces to remove the ip
 166     # we always match the beginning or a separator (also at the end)
 167     # so we do not accidentally remove a wrong ip
 168     # e.g. removing 10.0.0.1 should not remove 10.0.0.101 or 110.0.0.1
 169
 170     # remove vector containing this ip
 171     # format is [vX:ip:port/nonce,vY:ip:port/nonce]
 172     my $vectorpart_re = "v\\d+:\Q$addr\E:\\d+\\/\\d+";
 173     $monhost =~ s/(^|[ ,;]*)\[$vectorpart_re(?:,$vectorpart_re)*\](?:[ ,;]+|$)/$1/;
 174
 175     # ip (+ port)
 176     $monhost =~ s/(^|[ ,;]+)\Q$addr\E(?::\d+)?(?:[ ,;]+|$)/$1/;
 177
 178     # ipv6 only without brackets
 179     if ($addr =~ m/^\[?(.*?:.*?)\]?$/) {
 180         $addr = $1;
 181         $monhost =~ s/(^|[ ,;]+)\Q$addr\E(?:[ ,;]+|$)/$1/;
 182     }
 183
 184     # remove trailing separators
 185     $monhost =~ s/[ ,;]+$//;
 186
 187     return $monhost;
 188 };
 189
 190 __PACKAGE__->register_method ({
 191     name => 'listmon',
 192     path => '',
 193     method => 'GET',
 194     description => "Get Ceph monitor list.",
 195     proxyto => 'node',
 196     protected => 1,
 197     permissions => {
 198         check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
 199     },
 200     parameters => {
 201         additionalProperties => 0,
 202         properties => {
 203             node => get_standard_option('pve-node'),
 204         },
 205     },
 206     returns => {
 207         type => 'array',
 208         items => {
 209             type => "object",
 210             properties => {
 211                 addr => { type => 'string', optional => 1 },
 212                 ceph_version => { type => 'string', optional => 1 },
 213                 ceph_version_short => { type => 'string', optional => 1 },
 214                 direxists => { type => 'string', optional => 1 },
 215                 host => { type => 'boolean', optional => 1 },
 216                 name => { type => 'string' },
 217                 quorum => { type => 'boolean', optional => 1 },
 218                 rank => { type => 'integer', optional => 1 },
 219                 service => { type => 'integer', optional => 1 },
 220                 state => { type => 'string', optional => 1 },
 221             },
 222         },
 223         links => [ { rel => 'child', href => "{name}" } ],
 224     },
 225     code => sub {
 226         my ($param) = @_;
 227
 228         PVE::Ceph::Tools::check_ceph_inited();
 229
 230         my $res = [];
 231
 232         my $cfg = cfs_read_file('ceph.conf');
 233
 234         my $rados = eval { PVE::RADOS->new() };
 235         warn $@ if $@;
 236         my $monhash = PVE::Ceph::Services::get_services_info("mon", $cfg, $rados);
 237
 238         if ($rados) {
 239             my $monstat = $rados->mon_command({ prefix => 'quorum_status' });
 240
 241             my $mons = $monstat->{monmap}->{mons};
 242             foreach my $d (@$mons) {
 243                 next if !defined($d->{name});
 244                 my $name = $d->{name};
 245                 $monhash->{$name}->{rank} = $d->{rank};
 246                 $monhash->{$name}->{addr} = $d->{addr};
 247                 if (grep { $_ eq $d->{rank} } @{$monstat->{quorum}}) {
 248                     $monhash->{$name}->{quorum} = 1;
 249                     $monhash->{$name}->{state} = 'running';
 250                 }
 251             }
 252
 253         } else {
 254             # we cannot check the status if we do not have a RADOS
 255             # object, so set the state to unknown
 256             foreach my $monid (sort keys %$monhash) {
 257                 $monhash->{$monid}->{state} = 'unknown';
 258             }
 259         }
 260
 261         return PVE::RESTHandler::hash_to_array($monhash, 'name');
 262     }});
 263
 264 __PACKAGE__->register_method ({
 265     name => 'createmon',
 266     path => '{monid}',
 267     method => 'POST',
 268     description => "Create Ceph Monitor and Manager",
 269     proxyto => 'node',
 270     protected => 1,
 271     permissions => {
 272         check => ['perm', '/', [ 'Sys.Modify' ]],
 273     },
 274     parameters => {
 275         additionalProperties => 0,
 276         properties => {
 277             node => get_standard_option('pve-node'),
 278             monid => {
 279                 type => 'string',
 280                 optional => 1,
 281                 pattern => PVE::Ceph::Services::SERVICE_REGEX,
 282                 maxLength => 200,
 283                 description => "The ID for the monitor, when omitted the same as the nodename",
 284             },
 285             'mon-address' => {
 286                 description => 'Overwrites autodetected monitor IP address(es). ' .
 287                                'Must be in the public network(s) of Ceph.',
 288                 type => 'string', format => 'ip-list',
 289                 optional => 1,
 290             },
 291         },
 292     },
 293     returns => { type => 'string' },
 294     code => sub {
 295         my ($param) = @_;
 296
 297         PVE::Ceph::Tools::check_ceph_installed('ceph_mon');
 298         PVE::Ceph::Tools::check_ceph_inited();
 299         PVE::Ceph::Tools::setup_pve_symlinks();
 300
 301         my $rpcenv = PVE::RPCEnvironment::get();
 302         my $authuser = $rpcenv->get_user();
 303
 304         my $cfg = cfs_read_file('ceph.conf');
 305         my $rados = eval { PVE::RADOS->new() }; # try a rados connection, fails for first monitor
 306         my $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
 307
 308         my $is_first_monitor = !(scalar(keys %$monhash) || $cfg->{global}->{mon_host});
 309
 310         if (!defined($rados) && !$is_first_monitor) {
 311             die "Could not connect to ceph cluster despite configured monitors\n";
 312         }
 313
 314         my $monid = $param->{monid} // $param->{node};
 315         my $monsection = "mon.$monid";
 316         my $ips = $find_mon_ips->($cfg, $rados, $param->{node}, $param->{'mon-address'});
 317
 318         $assert_mon_prerequisites->($cfg, $monhash, $monid, $ips);
 319
 320         my $worker = sub  {
 321             my $upid = shift;
 322
 323             PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
 324                 # update cfg content and reassert prereqs inside the lock
 325                 $cfg = cfs_read_file('ceph.conf');
 326                 # reopen with longer timeout
 327                 if (defined($rados)) {
 328                     $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
 329                 }
 330                 $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
 331                 $assert_mon_prerequisites->($cfg, $monhash, $monid, $ips);
 332
 333                 my $client_keyring = PVE::Ceph::Tools::get_or_create_admin_keyring();
 334                 my $mon_keyring = PVE::Ceph::Tools::get_config('pve_mon_key_path');
 335
 336                 if (! -f $mon_keyring) {
 337                     print "creating new monitor keyring\n";
 338                     run_command([
 339                         'ceph-authtool',
 340                         '--create-keyring',
 341                         $mon_keyring,
 342                         '--gen-key',
 343                         '-n',
 344                         'mon.',
 345                         '--cap',
 346                         'mon',
 347                         'allow *',
 348                     ]);
 349                     run_command([
 350                         'ceph-authtool',
 351                         $mon_keyring,
 352                         '--import-keyring',
 353                         $client_keyring,
 354                     ]);
 355                 }
 356
 357                 my $ccname = PVE::Ceph::Tools::get_config('ccname');
 358                 my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 359                 -d $mondir && die "monitor filesystem '$mondir' already exist\n";
 360
 361                 my $monmap = "/tmp/monmap";
 362
 363                 eval {
 364                     mkdir $mondir;
 365
 366                     run_command(['chown', 'ceph:ceph', $mondir]);
 367
 368                     my $is_first_address = !defined($rados);
 369
 370                     my $monaddrs = [];
 371
 372                     for my $ip (@{$ips}) {
 373                         if (Net::IP::ip_is_ipv6($ip)) {
 374                             $cfg->{global}->{ms_bind_ipv6} = 'true';
 375                             $cfg->{global}->{ms_bind_ipv4} = 'false' if $is_first_address;
 376                         } else {
 377                             $cfg->{global}->{ms_bind_ipv4} = 'true';
 378                             $cfg->{global}->{ms_bind_ipv6} = 'false' if $is_first_address;
 379                         }
 380
 381                         my $monaddr = Net::IP::ip_is_ipv6($ip) ? "[$ip]" : $ip;
 382                         push @{$monaddrs}, "v2:$monaddr:3300";
 383                         push @{$monaddrs}, "v1:$monaddr:6789";
 384
 385                         $is_first_address = 0;
 386                     }
 387
 388                     my $monmaptool_cmd = [
 389                         'monmaptool',
 390                         '--clobber',
 391                         '--addv',
 392                         $monid,
 393                         "[" . join(',', @{$monaddrs}) . "]",
 394                         '--print',
 395                         $monmap,
 396                     ];
 397
 398                     if (defined($rados)) { # we can only have a RADOS object if we have a monitor
 399                         my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
 400                         file_set_contents($monmap, $mapdata);
 401                         run_command($monmaptool_cmd);
 402                     } else { # we need to create a monmap for the first monitor
 403                         push @{$monmaptool_cmd}, '--create';
 404                         run_command($monmaptool_cmd);
 405                     }
 406
 407                     run_command([
 408                         'ceph-mon',
 409                         '--mkfs',
 410                         '-i',
 411                         $monid,
 412                         '--monmap',
 413                         $monmap,
 414                         '--keyring',
 415                         $mon_keyring,
 416                     ]);
 417                     run_command(['chown', 'ceph:ceph', '-R', $mondir]);
 418                 };
 419                 my $err = $@;
 420                 unlink $monmap;
 421                 if ($err) {
 422                     File::Path::remove_tree($mondir);
 423                     die $err;
 424                 }
 425
 426                 # update ceph.conf
 427                 my $monhost = $cfg->{global}->{mon_host} // "";
 428                 # add all known monitor ips to mon_host if it does not exist
 429                 if (!defined($cfg->{global}->{mon_host})) {
 430                     for my $mon (sort keys %$monhash) {
 431                         $monhost .= " " . $monhash->{$mon}->{addr};
 432                     }
 433                 }
 434                 $monhost .= " " . join(' ', @{$ips});
 435                 $cfg->{global}->{mon_host} = $monhost;
 436                 # The IP is needed in the ceph.conf for the first boot
 437                 $cfg->{$monsection}->{public_addr} = $ips->[0];
 438
 439                 cfs_write_file('ceph.conf', $cfg);
 440
 441                 PVE::Ceph::Services::ceph_service_cmd('start', $monsection);
 442
 443                 if ($is_first_monitor) {
 444                     print "created the first monitor, assume it's safe to disable insecure global"
 445                         ." ID reclaim for new setup\n";
 446                     eval {
 447                         run_command(
 448                             ['ceph', 'config', 'set', 'mon', 'auth_allow_insecure_global_id_reclaim', 'false'],
 449                             errfunc => sub { print STDERR "$_[0]\n" },
 450                         )
 451                     };
 452                     warn "$@" if $@;
 453                 }
 454
 455                 eval { PVE::Ceph::Services::ceph_service_cmd('enable', $monsection) };
 456                 warn "Enable ceph-mon\@${monid}.service failed, do manually: $@\n" if $@;
 457
 458                 PVE::Ceph::Services::broadcast_ceph_services();
 459             });
 460             die $@ if $@;
 461             # automatically create manager after the first monitor is created
 462             if ($is_first_monitor) {
 463                 PVE::API2::Ceph::MGR->createmgr({
 464                     node => $param->{node},
 465                     id => $param->{node}
 466                 })
 467             }
 468         };
 469
 470         return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
 471     }});
 472
 473 __PACKAGE__->register_method ({
 474     name => 'destroymon',
 475     path => '{monid}',
 476     method => 'DELETE',
 477     description => "Destroy Ceph Monitor and Manager.",
 478     proxyto => 'node',
 479     protected => 1,
 480     permissions => {
 481         check => ['perm', '/', [ 'Sys.Modify' ]],
 482     },
 483     parameters => {
 484         additionalProperties => 0,
 485         properties => {
 486             node => get_standard_option('pve-node'),
 487             monid => {
 488                 description => 'Monitor ID',
 489                 type => 'string',
 490                 pattern => PVE::Ceph::Services::SERVICE_REGEX,
 491             },
 492         },
 493     },
 494     returns => { type => 'string' },
 495     code => sub {
 496         my ($param) = @_;
 497
 498         my $rpcenv = PVE::RPCEnvironment::get();
 499
 500         my $authuser = $rpcenv->get_user();
 501
 502         PVE::Ceph::Tools::check_ceph_inited();
 503
 504         my $cfg = cfs_read_file('ceph.conf');
 505
 506         my $monid = $param->{monid};
 507         my $monsection = "mon.$monid";
 508
 509         my $rados = PVE::RADOS->new();
 510         my $monstat = $rados->mon_command({ prefix => 'quorum_status' });
 511         my $monlist = $monstat->{monmap}->{mons};
 512         my $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
 513
 514         my $ccname = PVE::Ceph::Tools::get_config('ccname');
 515         my $mondir =  "/var/lib/ceph/mon/$ccname-$monid";
 516
 517         $assert_mon_can_remove->($monhash, $monlist, $monid, $mondir);
 518
 519         my $worker = sub {
 520             my $upid = shift;
 521             PVE::Cluster::cfs_lock_file('ceph.conf', undef, sub {
 522                 # reload info and recheck
 523                 $cfg = cfs_read_file('ceph.conf');
 524
 525                 # reopen with longer timeout
 526                 $rados = PVE::RADOS->new(timeout => PVE::Ceph::Tools::get_config('long_rados_timeout'));
 527                 $monhash = PVE::Ceph::Services::get_services_info('mon', $cfg, $rados);
 528                 $monstat = $rados->mon_command({ prefix => 'quorum_status' });
 529                 $monlist = $monstat->{monmap}->{mons};
 530
 531                 my $addrs = [];
 532
 533                 my $add_addr = sub {
 534                     my ($addr) = @_;
 535
 536                     # extract the ip without port and nonce (if present)
 537                     ($addr) = $addr =~ m|^(.*):\d+(/\d+)?$|;
 538                     ($addr) = $addr =~ m|^\[?(.*?)\]?$|; # remove brackets
 539                     push @{$addrs}, $addr;
 540                 };
 541
 542                 for my $mon (@$monlist) {
 543                     if ($mon->{name} eq $monid) {
 544                         if ($mon->{public_addrs} && $mon->{public_addrs}->{addrvec}) {
 545                             my $addrvec = $mon->{public_addrs}->{addrvec};
 546                             for my $addr (@{$addrvec}) {
 547                                 $add_addr->($addr->{addr});
 548                             }
 549                         } else {
 550                             $add_addr->($mon->{public_addr} // $mon->{addr});
 551                         }
 552                         last;
 553                     }
 554                 }
 555
 556                 $assert_mon_can_remove->($monhash, $monlist, $monid, $mondir);
 557
 558                 # this also stops the service
 559                 $rados->mon_command({ prefix => "mon remove", name => $monid, format => 'plain' });
 560
 561                 # delete section
 562                 delete $cfg->{$monsection};
 563
 564                 # delete from mon_host
 565                 if (my $monhost = $cfg->{global}->{mon_host}) {
 566                     my $mon_host_ips = $ips_from_mon_host->($cfg->{global}->{mon_host});
 567
 568                     for my $addr (@{$addrs}) {
 569                         $monhost = $remove_addr_from_mon_host->($monhost, $addr);
 570
 571                         # also remove matching IPs that differ syntactically
 572                         if (PVE::JSONSchema::pve_verify_ip($addr, 1)) {
 573                             $addr = PVE::Network::canonical_ip($addr);
 574
 575                             for my $mon_host_ip (@{$mon_host_ips}) {
 576                                 # match canonical addresses, but remove as present in mon_host
 577                                 if (PVE::Network::canonical_ip($mon_host_ip) eq $addr) {
 578                                     $monhost = $remove_addr_from_mon_host->($monhost, $mon_host_ip);
 579                                 }
 580                             }
 581                         }
 582                     }
 583                     $cfg->{global}->{mon_host} = $monhost;
 584                 }
 585
 586                 cfs_write_file('ceph.conf', $cfg);
 587                 File::Path::remove_tree($mondir);
 588                 eval { PVE::Ceph::Services::ceph_service_cmd('disable', $monsection) };
 589                 warn $@ if $@;
 590                 PVE::Ceph::Services::broadcast_ceph_services();
 591             });
 592
 593             die $@ if $@;
 594         };
 595
 596         return $rpcenv->fork_worker('cephdestroymon', $monsection,  $authuser, $worker);
 597     }});
 598
 599 1;