src/PVE/Cluster/Setup.pm

   1 package PVE::Cluster::Setup;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Digest::HMAC_SHA1;
   7 use Digest::SHA;
   8 use IO::File;
   9 use MIME::Base64;
  10 use Net::IP;
  11 use UUID;
  12 use POSIX qw(EEXIST);
  13
  14 use PVE::APIClient::LWP;
  15 use PVE::Cluster;
  16 use PVE::Corosync;
  17 use PVE::INotify;
  18 use PVE::JSONSchema;
  19 use PVE::Network;
  20 use PVE::Tools;
  21 use PVE::Certificate;
  22
  23 # relevant for joining or getting joined checks, after that versions in cluster can differ
  24 use constant JOIN_API_VERSION => 1;
  25 # (APIVER-this) is oldest version a new node in addnode can have and still be accepted
  26 use constant JOIN_API_AGE_AS_CLUSTER => 1;
  27 # (APIVER-this) is oldest version a cluster node can have to still try joining
  28 use constant JOIN_API_AGE_AS_JOINEE => 1;
  29
  30 sub assert_we_can_join_cluster_version {
  31     my ($version) = @_;
  32     my $min_version = JOIN_API_VERSION - JOIN_API_AGE_AS_JOINEE;
  33     return if $version >= $min_version;
  34     die "error: incompatible join API version on cluster ($version), local node"
  35         ." has ". JOIN_API_VERSION ." and supports >= $min_version. Make sure"
  36         ."all cluster nodes are up-to-date.\n";
  37 }
  38
  39 sub assert_node_can_join_our_version {
  40     my ($version) = @_;
  41     my $min_version = JOIN_API_VERSION - JOIN_API_AGE_AS_CLUSTER;
  42     return if $version >= $min_version;
  43     die "error: unsupported old API version on joining node ($version), cluster"
  44         ." node has ". JOIN_API_VERSION ." and supports >= $min_version. Please"
  45         ." upgrade node before joining\n";
  46 }
  47
  48 my $pmxcfs_base_dir = PVE::Cluster::base_dir();
  49 my $pmxcfs_auth_dir = PVE::Cluster::auth_dir();
  50
  51 # only write output if something fails
  52 sub run_silent_cmd {
  53     my ($cmd) = @_;
  54
  55     my $outbuf = '';
  56     my $record = sub { $outbuf .= shift . "\n"; };
  57
  58     eval { PVE::Tools::run_command($cmd, outfunc => $record, errfunc => $record) };
  59
  60     if (my $err = $@) {
  61         print STDERR $outbuf;
  62         die $err;
  63     }
  64 }
  65
  66 # Corosync related files
  67 my $localclusterdir = "/etc/corosync";
  68 my $localclusterconf = "$localclusterdir/corosync.conf";
  69 my $authfile = "$localclusterdir/authkey";
  70 my $clusterconf = "$pmxcfs_base_dir/corosync.conf";
  71
  72 # CA/certificate related files
  73 my $pveca_key_fn = "$pmxcfs_auth_dir/pve-root-ca.key";
  74 my $pveca_srl_fn = "$pmxcfs_auth_dir/pve-root-ca.srl";
  75 my $pveca_cert_fn = "$pmxcfs_base_dir/pve-root-ca.pem";
  76 # this is just a secret accessable by the web browser
  77 # and is used for CSRF prevention
  78 my $pvewww_key_fn = "$pmxcfs_base_dir/pve-www.key";
  79
  80 # ssh related files
  81 my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
  82 my $ssh_cluster_known_hosts = "$pmxcfs_auth_dir/known_hosts";
  83 my $ssh_cluster_authorized_keys = "$pmxcfs_auth_dir/authorized_keys";
  84 my $ssh_system_known_hosts = "/etc/ssh/ssh_known_hosts";
  85 my $ssh_system_server_config = "/etc/ssh/sshd_config";
  86
  87 my $ssh_root_rsa_key_private = "/root/.ssh/id_rsa";
  88 my $ssh_root_rsa_key_public = "/root/.ssh/id_rsa.pub";
  89 my $ssh_root_authorized_keys = "/root/.ssh/authorized_keys";
  90 my $ssh_root_authorized_keys_backup = "${ssh_root_authorized_keys}.org";
  91 my $ssh_root_client_config = "/root/.ssh/config";
  92
  93 # ssh related utility functions
  94
  95 sub ssh_merge_keys {
  96     # remove duplicate keys in $ssh_cluster_authorized_keys
  97     # ssh-copy-id simply add keys, so the file can grow to large
  98
  99     my $data = '';
 100     if (-f $ssh_cluster_authorized_keys) {
 101         $data = PVE::Tools::file_get_contents($ssh_cluster_authorized_keys);
 102         chomp($data);
 103     }
 104
 105     my $found_backup;
 106     if (-f $ssh_root_authorized_keys_backup) {
 107         $data .= "\n";
 108         $data .= PVE::Tools::file_get_contents($ssh_root_authorized_keys_backup);
 109         chomp($data);
 110         $found_backup = 1;
 111     }
 112
 113     # always add ourself
 114     if (-f $ssh_root_rsa_key_public) {
 115         my $pub = PVE::Tools::file_get_contents($ssh_root_rsa_key_public);
 116         chomp($pub);
 117         $data .= "\n$pub\n";
 118     }
 119
 120     my $newdata = "";
 121     my $vhash = {};
 122     my @lines = split(/\n/, $data);
 123     foreach my $line (@lines) {
 124         if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
 125             next if $vhash->{$3}++;
 126         }
 127         $newdata .= "$line\n";
 128     }
 129
 130         PVE::Tools::file_set_contents($ssh_cluster_authorized_keys, $newdata, 0600);
 131
 132         if ($found_backup && -l $ssh_root_authorized_keys) {
 133             # everything went well, so we can remove the backup
 134             unlink $ssh_root_authorized_keys_backup;
 135         }
 136     }
 137
 138 sub setup_sshd_config {
 139     my () = @_;
 140
 141     my $conf = PVE::Tools::file_get_contents($ssh_system_server_config);
 142
 143     return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
 144
 145     if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
 146         chomp $conf;
 147         $conf .= "\nPermitRootLogin yes\n";
 148     }
 149
 150     PVE::Tools::file_set_contents($ssh_system_server_config, $conf);
 151
 152     PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
 153 }
 154
 155 sub setup_rootsshconfig {
 156
 157     # create ssh key if it does not exist
 158     if (! -f $ssh_root_rsa_key_public) {
 159         mkdir '/root/.ssh/';
 160         system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_root_rsa_key_private}");
 161     }
 162
 163     # create ssh config if it does not exist
 164     if (! -f $ssh_root_client_config) {
 165         mkdir '/root/.ssh';
 166         if (my $fh = IO::File->new($ssh_root_client_config, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
 167             # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k  26 Jan 2017)
 168             # changed order to put AES before Chacha20 (most hardware has AESNI)
 169             print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
 170             close($fh);
 171         }
 172     }
 173 }
 174
 175 sub setup_ssh_keys {
 176
 177     mkdir $pmxcfs_auth_dir;
 178
 179     my $import_ok;
 180
 181     if (! -f $ssh_cluster_authorized_keys) {
 182         my $old;
 183         if (-f $ssh_root_authorized_keys) {
 184             $old = PVE::Tools::file_get_contents($ssh_root_authorized_keys);
 185         }
 186         if (my $fh = IO::File->new ($ssh_cluster_authorized_keys, O_CREAT|O_WRONLY|O_EXCL, 0400)) {
 187             PVE::Tools::safe_print($ssh_cluster_authorized_keys, $fh, $old) if $old;
 188             close($fh);
 189             $import_ok = 1;
 190         }
 191     }
 192
 193     warn "can't create shared ssh key database '$ssh_cluster_authorized_keys'\n"
 194         if ! -f $ssh_cluster_authorized_keys;
 195
 196     if (-f $ssh_root_authorized_keys && ! -l $ssh_root_authorized_keys) {
 197         if (!rename($ssh_root_authorized_keys , $ssh_root_authorized_keys_backup)) {
 198             warn "rename $ssh_root_authorized_keys failed - $!\n";
 199         }
 200     }
 201
 202     if (! -l $ssh_root_authorized_keys) {
 203         symlink $ssh_cluster_authorized_keys, $ssh_root_authorized_keys;
 204     }
 205
 206     if (! -l $ssh_root_authorized_keys) {
 207         warn "can't create symlink for ssh keys '$ssh_root_authorized_keys' -> '$ssh_cluster_authorized_keys'\n";
 208     } else {
 209         unlink $ssh_root_authorized_keys_backup if $import_ok;
 210     }
 211 }
 212
 213 sub ssh_unmerge_known_hosts {
 214     return if ! -l $ssh_system_known_hosts;
 215
 216     my $old = '';
 217     $old = PVE::Tools::file_get_contents($ssh_cluster_known_hosts)
 218         if -f $ssh_cluster_known_hosts;
 219
 220     PVE::Tools::file_set_contents($ssh_system_known_hosts, $old);
 221 }
 222
 223 sub ssh_merge_known_hosts {
 224     my ($nodename, $ip_address, $createLink) = @_;
 225
 226     die "no node name specified" if !$nodename;
 227     die "no ip address specified" if !$ip_address;
 228
 229     # ssh lowercases hostnames (aliases) before comparision, so we need too
 230     $nodename = lc($nodename);
 231     $ip_address = lc($ip_address);
 232
 233     mkdir $pmxcfs_auth_dir;
 234
 235     if (! -f $ssh_cluster_known_hosts) {
 236         if (my $fh = IO::File->new($ssh_cluster_known_hosts, O_CREAT|O_WRONLY|O_EXCL, 0600)) {
 237             close($fh);
 238         }
 239     }
 240
 241     my $old = PVE::Tools::file_get_contents($ssh_cluster_known_hosts);
 242
 243     my $new = '';
 244
 245     if ((! -l $ssh_system_known_hosts) && (-f $ssh_system_known_hosts)) {
 246         $new = PVE::Tools::file_get_contents($ssh_system_known_hosts);
 247     }
 248
 249     my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
 250     # Note: file sometimes containe emty lines at start, so we use multiline match
 251     die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
 252     $hostkey = $1;
 253
 254     my $data = '';
 255     my $vhash = {};
 256
 257     my $found_nodename;
 258     my $found_local_ip;
 259
 260     my $merge_line = sub {
 261         my ($line, $all) = @_;
 262
 263         return if $line =~ m/^\s*$/; # skip empty lines
 264         return if $line =~ m/^#/; # skip comments
 265
 266         if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
 267             my $key = $1;
 268             my $rsakey = $2;
 269             if (!$vhash->{$key}) {
 270                 $vhash->{$key} = 1;
 271                 if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
 272                     my $salt = decode_base64($1);
 273                     my $digest = $2;
 274                     my $hmac = Digest::HMAC_SHA1->new($salt);
 275                     $hmac->add($nodename);
 276                     my $hd = $hmac->b64digest . '=';
 277                     if ($digest eq $hd) {
 278                         if ($rsakey eq $hostkey) {
 279                             $found_nodename = 1;
 280                             $data .= $line;
 281                         }
 282                         return;
 283                     }
 284                     $hmac = Digest::HMAC_SHA1->new($salt);
 285                     $hmac->add($ip_address);
 286                     $hd = $hmac->b64digest . '=';
 287                     if ($digest eq $hd) {
 288                         if ($rsakey eq $hostkey) {
 289                             $found_local_ip = 1;
 290                             $data .= $line;
 291                         }
 292                         return;
 293                     }
 294                 } else {
 295                     $key = lc($key); # avoid duplicate entries, ssh compares lowercased
 296                     if ($key eq $ip_address) {
 297                         $found_local_ip = 1 if $rsakey eq $hostkey;
 298                     } elsif ($key eq $nodename) {
 299                         $found_nodename = 1 if $rsakey eq $hostkey;
 300                     }
 301                 }
 302                 $data .= $line;
 303             }
 304         } elsif ($all) {
 305             $data .= $line;
 306         }
 307     };
 308
 309     while ($old && $old =~ s/^((.*?)(\n|$))//) {
 310         my $line = "$2\n";
 311         &$merge_line($line, 1);
 312     }
 313
 314     while ($new && $new =~ s/^((.*?)(\n|$))//) {
 315         my $line = "$2\n";
 316         &$merge_line($line);
 317     }
 318
 319     # add our own key if not already there
 320     $data .= "$nodename $hostkey\n" if !$found_nodename;
 321     $data .= "$ip_address $hostkey\n" if !$found_local_ip;
 322
 323     PVE::Tools::file_set_contents($ssh_cluster_known_hosts, $data);
 324
 325     return if !$createLink;
 326
 327     unlink $ssh_system_known_hosts;
 328     symlink $ssh_cluster_known_hosts, $ssh_system_known_hosts;
 329
 330     warn "can't create symlink for ssh known hosts '$ssh_system_known_hosts' -> '$ssh_cluster_known_hosts'\n"
 331         if ! -l $ssh_system_known_hosts;
 332
 333 }
 334
 335 # directory and file creation
 336
 337 sub gen_local_dirs {
 338     my ($nodename) = @_;
 339
 340     PVE::Cluster::check_cfs_is_mounted();
 341
 342     my @required_dirs = (
 343         "$pmxcfs_base_dir/priv",
 344         "$pmxcfs_base_dir/nodes",
 345         "$pmxcfs_base_dir/nodes/$nodename",
 346         "$pmxcfs_base_dir/nodes/$nodename/lxc",
 347         "$pmxcfs_base_dir/nodes/$nodename/qemu-server",
 348         "$pmxcfs_base_dir/nodes/$nodename/openvz",
 349         "$pmxcfs_base_dir/nodes/$nodename/priv");
 350
 351     foreach my $dir (@required_dirs) {
 352         if (! -d $dir) {
 353             mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
 354         }
 355     }
 356 }
 357
 358 sub gen_auth_key {
 359     my $authprivkeyfn = "$pmxcfs_auth_dir/authkey.key";
 360     my $authpubkeyfn = "$pmxcfs_base_dir/authkey.pub";
 361
 362     return if -f "$authprivkeyfn";
 363
 364     PVE::Cluster::check_cfs_is_mounted();
 365
 366     PVE::Cluster::cfs_lock_authkey(undef, sub {
 367         mkdir $pmxcfs_auth_dir || $! == EEXIST || die "unable to create dir '$pmxcfs_auth_dir' - $!\n";
 368
 369         run_silent_cmd(['openssl', 'genrsa', '-out', $authprivkeyfn, '2048']);
 370
 371         run_silent_cmd(['openssl', 'rsa', '-in', $authprivkeyfn, '-pubout', '-out', $authpubkeyfn]);
 372     });
 373
 374     die "$@\n" if $@;
 375 }
 376
 377 sub gen_pveca_key {
 378
 379     return if -f $pveca_key_fn;
 380
 381     eval {
 382         run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']);
 383     };
 384
 385     die "unable to generate pve ca key:\n$@" if $@;
 386 }
 387
 388 sub gen_pveca_cert {
 389
 390     if (-f $pveca_key_fn && -f $pveca_cert_fn) {
 391         return 0;
 392     }
 393
 394     gen_pveca_key();
 395
 396     # we try to generate an unique 'subject' to avoid browser problems
 397     # (reused serial numbers, ..)
 398     my $uuid;
 399     UUID::generate($uuid);
 400     my $uuid_str;
 401     UUID::unparse($uuid, $uuid_str);
 402
 403     eval {
 404         # wrap openssl with faketime to prevent bug #904
 405         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'req', '-batch',
 406                         '-days', '3650', '-new', '-x509', '-nodes', '-key',
 407                         $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
 408                         "/CN=Proxmox Virtual Environment/OU=$uuid_str/O=PVE Cluster Manager CA/"]);
 409     };
 410
 411     die "generating pve root certificate failed:\n$@" if $@;
 412
 413     return 1;
 414 }
 415
 416 sub gen_pve_ssl_key {
 417     my ($nodename) = @_;
 418
 419     die "no node name specified" if !$nodename;
 420
 421     my $pvessl_key_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.key";
 422
 423     return if -f $pvessl_key_fn;
 424
 425     eval {
 426         run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
 427     };
 428
 429     die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
 430 }
 431
 432 sub gen_pve_www_key {
 433
 434     return if -f $pvewww_key_fn;
 435
 436     eval {
 437         run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
 438     };
 439
 440     die "unable to generate pve www key:\n$@" if $@;
 441 }
 442
 443 sub update_serial {
 444     my ($serial) = @_;
 445
 446     PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
 447 }
 448
 449 sub gen_pve_ssl_cert {
 450     my ($force, $nodename, $ip) = @_;
 451
 452     die "no node name specified" if !$nodename;
 453     die "no IP specified" if !$ip;
 454
 455     my $pvessl_cert_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.pem";
 456
 457     return if !$force && -f $pvessl_cert_fn;
 458
 459     my $names = "IP:127.0.0.1,IP:::1,DNS:localhost";
 460
 461     my $rc = PVE::INotify::read_file('resolvconf');
 462
 463     $names .= ",IP:$ip";
 464
 465     $names .= ",DNS:$nodename";
 466
 467     my $fqdn = $nodename;
 468     if ($rc && $rc->{search}) {
 469         $fqdn .= ".$rc->{search}";
 470         $names .= ",DNS:$fqdn";
 471     }
 472
 473     my $sslconf = <<__EOD;
 474 RANDFILE = /root/.rnd
 475 extensions = v3_req
 476
 477 [ req ]
 478 default_bits = 2048
 479 distinguished_name = req_distinguished_name
 480 req_extensions = v3_req
 481 prompt = no
 482 string_mask = nombstr
 483
 484 [ req_distinguished_name ]
 485 organizationalUnitName = PVE Cluster Node
 486 organizationName = Proxmox Virtual Environment
 487 commonName = $fqdn
 488
 489 [ v3_req ]
 490 basicConstraints = CA:FALSE
 491 extendedKeyUsage = serverAuth
 492 subjectAltName = $names
 493 __EOD
 494
 495     my $cfgfn = "/tmp/pvesslconf-$$.tmp";
 496     my $fh = IO::File->new ($cfgfn, "w");
 497     print $fh $sslconf;
 498     close ($fh);
 499
 500     my $reqfn = "/tmp/pvecertreq-$$.tmp";
 501     unlink $reqfn;
 502
 503     my $pvessl_key_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.key";
 504     eval {
 505         run_silent_cmd([
 506             'openssl', 'req', '-batch', '-new', '-config', $cfgfn, '-key', $pvessl_key_fn, '-out', $reqfn
 507         ]);
 508     };
 509
 510     if (my $err = $@) {
 511         unlink $reqfn;
 512         unlink $cfgfn;
 513         die "unable to generate pve certificate request:\n$err";
 514     }
 515
 516     update_serial("0000000000000000") if ! -f $pveca_srl_fn;
 517
 518     # get ca expiry
 519     my $cainfo = PVE::Certificate::get_certificate_info($pveca_cert_fn);
 520     my $daysleft = int(($cainfo->{notafter} - time())/(24*60*60));
 521
 522     if ($daysleft < 14) {
 523         die "CA expires in less than 2 weeks, unable to generate certificate.\n";
 524     }
 525
 526     # let the certificate expire a little sooner that the ca, so subtract 2 days
 527     $daysleft -= 2;
 528
 529     # we want the certificates to only last 2 years, since some browsers
 530     # do not accept certificates with very long expiry time
 531     if ($daysleft >= 2*365) {
 532         $daysleft = 2*365;
 533     }
 534
 535     eval {
 536         run_silent_cmd([
 537             'faketime', 'yesterday', # NOTE: wrap openssl with faketime to prevent bug #904
 538             'openssl', 'x509', '-req', '-in', $reqfn, '-days', $daysleft, '-out', $pvessl_cert_fn,
 539             '-CAkey', $pveca_key_fn, '-CA', $pveca_cert_fn, '-CAserial', $pveca_srl_fn, '-extfile', $cfgfn
 540         ]);
 541     };
 542
 543     if (my $err = $@) {
 544         unlink $reqfn;
 545         unlink $cfgfn;
 546         die "unable to generate pve ssl certificate:\n$err";
 547     }
 548
 549     unlink $cfgfn;
 550     unlink $reqfn;
 551 }
 552
 553 sub gen_pve_node_files {
 554     my ($nodename, $ip, $opt_force) = @_;
 555
 556     gen_local_dirs($nodename);
 557
 558     gen_auth_key();
 559
 560     # make sure we have a (cluster wide) secret
 561     # for CSRFR prevention
 562     gen_pve_www_key();
 563
 564     # make sure we have a (per node) private key
 565     gen_pve_ssl_key($nodename);
 566
 567     # make sure we have a CA
 568     my $force = gen_pveca_cert();
 569
 570     $force = 1 if $opt_force;
 571
 572     gen_pve_ssl_cert($force, $nodename, $ip);
 573 }
 574
 575 my $vzdump_cron_dummy = <<__EOD;
 576 # cluster wide vzdump cron schedule
 577 # Atomatically generated file - do not edit
 578
 579 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
 580
 581 __EOD
 582
 583 sub gen_pve_vzdump_symlink {
 584
 585     my $filename = "/etc/pve/vzdump.cron";
 586
 587     my $link_fn = "/etc/cron.d/vzdump";
 588
 589     if ((-f $filename) && (! -l $link_fn)) {
 590         rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
 591         symlink($filename, $link_fn);
 592     }
 593 }
 594
 595 sub gen_pve_vzdump_files {
 596
 597     my $filename = "/etc/pve/vzdump.cron";
 598
 599     PVE::Tools::file_set_contents($filename, $vzdump_cron_dummy)
 600         if ! -f $filename;
 601
 602     gen_pve_vzdump_symlink();
 603 };
 604
 605 # join helpers
 606
 607 sub assert_joinable {
 608     my ($local_addr, $links, $force) = @_;
 609
 610     my $errors = '';
 611     my $error = sub { $errors .= "* $_[0]\n"; };
 612
 613     if (-f $authfile) {
 614         $error->("authentication key '$authfile' already exists");
 615     }
 616
 617     if (-f $clusterconf)  {
 618         $error->("cluster config '$clusterconf' already exists");
 619     }
 620
 621     my $vmlist = PVE::Cluster::get_vmlist();
 622     if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
 623         $error->("this host already contains virtual guests");
 624     }
 625
 626     if (PVE::Tools::run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
 627         $error->("corosync is already running, is this node already in a cluster?!");
 628     }
 629
 630     # check if corosync ring IPs are configured on the current nodes interfaces
 631     my $check_ip = sub {
 632         my $ip = shift // return;
 633         my $logid = shift;
 634         if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
 635             my $host = $ip;
 636             eval { $ip = PVE::Network::get_ip_from_hostname($host); };
 637             if ($@) {
 638                 $error->("$logid: cannot use '$host': $@\n") ;
 639                 return;
 640             }
 641         }
 642
 643         my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
 644         my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
 645
 646         $error->("$logid: cannot use IP '$ip', not found on local node!\n")
 647             if scalar(@$configured_ips) < 1;
 648     };
 649
 650     $check_ip->($local_addr, 'local node address');
 651
 652     foreach my $link (keys %$links) {
 653         $check_ip->($links->{$link}->{address}, "link$link");
 654     }
 655
 656     if ($errors) {
 657         warn "detected the following error(s):\n$errors";
 658         die "Check if node may join a cluster failed!\n" if !$force;
 659         warn "\nWARNING : detected error but forced to continue!\n\n";
 660     }
 661 }
 662
 663 sub join {
 664     my ($param) = @_;
 665
 666     my $nodename = PVE::INotify::nodename();
 667     my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 668
 669     my $links = PVE::Corosync::extract_corosync_link_args($param);
 670
 671     # check if we can join with the given parameters and current node state
 672     assert_joinable($local_ip_address, $links, $param->{force});
 673
 674     setup_sshd_config();
 675     setup_rootsshconfig();
 676     setup_ssh_keys();
 677
 678     # make sure known_hosts is on local filesystem
 679     ssh_unmerge_known_hosts();
 680
 681     my $host = $param->{hostname};
 682     my $conn_args = {
 683         username => 'root@pam',
 684         password => $param->{password},
 685         cookie_name => 'PVEAuthCookie',
 686         protocol => 'https',
 687         host => $host,
 688         port => 8006,
 689     };
 690
 691     if (my $fp = $param->{fingerprint}) {
 692         $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
 693     } else {
 694         # API schema ensures that we can only get here from CLI handler
 695         $conn_args->{manual_verification} = 1;
 696     }
 697
 698     print "Establishing API connection with host '$host'\n";
 699
 700     my $conn = PVE::APIClient::LWP->new(%$conn_args);
 701     $conn->login();
 702
 703     # login raises an exception on failure, so if we get here we're good
 704     print "Login succeeded.\n";
 705
 706     print "check cluster join API version\n";
 707     my $apiver = eval { $conn->get("/cluster/config/apiversion") } // 0;
 708     assert_we_can_join_cluster_version($apiver);
 709
 710     my $args = {};
 711     $args->{force} = $param->{force} if defined($param->{force});
 712     $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
 713     $args->{votes} = $param->{votes} if defined($param->{votes});
 714     foreach my $link (keys %$links) {
 715         $args->{"link$link"} = PVE::Corosync::print_corosync_link($links->{$link});
 716     }
 717
 718     # this will be used as fallback if no links are specified
 719     if (!%$links) {
 720         $args->{link0} = $local_ip_address if $apiver == 0;
 721         $args->{new_node_ip} = $local_ip_address if $apiver >= 1;
 722
 723         print "No cluster network links passed explicitly, fallback to local node"
 724             . " IP '$local_ip_address'\n";
 725     }
 726
 727     if ($apiver >= 1) {
 728         $args->{apiversion} = JOIN_API_VERSION;
 729     }
 730
 731     print "Request addition of this node\n";
 732     my $res = eval { $conn->post("/cluster/config/nodes/$nodename", $args); };
 733     if (my $err = $@) {
 734         if (ref($err) && $err->isa('PVE::APIClient::Exception')) {
 735             # we received additional info about the error, show the user
 736             chomp $err->{msg};
 737             warn "An error occurred on the cluster node: $err->{msg}\n";
 738             foreach my $key (sort keys %{$err->{errors}}) {
 739                 my $symbol = ($key =~ m/^warning/) ? '*' : '!';
 740                 warn "$symbol $err->{errors}->{$key}\n";
 741             }
 742
 743             die "Cluster join aborted!\n";
 744         }
 745
 746         die $@;
 747     }
 748
 749     if (defined($res->{warnings})) {
 750         foreach my $warn (@{$res->{warnings}}) {
 751             warn "cluster: $warn\n";
 752         }
 753     }
 754
 755     print "Join request OK, finishing setup locally\n";
 756
 757     # added successfuly - now prepare local node
 758     finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
 759 }
 760
 761 sub finish_join {
 762     my ($nodename, $corosync_conf, $corosync_authkey) = @_;
 763
 764     mkdir "$localclusterdir";
 765     PVE::Tools::file_set_contents($authfile, $corosync_authkey);
 766     PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
 767
 768     print "stopping pve-cluster service\n";
 769     my $cmd = ['systemctl', 'stop', 'pve-cluster'];
 770     PVE::Tools::run_command($cmd, errmsg => "can't stop pve-cluster service");
 771
 772     my $dbfile = PVE::Cluster::cfs_backup_database();
 773     unlink $dbfile;
 774
 775     $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
 776     PVE::Tools::run_command($cmd, errmsg => "starting pve-cluster failed");
 777
 778     # wait for quorum
 779     my $printqmsg = 1;
 780     while (!PVE::Cluster::check_cfs_quorum(1)) {
 781         if ($printqmsg) {
 782             print "waiting for quorum...";
 783             STDOUT->flush();
 784             $printqmsg = 0;
 785         }
 786         sleep(1);
 787     }
 788     print "OK\n" if !$printqmsg;
 789
 790     generate_local_files();
 791     updatecerts_and_ssh(1);
 792
 793     print "generated new node certificate, restart pveproxy and pvedaemon services\n";
 794     PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
 795
 796     print "successfully added node '$nodename' to cluster.\n";
 797 }
 798
 799 sub generate_local_files {
 800     setup_rootsshconfig();
 801     gen_pve_vzdump_symlink();
 802 }
 803
 804 sub updatecerts_and_ssh {
 805     my ($force_new_cert, $silent) = @_;
 806
 807     my $p = sub { print "$_[0]\n" if !$silent };
 808
 809     if (!PVE::Cluster::check_cfs_quorum(1)) {
 810         return undef if $silent;
 811         die "no quorum - unable to update files\n";
 812     }
 813
 814     setup_ssh_keys();
 815
 816     my $nodename = PVE::INotify::nodename();
 817     my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 818
 819     $p->("(re)generate node files");
 820     $p->("generate new node certificate") if $force_new_cert;
 821     gen_pve_node_files($nodename, $local_ip_address, $force_new_cert);
 822
 823     $p->("merge authorized SSH keys and known hosts");
 824     ssh_merge_keys();
 825     ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 826     gen_pve_vzdump_files();
 827 }
 828
 829 1;