data/PVE/Cluster/Setup.pm

   1 package PVE::Cluster::Setup;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Digest::HMAC_SHA1;
   7 use Digest::SHA;
   8 use IO::File;
   9 use MIME::Base64;
  10 use Net::IP;
  11 use UUID;
  12 use POSIX qw(EEXIST);
  13
  14 use PVE::APIClient::LWP;
  15 use PVE::Cluster;
  16 use PVE::Corosync;
  17 use PVE::INotify;
  18 use PVE::JSONSchema;
  19 use PVE::Network;
  20 use PVE::Tools;
  21 use PVE::Certificate;
  22
  23 # relevant for joining or getting joined checks, after that versions in cluster can differ
  24 use constant JOIN_API_VERSION => 1;
  25 # (APIVER-this) is oldest version a new node in addnode can have and still be accepted
  26 use constant JOIN_API_AGE_AS_CLUSTER => 1;
  27 # (APIVER-this) is oldest version a cluster node can have to still try joining
  28 use constant JOIN_API_AGE_AS_JOINEE => 1;
  29
  30 sub assert_we_can_join_cluster_version {
  31     my ($version) = @_;
  32     my $min_version = JOIN_API_VERSION - JOIN_API_AGE_AS_JOINEE;
  33     return if $version >= $min_version;
  34     die "error: incompatible join API version on cluster ($version), local node"
  35         ." has ". JOIN_API_VERSION ." and supports >= $min_version. Make sure"
  36         ."all cluster nodes are up-to-date.\n";
  37 }
  38
  39 sub assert_node_can_join_our_version {
  40     my ($version) = @_;
  41     my $min_version = JOIN_API_VERSION - JOIN_API_AGE_AS_CLUSTER;
  42     return if $version >= $min_version;
  43     die "error: unsupported old API version on joining node ($version), cluster"
  44         ." node has ". JOIN_API_VERSION ." and supports >= $min_version. Please"
  45         ." upgrade node before joining\n";
  46 }
  47
  48 my $pmxcfs_base_dir = PVE::Cluster::base_dir();
  49 my $pmxcfs_auth_dir = PVE::Cluster::auth_dir();
  50
  51 # only write output if something fails
  52 sub run_silent_cmd {
  53     my ($cmd) = @_;
  54
  55     my $outbuf = '';
  56     my $record = sub { $outbuf .= shift . "\n"; };
  57
  58     eval { PVE::Tools::run_command($cmd, outfunc => $record, errfunc => $record) };
  59
  60     if (my $err = $@) {
  61         print STDERR $outbuf;
  62         die $err;
  63     }
  64 }
  65
  66 # Corosync related files
  67 my $localclusterdir = "/etc/corosync";
  68 my $localclusterconf = "$localclusterdir/corosync.conf";
  69 my $authfile = "$localclusterdir/authkey";
  70 my $clusterconf = "$pmxcfs_base_dir/corosync.conf";
  71
  72 # CA/certificate related files
  73 my $pveca_key_fn = "$pmxcfs_auth_dir/pve-root-ca.key";
  74 my $pveca_srl_fn = "$pmxcfs_auth_dir/pve-root-ca.srl";
  75 my $pveca_cert_fn = "$pmxcfs_base_dir/pve-root-ca.pem";
  76 # this is just a secret accessable by the web browser
  77 # and is used for CSRF prevention
  78 my $pvewww_key_fn = "$pmxcfs_base_dir/pve-www.key";
  79
  80 # ssh related files
  81 my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
  82 my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
  83 my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
  84 my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
  85 my $sshknownhosts = "$pmxcfs_auth_dir/known_hosts";
  86 my $sshauthkeys = "$pmxcfs_auth_dir/authorized_keys";
  87 my $sshd_config_fn = "/etc/ssh/sshd_config";
  88 my $rootsshauthkeys = "/root/.ssh/authorized_keys";
  89 my $rootsshauthkeysbackup = "${rootsshauthkeys}.org";
  90 my $rootsshconfig = "/root/.ssh/config";
  91
  92 # ssh related utility functions
  93
  94 sub ssh_merge_keys {
  95     # remove duplicate keys in $sshauthkeys
  96     # ssh-copy-id simply add keys, so the file can grow to large
  97
  98     my $data = '';
  99     if (-f $sshauthkeys) {
 100         $data = PVE::Tools::file_get_contents($sshauthkeys, 128*1024);
 101         chomp($data);
 102     }
 103
 104     my $found_backup;
 105     if (-f $rootsshauthkeysbackup) {
 106         $data .= "\n";
 107         $data .= PVE::Tools::file_get_contents($rootsshauthkeysbackup, 128*1024);
 108         chomp($data);
 109         $found_backup = 1;
 110     }
 111
 112     # always add ourself
 113     if (-f $ssh_rsa_id) {
 114         my $pub = PVE::Tools::file_get_contents($ssh_rsa_id);
 115         chomp($pub);
 116         $data .= "\n$pub\n";
 117     }
 118
 119     my $newdata = "";
 120     my $vhash = {};
 121     my @lines = split(/\n/, $data);
 122     foreach my $line (@lines) {
 123         if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
 124             next if $vhash->{$3}++;
 125         }
 126         $newdata .= "$line\n";
 127     }
 128
 129     PVE::Tools::file_set_contents($sshauthkeys, $newdata, 0600);
 130
 131     if ($found_backup && -l $rootsshauthkeys) {
 132         # everything went well, so we can remove the backup
 133         unlink $rootsshauthkeysbackup;
 134     }
 135 }
 136
 137 sub setup_sshd_config {
 138     my () = @_;
 139
 140     my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
 141
 142     return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
 143
 144     if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
 145         chomp $conf;
 146         $conf .= "\nPermitRootLogin yes\n";
 147     }
 148
 149     PVE::Tools::file_set_contents($sshd_config_fn, $conf);
 150
 151     PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
 152 }
 153
 154 sub setup_rootsshconfig {
 155
 156     # create ssh key if it does not exist
 157     if (! -f $ssh_rsa_id) {
 158         mkdir '/root/.ssh/';
 159         system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
 160     }
 161
 162     # create ssh config if it does not exist
 163     if (! -f $rootsshconfig) {
 164         mkdir '/root/.ssh';
 165         if (my $fh = IO::File->new($rootsshconfig, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
 166             # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k  26 Jan 2017)
 167             # changed order to put AES before Chacha20 (most hardware has AESNI)
 168             print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
 169             close($fh);
 170         }
 171     }
 172 }
 173
 174 sub setup_ssh_keys {
 175
 176     mkdir $pmxcfs_auth_dir;
 177
 178     my $import_ok;
 179
 180     if (! -f $sshauthkeys) {
 181         my $old;
 182         if (-f $rootsshauthkeys) {
 183             $old = PVE::Tools::file_get_contents($rootsshauthkeys, 128*1024);
 184         }
 185         if (my $fh = IO::File->new ($sshauthkeys, O_CREAT|O_WRONLY|O_EXCL, 0400)) {
 186             PVE::Tools::safe_print($sshauthkeys, $fh, $old) if $old;
 187             close($fh);
 188             $import_ok = 1;
 189         }
 190     }
 191
 192     warn "can't create shared ssh key database '$sshauthkeys'\n"
 193         if ! -f $sshauthkeys;
 194
 195     if (-f $rootsshauthkeys && ! -l $rootsshauthkeys) {
 196         if (!rename($rootsshauthkeys , $rootsshauthkeysbackup)) {
 197             warn "rename $rootsshauthkeys failed - $!\n";
 198         }
 199     }
 200
 201     if (! -l $rootsshauthkeys) {
 202         symlink $sshauthkeys, $rootsshauthkeys;
 203     }
 204
 205     if (! -l $rootsshauthkeys) {
 206         warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n";
 207     } else {
 208         unlink $rootsshauthkeysbackup if $import_ok;
 209     }
 210 }
 211
 212 sub ssh_unmerge_known_hosts {
 213     return if ! -l $sshglobalknownhosts;
 214
 215     my $old = '';
 216     $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024)
 217         if -f $sshknownhosts;
 218
 219     PVE::Tools::file_set_contents($sshglobalknownhosts, $old);
 220 }
 221
 222 sub ssh_merge_known_hosts {
 223     my ($nodename, $ip_address, $createLink) = @_;
 224
 225     die "no node name specified" if !$nodename;
 226     die "no ip address specified" if !$ip_address;
 227
 228     # ssh lowercases hostnames (aliases) before comparision, so we need too
 229     $nodename = lc($nodename);
 230     $ip_address = lc($ip_address);
 231
 232     mkdir $pmxcfs_auth_dir;
 233
 234     if (! -f $sshknownhosts) {
 235         if (my $fh = IO::File->new($sshknownhosts, O_CREAT|O_WRONLY|O_EXCL, 0600)) {
 236             close($fh);
 237         }
 238     }
 239
 240     my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
 241
 242     my $new = '';
 243
 244     if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
 245         $new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
 246     }
 247
 248     my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
 249     # Note: file sometimes containe emty lines at start, so we use multiline match
 250     die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
 251     $hostkey = $1;
 252
 253     my $data = '';
 254     my $vhash = {};
 255
 256     my $found_nodename;
 257     my $found_local_ip;
 258
 259     my $merge_line = sub {
 260         my ($line, $all) = @_;
 261
 262         return if $line =~ m/^\s*$/; # skip empty lines
 263         return if $line =~ m/^#/; # skip comments
 264
 265         if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
 266             my $key = $1;
 267             my $rsakey = $2;
 268             if (!$vhash->{$key}) {
 269                 $vhash->{$key} = 1;
 270                 if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
 271                     my $salt = decode_base64($1);
 272                     my $digest = $2;
 273                     my $hmac = Digest::HMAC_SHA1->new($salt);
 274                     $hmac->add($nodename);
 275                     my $hd = $hmac->b64digest . '=';
 276                     if ($digest eq $hd) {
 277                         if ($rsakey eq $hostkey) {
 278                             $found_nodename = 1;
 279                             $data .= $line;
 280                         }
 281                         return;
 282                     }
 283                     $hmac = Digest::HMAC_SHA1->new($salt);
 284                     $hmac->add($ip_address);
 285                     $hd = $hmac->b64digest . '=';
 286                     if ($digest eq $hd) {
 287                         if ($rsakey eq $hostkey) {
 288                             $found_local_ip = 1;
 289                             $data .= $line;
 290                         }
 291                         return;
 292                     }
 293                 } else {
 294                     $key = lc($key); # avoid duplicate entries, ssh compares lowercased
 295                     if ($key eq $ip_address) {
 296                         $found_local_ip = 1 if $rsakey eq $hostkey;
 297                     } elsif ($key eq $nodename) {
 298                         $found_nodename = 1 if $rsakey eq $hostkey;
 299                     }
 300                 }
 301                 $data .= $line;
 302             }
 303         } elsif ($all) {
 304             $data .= $line;
 305         }
 306     };
 307
 308     while ($old && $old =~ s/^((.*?)(\n|$))//) {
 309         my $line = "$2\n";
 310         &$merge_line($line, 1);
 311     }
 312
 313     while ($new && $new =~ s/^((.*?)(\n|$))//) {
 314         my $line = "$2\n";
 315         &$merge_line($line);
 316     }
 317
 318     # add our own key if not already there
 319     $data .= "$nodename $hostkey\n" if !$found_nodename;
 320     $data .= "$ip_address $hostkey\n" if !$found_local_ip;
 321
 322     PVE::Tools::file_set_contents($sshknownhosts, $data);
 323
 324     return if !$createLink;
 325
 326     unlink $sshglobalknownhosts;
 327     symlink $sshknownhosts, $sshglobalknownhosts;
 328
 329     warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
 330         if ! -l $sshglobalknownhosts;
 331
 332 }
 333
 334 # directory and file creation
 335
 336 sub gen_local_dirs {
 337     my ($nodename) = @_;
 338
 339     PVE::Cluster::check_cfs_is_mounted();
 340
 341     my @required_dirs = (
 342         "$pmxcfs_base_dir/priv",
 343         "$pmxcfs_base_dir/nodes",
 344         "$pmxcfs_base_dir/nodes/$nodename",
 345         "$pmxcfs_base_dir/nodes/$nodename/lxc",
 346         "$pmxcfs_base_dir/nodes/$nodename/qemu-server",
 347         "$pmxcfs_base_dir/nodes/$nodename/openvz",
 348         "$pmxcfs_base_dir/nodes/$nodename/priv");
 349
 350     foreach my $dir (@required_dirs) {
 351         if (! -d $dir) {
 352             mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
 353         }
 354     }
 355 }
 356
 357 sub gen_auth_key {
 358     my $authprivkeyfn = "$pmxcfs_auth_dir/authkey.key";
 359     my $authpubkeyfn = "$pmxcfs_base_dir/authkey.pub";
 360
 361     return if -f "$authprivkeyfn";
 362
 363     PVE::Cluster::check_cfs_is_mounted();
 364
 365     PVE::Cluster::cfs_lock_authkey(undef, sub {
 366         mkdir $pmxcfs_auth_dir || $! == EEXIST || die "unable to create dir '$pmxcfs_auth_dir' - $!\n";
 367
 368         run_silent_cmd(['openssl', 'genrsa', '-out', $authprivkeyfn, '2048']);
 369
 370         run_silent_cmd(['openssl', 'rsa', '-in', $authprivkeyfn, '-pubout', '-out', $authpubkeyfn]);
 371     });
 372
 373     die "$@\n" if $@;
 374 }
 375
 376 sub gen_pveca_key {
 377
 378     return if -f $pveca_key_fn;
 379
 380     eval {
 381         run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']);
 382     };
 383
 384     die "unable to generate pve ca key:\n$@" if $@;
 385 }
 386
 387 sub gen_pveca_cert {
 388
 389     if (-f $pveca_key_fn && -f $pveca_cert_fn) {
 390         return 0;
 391     }
 392
 393     gen_pveca_key();
 394
 395     # we try to generate an unique 'subject' to avoid browser problems
 396     # (reused serial numbers, ..)
 397     my $uuid;
 398     UUID::generate($uuid);
 399     my $uuid_str;
 400     UUID::unparse($uuid, $uuid_str);
 401
 402     eval {
 403         # wrap openssl with faketime to prevent bug #904
 404         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'req', '-batch',
 405                         '-days', '3650', '-new', '-x509', '-nodes', '-key',
 406                         $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
 407                         "/CN=Proxmox Virtual Environment/OU=$uuid_str/O=PVE Cluster Manager CA/"]);
 408     };
 409
 410     die "generating pve root certificate failed:\n$@" if $@;
 411
 412     return 1;
 413 }
 414
 415 sub gen_pve_ssl_key {
 416     my ($nodename) = @_;
 417
 418     die "no node name specified" if !$nodename;
 419
 420     my $pvessl_key_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.key";
 421
 422     return if -f $pvessl_key_fn;
 423
 424     eval {
 425         run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
 426     };
 427
 428     die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
 429 }
 430
 431 sub gen_pve_www_key {
 432
 433     return if -f $pvewww_key_fn;
 434
 435     eval {
 436         run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
 437     };
 438
 439     die "unable to generate pve www key:\n$@" if $@;
 440 }
 441
 442 sub update_serial {
 443     my ($serial) = @_;
 444
 445     PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
 446 }
 447
 448 sub gen_pve_ssl_cert {
 449     my ($force, $nodename, $ip) = @_;
 450
 451     die "no node name specified" if !$nodename;
 452     die "no IP specified" if !$ip;
 453
 454     my $pvessl_cert_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.pem";
 455
 456     return if !$force && -f $pvessl_cert_fn;
 457
 458     my $names = "IP:127.0.0.1,IP:::1,DNS:localhost";
 459
 460     my $rc = PVE::INotify::read_file('resolvconf');
 461
 462     $names .= ",IP:$ip";
 463
 464     $names .= ",DNS:$nodename";
 465
 466     my $fqdn = $nodename;
 467     if ($rc && $rc->{search}) {
 468         $fqdn .= ".$rc->{search}";
 469         $names .= ",DNS:$fqdn";
 470     }
 471
 472     my $sslconf = <<__EOD;
 473 RANDFILE = /root/.rnd
 474 extensions = v3_req
 475
 476 [ req ]
 477 default_bits = 2048
 478 distinguished_name = req_distinguished_name
 479 req_extensions = v3_req
 480 prompt = no
 481 string_mask = nombstr
 482
 483 [ req_distinguished_name ]
 484 organizationalUnitName = PVE Cluster Node
 485 organizationName = Proxmox Virtual Environment
 486 commonName = $fqdn
 487
 488 [ v3_req ]
 489 basicConstraints = CA:FALSE
 490 extendedKeyUsage = serverAuth
 491 subjectAltName = $names
 492 __EOD
 493
 494     my $cfgfn = "/tmp/pvesslconf-$$.tmp";
 495     my $fh = IO::File->new ($cfgfn, "w");
 496     print $fh $sslconf;
 497     close ($fh);
 498
 499     my $reqfn = "/tmp/pvecertreq-$$.tmp";
 500     unlink $reqfn;
 501
 502     my $pvessl_key_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.key";
 503     eval {
 504         run_silent_cmd([
 505             'openssl', 'req', '-batch', '-new', '-config', $cfgfn, '-key', $pvessl_key_fn, '-out', $reqfn
 506         ]);
 507     };
 508
 509     if (my $err = $@) {
 510         unlink $reqfn;
 511         unlink $cfgfn;
 512         die "unable to generate pve certificate request:\n$err";
 513     }
 514
 515     update_serial("0000000000000000") if ! -f $pveca_srl_fn;
 516
 517     # get ca expiry
 518     my $cainfo = PVE::Certificate::get_certificate_info($pveca_cert_fn);
 519     my $daysleft = int(($cainfo->{notafter} - time())/(24*60*60));
 520
 521     if ($daysleft < 14) {
 522         die "CA expires in less than 2 weeks, unable to generate certificate.\n";
 523     }
 524
 525     # let the certificate expire a little sooner that the ca, so subtract 2 days
 526     $daysleft -= 2;
 527
 528     # we want the certificates to only last 2 years, since some browsers
 529     # do not accept certificates with very long expiry time
 530     if ($daysleft >= 2*365) {
 531         $daysleft = 2*365;
 532     }
 533
 534     eval {
 535         run_silent_cmd([
 536             'faketime', 'yesterday', # NOTE: wrap openssl with faketime to prevent bug #904
 537             'openssl', 'x509', '-req', '-in', $reqfn, '-days', $daysleft, '-out', $pvessl_cert_fn,
 538             '-CAkey', $pveca_key_fn, '-CA', $pveca_cert_fn, '-CAserial', $pveca_srl_fn, '-extfile', $cfgfn
 539         ]);
 540     };
 541
 542     if (my $err = $@) {
 543         unlink $reqfn;
 544         unlink $cfgfn;
 545         die "unable to generate pve ssl certificate:\n$err";
 546     }
 547
 548     unlink $cfgfn;
 549     unlink $reqfn;
 550 }
 551
 552 sub gen_pve_node_files {
 553     my ($nodename, $ip, $opt_force) = @_;
 554
 555     gen_local_dirs($nodename);
 556
 557     gen_auth_key();
 558
 559     # make sure we have a (cluster wide) secret
 560     # for CSRFR prevention
 561     gen_pve_www_key();
 562
 563     # make sure we have a (per node) private key
 564     gen_pve_ssl_key($nodename);
 565
 566     # make sure we have a CA
 567     my $force = gen_pveca_cert();
 568
 569     $force = 1 if $opt_force;
 570
 571     gen_pve_ssl_cert($force, $nodename, $ip);
 572 }
 573
 574 my $vzdump_cron_dummy = <<__EOD;
 575 # cluster wide vzdump cron schedule
 576 # Atomatically generated file - do not edit
 577
 578 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
 579
 580 __EOD
 581
 582 sub gen_pve_vzdump_symlink {
 583
 584     my $filename = "/etc/pve/vzdump.cron";
 585
 586     my $link_fn = "/etc/cron.d/vzdump";
 587
 588     if ((-f $filename) && (! -l $link_fn)) {
 589         rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
 590         symlink($filename, $link_fn);
 591     }
 592 }
 593
 594 sub gen_pve_vzdump_files {
 595
 596     my $filename = "/etc/pve/vzdump.cron";
 597
 598     PVE::Tools::file_set_contents($filename, $vzdump_cron_dummy)
 599         if ! -f $filename;
 600
 601     gen_pve_vzdump_symlink();
 602 };
 603
 604 # join helpers
 605
 606 sub assert_joinable {
 607     my ($local_addr, $links, $force) = @_;
 608
 609     my $errors = '';
 610     my $error = sub { $errors .= "* $_[0]\n"; };
 611
 612     if (-f $authfile) {
 613         $error->("authentication key '$authfile' already exists");
 614     }
 615
 616     if (-f $clusterconf)  {
 617         $error->("cluster config '$clusterconf' already exists");
 618     }
 619
 620     my $vmlist = PVE::Cluster::get_vmlist();
 621     if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
 622         $error->("this host already contains virtual guests");
 623     }
 624
 625     if (PVE::Tools::run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
 626         $error->("corosync is already running, is this node already in a cluster?!");
 627     }
 628
 629     # check if corosync ring IPs are configured on the current nodes interfaces
 630     my $check_ip = sub {
 631         my $ip = shift // return;
 632         my $logid = shift;
 633         if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
 634             my $host = $ip;
 635             eval { $ip = PVE::Network::get_ip_from_hostname($host); };
 636             if ($@) {
 637                 $error->("$logid: cannot use '$host': $@\n") ;
 638                 return;
 639             }
 640         }
 641
 642         my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
 643         my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
 644
 645         $error->("$logid: cannot use IP '$ip', not found on local node!\n")
 646             if scalar(@$configured_ips) < 1;
 647     };
 648
 649     $check_ip->($local_addr, 'local node address');
 650
 651     foreach my $link (keys %$links) {
 652         $check_ip->($links->{$link}->{address}, "link$link");
 653     }
 654
 655     if ($errors) {
 656         warn "detected the following error(s):\n$errors";
 657         die "Check if node may join a cluster failed!\n" if !$force;
 658         warn "\nWARNING : detected error but forced to continue!\n\n";
 659     }
 660 }
 661
 662 sub join {
 663     my ($param) = @_;
 664
 665     my $nodename = PVE::INotify::nodename();
 666     my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 667
 668     my $links = PVE::Corosync::extract_corosync_link_args($param);
 669
 670     # check if we can join with the given parameters and current node state
 671     assert_joinable($local_ip_address, $links, $param->{force});
 672
 673     setup_sshd_config();
 674     setup_rootsshconfig();
 675     setup_ssh_keys();
 676
 677     # make sure known_hosts is on local filesystem
 678     ssh_unmerge_known_hosts();
 679
 680     my $host = $param->{hostname};
 681     my $conn_args = {
 682         username => 'root@pam',
 683         password => $param->{password},
 684         cookie_name => 'PVEAuthCookie',
 685         protocol => 'https',
 686         host => $host,
 687         port => 8006,
 688     };
 689
 690     if (my $fp = $param->{fingerprint}) {
 691         $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
 692     } else {
 693         # API schema ensures that we can only get here from CLI handler
 694         $conn_args->{manual_verification} = 1;
 695     }
 696
 697     print "Establishing API connection with host '$host'\n";
 698
 699     my $conn = PVE::APIClient::LWP->new(%$conn_args);
 700     $conn->login();
 701
 702     # login raises an exception on failure, so if we get here we're good
 703     print "Login succeeded.\n";
 704
 705     print "check cluster join API version\n";
 706     my $apiver = eval { $conn->get("/cluster/config/apiversion") } // 0;
 707     assert_we_can_join_cluster_version($apiver);
 708
 709     my $args = {};
 710     $args->{force} = $param->{force} if defined($param->{force});
 711     $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
 712     $args->{votes} = $param->{votes} if defined($param->{votes});
 713     foreach my $link (keys %$links) {
 714         $args->{"link$link"} = PVE::Corosync::print_corosync_link($links->{$link});
 715     }
 716
 717     # this will be used as fallback if no links are specified
 718     if (!%$links) {
 719         $args->{link0} = $local_ip_address if $apiver == 0;
 720         $args->{new_node_ip} = $local_ip_address if $apiver >= 1;
 721
 722         print "No cluster network links passed explicitly, fallback to local node"
 723             . " IP '$local_ip_address'\n";
 724     }
 725
 726     if ($apiver >= 1) {
 727         $args->{apiversion} = JOIN_API_VERSION;
 728     }
 729
 730     print "Request addition of this node\n";
 731     my $res = eval { $conn->post("/cluster/config/nodes/$nodename", $args); };
 732     if (my $err = $@) {
 733         if (ref($err) && $err->isa('PVE::APIClient::Exception')) {
 734             # we received additional info about the error, show the user
 735             chomp $err->{msg};
 736             warn "An error occurred on the cluster node: $err->{msg}\n";
 737             foreach my $key (sort keys %{$err->{errors}}) {
 738                 my $symbol = ($key =~ m/^warning/) ? '*' : '!';
 739                 warn "$symbol $err->{errors}->{$key}\n";
 740             }
 741
 742             die "Cluster join aborted!\n";
 743         }
 744
 745         die $@;
 746     }
 747
 748     if (defined($res->{warnings})) {
 749         foreach my $warn (@{$res->{warnings}}) {
 750             warn "cluster: $warn\n";
 751         }
 752     }
 753
 754     print "Join request OK, finishing setup locally\n";
 755
 756     # added successfuly - now prepare local node
 757     finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
 758 }
 759
 760 sub finish_join {
 761     my ($nodename, $corosync_conf, $corosync_authkey) = @_;
 762
 763     mkdir "$localclusterdir";
 764     PVE::Tools::file_set_contents($authfile, $corosync_authkey);
 765     PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
 766
 767     print "stopping pve-cluster service\n";
 768     my $cmd = ['systemctl', 'stop', 'pve-cluster'];
 769     PVE::Tools::run_command($cmd, errmsg => "can't stop pve-cluster service");
 770
 771     my $dbfile = PVE::Cluster::cfs_backup_database();
 772     unlink $dbfile;
 773
 774     $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
 775     PVE::Tools::run_command($cmd, errmsg => "starting pve-cluster failed");
 776
 777     # wait for quorum
 778     my $printqmsg = 1;
 779     while (!PVE::Cluster::check_cfs_quorum(1)) {
 780         if ($printqmsg) {
 781             print "waiting for quorum...";
 782             STDOUT->flush();
 783             $printqmsg = 0;
 784         }
 785         sleep(1);
 786     }
 787     print "OK\n" if !$printqmsg;
 788
 789     updatecerts_and_ssh(1);
 790
 791     print "generated new node certificate, restart pveproxy and pvedaemon services\n";
 792     PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
 793
 794     print "successfully added node '$nodename' to cluster.\n";
 795 }
 796
 797 sub updatecerts_and_ssh {
 798     my ($force_new_cert, $silent) = @_;
 799
 800     my $p = sub { print "$_[0]\n" if !$silent };
 801
 802     setup_rootsshconfig();
 803
 804     gen_pve_vzdump_symlink();
 805
 806     if (!PVE::Cluster::check_cfs_quorum(1)) {
 807         return undef if $silent;
 808         die "no quorum - unable to update files\n";
 809     }
 810
 811     setup_ssh_keys();
 812
 813     my $nodename = PVE::INotify::nodename();
 814     my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 815
 816     $p->("(re)generate node files");
 817     $p->("generate new node certificate") if $force_new_cert;
 818     gen_pve_node_files($nodename, $local_ip_address, $force_new_cert);
 819
 820     $p->("merge authorized SSH keys and known hosts");
 821     ssh_merge_keys();
 822     ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 823     gen_pve_vzdump_files();
 824 }
 825
 826 1;