data/PVE/Cluster/Setup.pm

   1 package PVE::Cluster::Setup;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Digest::HMAC_SHA1;
   7 use Digest::SHA;
   8 use IO::File;
   9 use MIME::Base64;
  10 use Net::IP;
  11 use UUID;
  12 use POSIX qw(EEXIST);
  13
  14 use PVE::APIClient::LWP;
  15 use PVE::Cluster;
  16 use PVE::Corosync;
  17 use PVE::INotify;
  18 use PVE::JSONSchema;
  19 use PVE::Network;
  20 use PVE::Tools;
  21 use PVE::Certificate;
  22
  23 # Only relevant for pre-join checks, after join happened versions can differ
  24 use constant JOIN_API_VERSION => 1;
  25 # (APIVER-this) is oldest version a new node in addnode can have to be accepted
  26 use constant JOIN_API_AGE_AS_CLUSTER => 1;
  27 # (APIVER-this) is oldest version a cluster node can have to still try joining
  28 use constant JOIN_API_AGE_AS_JOINEE => 1;
  29
  30 my $pmxcfs_base_dir = PVE::Cluster::base_dir();
  31 my $pmxcfs_auth_dir = PVE::Cluster::auth_dir();
  32
  33 # only write output if something fails
  34 sub run_silent_cmd {
  35     my ($cmd) = @_;
  36
  37     my $outbuf = '';
  38     my $record = sub { $outbuf .= shift . "\n"; };
  39
  40     eval { PVE::Tools::run_command($cmd, outfunc => $record, errfunc => $record) };
  41
  42     if (my $err = $@) {
  43         print STDERR $outbuf;
  44         die $err;
  45     }
  46 }
  47
  48 # Corosync related files
  49 my $localclusterdir = "/etc/corosync";
  50 my $localclusterconf = "$localclusterdir/corosync.conf";
  51 my $authfile = "$localclusterdir/authkey";
  52 my $clusterconf = "$pmxcfs_base_dir/corosync.conf";
  53
  54 # CA/certificate related files
  55 my $pveca_key_fn = "$pmxcfs_auth_dir/pve-root-ca.key";
  56 my $pveca_srl_fn = "$pmxcfs_auth_dir/pve-root-ca.srl";
  57 my $pveca_cert_fn = "$pmxcfs_base_dir/pve-root-ca.pem";
  58 # this is just a secret accessable by the web browser
  59 # and is used for CSRF prevention
  60 my $pvewww_key_fn = "$pmxcfs_base_dir/pve-www.key";
  61
  62 # ssh related files
  63 my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
  64 my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
  65 my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
  66 my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
  67 my $sshknownhosts = "$pmxcfs_auth_dir/known_hosts";
  68 my $sshauthkeys = "$pmxcfs_auth_dir/authorized_keys";
  69 my $sshd_config_fn = "/etc/ssh/sshd_config";
  70 my $rootsshauthkeys = "/root/.ssh/authorized_keys";
  71 my $rootsshauthkeysbackup = "${rootsshauthkeys}.org";
  72 my $rootsshconfig = "/root/.ssh/config";
  73
  74 # ssh related utility functions
  75
  76 sub ssh_merge_keys {
  77     # remove duplicate keys in $sshauthkeys
  78     # ssh-copy-id simply add keys, so the file can grow to large
  79
  80     my $data = '';
  81     if (-f $sshauthkeys) {
  82         $data = PVE::Tools::file_get_contents($sshauthkeys, 128*1024);
  83         chomp($data);
  84     }
  85
  86     my $found_backup;
  87     if (-f $rootsshauthkeysbackup) {
  88         $data .= "\n";
  89         $data .= PVE::Tools::file_get_contents($rootsshauthkeysbackup, 128*1024);
  90         chomp($data);
  91         $found_backup = 1;
  92     }
  93
  94     # always add ourself
  95     if (-f $ssh_rsa_id) {
  96         my $pub = PVE::Tools::file_get_contents($ssh_rsa_id);
  97         chomp($pub);
  98         $data .= "\n$pub\n";
  99     }
 100
 101     my $newdata = "";
 102     my $vhash = {};
 103     my @lines = split(/\n/, $data);
 104     foreach my $line (@lines) {
 105         if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
 106             next if $vhash->{$3}++;
 107         }
 108         $newdata .= "$line\n";
 109     }
 110
 111     PVE::Tools::file_set_contents($sshauthkeys, $newdata, 0600);
 112
 113     if ($found_backup && -l $rootsshauthkeys) {
 114         # everything went well, so we can remove the backup
 115         unlink $rootsshauthkeysbackup;
 116     }
 117 }
 118
 119 sub setup_sshd_config {
 120     my () = @_;
 121
 122     my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
 123
 124     return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
 125
 126     if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
 127         chomp $conf;
 128         $conf .= "\nPermitRootLogin yes\n";
 129     }
 130
 131     PVE::Tools::file_set_contents($sshd_config_fn, $conf);
 132
 133     PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
 134 }
 135
 136 sub setup_rootsshconfig {
 137
 138     # create ssh key if it does not exist
 139     if (! -f $ssh_rsa_id) {
 140         mkdir '/root/.ssh/';
 141         system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
 142     }
 143
 144     # create ssh config if it does not exist
 145     if (! -f $rootsshconfig) {
 146         mkdir '/root/.ssh';
 147         if (my $fh = IO::File->new($rootsshconfig, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
 148             # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k  26 Jan 2017)
 149             # changed order to put AES before Chacha20 (most hardware has AESNI)
 150             print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
 151             close($fh);
 152         }
 153     }
 154 }
 155
 156 sub setup_ssh_keys {
 157
 158     mkdir $pmxcfs_auth_dir;
 159
 160     my $import_ok;
 161
 162     if (! -f $sshauthkeys) {
 163         my $old;
 164         if (-f $rootsshauthkeys) {
 165             $old = PVE::Tools::file_get_contents($rootsshauthkeys, 128*1024);
 166         }
 167         if (my $fh = IO::File->new ($sshauthkeys, O_CREAT|O_WRONLY|O_EXCL, 0400)) {
 168             PVE::Tools::safe_print($sshauthkeys, $fh, $old) if $old;
 169             close($fh);
 170             $import_ok = 1;
 171         }
 172     }
 173
 174     warn "can't create shared ssh key database '$sshauthkeys'\n"
 175         if ! -f $sshauthkeys;
 176
 177     if (-f $rootsshauthkeys && ! -l $rootsshauthkeys) {
 178         if (!rename($rootsshauthkeys , $rootsshauthkeysbackup)) {
 179             warn "rename $rootsshauthkeys failed - $!\n";
 180         }
 181     }
 182
 183     if (! -l $rootsshauthkeys) {
 184         symlink $sshauthkeys, $rootsshauthkeys;
 185     }
 186
 187     if (! -l $rootsshauthkeys) {
 188         warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n";
 189     } else {
 190         unlink $rootsshauthkeysbackup if $import_ok;
 191     }
 192 }
 193
 194 sub ssh_unmerge_known_hosts {
 195     return if ! -l $sshglobalknownhosts;
 196
 197     my $old = '';
 198     $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024)
 199         if -f $sshknownhosts;
 200
 201     PVE::Tools::file_set_contents($sshglobalknownhosts, $old);
 202 }
 203
 204 sub ssh_merge_known_hosts {
 205     my ($nodename, $ip_address, $createLink) = @_;
 206
 207     die "no node name specified" if !$nodename;
 208     die "no ip address specified" if !$ip_address;
 209
 210     # ssh lowercases hostnames (aliases) before comparision, so we need too
 211     $nodename = lc($nodename);
 212     $ip_address = lc($ip_address);
 213
 214     mkdir $pmxcfs_auth_dir;
 215
 216     if (! -f $sshknownhosts) {
 217         if (my $fh = IO::File->new($sshknownhosts, O_CREAT|O_WRONLY|O_EXCL, 0600)) {
 218             close($fh);
 219         }
 220     }
 221
 222     my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
 223
 224     my $new = '';
 225
 226     if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
 227         $new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
 228     }
 229
 230     my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
 231     # Note: file sometimes containe emty lines at start, so we use multiline match
 232     die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
 233     $hostkey = $1;
 234
 235     my $data = '';
 236     my $vhash = {};
 237
 238     my $found_nodename;
 239     my $found_local_ip;
 240
 241     my $merge_line = sub {
 242         my ($line, $all) = @_;
 243
 244         return if $line =~ m/^\s*$/; # skip empty lines
 245         return if $line =~ m/^#/; # skip comments
 246
 247         if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
 248             my $key = $1;
 249             my $rsakey = $2;
 250             if (!$vhash->{$key}) {
 251                 $vhash->{$key} = 1;
 252                 if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
 253                     my $salt = decode_base64($1);
 254                     my $digest = $2;
 255                     my $hmac = Digest::HMAC_SHA1->new($salt);
 256                     $hmac->add($nodename);
 257                     my $hd = $hmac->b64digest . '=';
 258                     if ($digest eq $hd) {
 259                         if ($rsakey eq $hostkey) {
 260                             $found_nodename = 1;
 261                             $data .= $line;
 262                         }
 263                         return;
 264                     }
 265                     $hmac = Digest::HMAC_SHA1->new($salt);
 266                     $hmac->add($ip_address);
 267                     $hd = $hmac->b64digest . '=';
 268                     if ($digest eq $hd) {
 269                         if ($rsakey eq $hostkey) {
 270                             $found_local_ip = 1;
 271                             $data .= $line;
 272                         }
 273                         return;
 274                     }
 275                 } else {
 276                     $key = lc($key); # avoid duplicate entries, ssh compares lowercased
 277                     if ($key eq $ip_address) {
 278                         $found_local_ip = 1 if $rsakey eq $hostkey;
 279                     } elsif ($key eq $nodename) {
 280                         $found_nodename = 1 if $rsakey eq $hostkey;
 281                     }
 282                 }
 283                 $data .= $line;
 284             }
 285         } elsif ($all) {
 286             $data .= $line;
 287         }
 288     };
 289
 290     while ($old && $old =~ s/^((.*?)(\n|$))//) {
 291         my $line = "$2\n";
 292         &$merge_line($line, 1);
 293     }
 294
 295     while ($new && $new =~ s/^((.*?)(\n|$))//) {
 296         my $line = "$2\n";
 297         &$merge_line($line);
 298     }
 299
 300     # add our own key if not already there
 301     $data .= "$nodename $hostkey\n" if !$found_nodename;
 302     $data .= "$ip_address $hostkey\n" if !$found_local_ip;
 303
 304     PVE::Tools::file_set_contents($sshknownhosts, $data);
 305
 306     return if !$createLink;
 307
 308     unlink $sshglobalknownhosts;
 309     symlink $sshknownhosts, $sshglobalknownhosts;
 310
 311     warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
 312         if ! -l $sshglobalknownhosts;
 313
 314 }
 315
 316 # directory and file creation
 317
 318 sub gen_local_dirs {
 319     my ($nodename) = @_;
 320
 321     PVE::Cluster::check_cfs_is_mounted();
 322
 323     my @required_dirs = (
 324         "$pmxcfs_base_dir/priv",
 325         "$pmxcfs_base_dir/nodes",
 326         "$pmxcfs_base_dir/nodes/$nodename",
 327         "$pmxcfs_base_dir/nodes/$nodename/lxc",
 328         "$pmxcfs_base_dir/nodes/$nodename/qemu-server",
 329         "$pmxcfs_base_dir/nodes/$nodename/openvz",
 330         "$pmxcfs_base_dir/nodes/$nodename/priv");
 331
 332     foreach my $dir (@required_dirs) {
 333         if (! -d $dir) {
 334             mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
 335         }
 336     }
 337 }
 338
 339 sub gen_auth_key {
 340     my $authprivkeyfn = "$pmxcfs_auth_dir/authkey.key";
 341     my $authpubkeyfn = "$pmxcfs_base_dir/authkey.pub";
 342
 343     return if -f "$authprivkeyfn";
 344
 345     PVE::Cluster::check_cfs_is_mounted();
 346
 347     PVE::Cluster::cfs_lock_authkey(undef, sub {
 348         mkdir $pmxcfs_auth_dir || $! == EEXIST || die "unable to create dir '$pmxcfs_auth_dir' - $!\n";
 349
 350         run_silent_cmd(['openssl', 'genrsa', '-out', $authprivkeyfn, '2048']);
 351
 352         run_silent_cmd(['openssl', 'rsa', '-in', $authprivkeyfn, '-pubout', '-out', $authpubkeyfn]);
 353     });
 354
 355     die "$@\n" if $@;
 356 }
 357
 358 sub gen_pveca_key {
 359
 360     return if -f $pveca_key_fn;
 361
 362     eval {
 363         run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']);
 364     };
 365
 366     die "unable to generate pve ca key:\n$@" if $@;
 367 }
 368
 369 sub gen_pveca_cert {
 370
 371     if (-f $pveca_key_fn && -f $pveca_cert_fn) {
 372         return 0;
 373     }
 374
 375     gen_pveca_key();
 376
 377     # we try to generate an unique 'subject' to avoid browser problems
 378     # (reused serial numbers, ..)
 379     my $uuid;
 380     UUID::generate($uuid);
 381     my $uuid_str;
 382     UUID::unparse($uuid, $uuid_str);
 383
 384     eval {
 385         # wrap openssl with faketime to prevent bug #904
 386         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'req', '-batch',
 387                         '-days', '3650', '-new', '-x509', '-nodes', '-key',
 388                         $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
 389                         "/CN=Proxmox Virtual Environment/OU=$uuid_str/O=PVE Cluster Manager CA/"]);
 390     };
 391
 392     die "generating pve root certificate failed:\n$@" if $@;
 393
 394     return 1;
 395 }
 396
 397 sub gen_pve_ssl_key {
 398     my ($nodename) = @_;
 399
 400     die "no node name specified" if !$nodename;
 401
 402     my $pvessl_key_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.key";
 403
 404     return if -f $pvessl_key_fn;
 405
 406     eval {
 407         run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
 408     };
 409
 410     die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
 411 }
 412
 413 sub gen_pve_www_key {
 414
 415     return if -f $pvewww_key_fn;
 416
 417     eval {
 418         run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
 419     };
 420
 421     die "unable to generate pve www key:\n$@" if $@;
 422 }
 423
 424 sub update_serial {
 425     my ($serial) = @_;
 426
 427     PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
 428 }
 429
 430 sub gen_pve_ssl_cert {
 431     my ($force, $nodename, $ip) = @_;
 432
 433     die "no node name specified" if !$nodename;
 434     die "no IP specified" if !$ip;
 435
 436     my $pvessl_cert_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.pem";
 437
 438     return if !$force && -f $pvessl_cert_fn;
 439
 440     my $names = "IP:127.0.0.1,IP:::1,DNS:localhost";
 441
 442     my $rc = PVE::INotify::read_file('resolvconf');
 443
 444     $names .= ",IP:$ip";
 445
 446     my $fqdn = $nodename;
 447
 448     $names .= ",DNS:$nodename";
 449
 450     if ($rc && $rc->{search}) {
 451         $fqdn = $nodename . "." . $rc->{search};
 452         $names .= ",DNS:$fqdn";
 453     }
 454
 455     my $sslconf = <<__EOD;
 456 RANDFILE = /root/.rnd
 457 extensions = v3_req
 458
 459 [ req ]
 460 default_bits = 2048
 461 distinguished_name = req_distinguished_name
 462 req_extensions = v3_req
 463 prompt = no
 464 string_mask = nombstr
 465
 466 [ req_distinguished_name ]
 467 organizationalUnitName = PVE Cluster Node
 468 organizationName = Proxmox Virtual Environment
 469 commonName = $fqdn
 470
 471 [ v3_req ]
 472 basicConstraints = CA:FALSE
 473 extendedKeyUsage = serverAuth
 474 subjectAltName = $names
 475 __EOD
 476
 477     my $cfgfn = "/tmp/pvesslconf-$$.tmp";
 478     my $fh = IO::File->new ($cfgfn, "w");
 479     print $fh $sslconf;
 480     close ($fh);
 481
 482     my $reqfn = "/tmp/pvecertreq-$$.tmp";
 483     unlink $reqfn;
 484
 485     my $pvessl_key_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.key";
 486     eval {
 487         run_silent_cmd(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
 488                         '-key', $pvessl_key_fn, '-out', $reqfn]);
 489     };
 490
 491     if (my $err = $@) {
 492         unlink $reqfn;
 493         unlink $cfgfn;
 494         die "unable to generate pve certificate request:\n$err";
 495     }
 496
 497     update_serial("0000000000000000") if ! -f $pveca_srl_fn;
 498
 499     # get ca expiry
 500     my $cainfo = PVE::Certificate::get_certificate_info($pveca_cert_fn);
 501     my $daysleft = int(($cainfo->{notafter} - time())/(24*60*60));
 502
 503     if ($daysleft < 14) {
 504         die "CA expires in less than 2 weeks, unable to generate certificate.\n";
 505     }
 506
 507     # let the certificate expire a little sooner that the ca, so subtract 2 days
 508     $daysleft -= 2;
 509
 510     # we want the certificates to only last 2 years, since some browsers
 511     # do not accept certificates with very long expiry time
 512     if ($daysleft >= 2*365) {
 513         $daysleft = 2*365;
 514     }
 515
 516     eval {
 517         # wrap openssl with faketime to prevent bug #904
 518         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'x509', '-req',
 519                         '-in', $reqfn, '-days', $daysleft, '-out', $pvessl_cert_fn,
 520                         '-CAkey', $pveca_key_fn, '-CA', $pveca_cert_fn,
 521                         '-CAserial', $pveca_srl_fn, '-extfile', $cfgfn]);
 522     };
 523
 524     if (my $err = $@) {
 525         unlink $reqfn;
 526         unlink $cfgfn;
 527         die "unable to generate pve ssl certificate:\n$err";
 528     }
 529
 530     unlink $cfgfn;
 531     unlink $reqfn;
 532 }
 533
 534 sub gen_pve_node_files {
 535     my ($nodename, $ip, $opt_force) = @_;
 536
 537     gen_local_dirs($nodename);
 538
 539     gen_auth_key();
 540
 541     # make sure we have a (cluster wide) secret
 542     # for CSRFR prevention
 543     gen_pve_www_key();
 544
 545     # make sure we have a (per node) private key
 546     gen_pve_ssl_key($nodename);
 547
 548     # make sure we have a CA
 549     my $force = gen_pveca_cert();
 550
 551     $force = 1 if $opt_force;
 552
 553     gen_pve_ssl_cert($force, $nodename, $ip);
 554 }
 555
 556 my $vzdump_cron_dummy = <<__EOD;
 557 # cluster wide vzdump cron schedule
 558 # Atomatically generated file - do not edit
 559
 560 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
 561
 562 __EOD
 563
 564 sub gen_pve_vzdump_symlink {
 565
 566     my $filename = "/etc/pve/vzdump.cron";
 567
 568     my $link_fn = "/etc/cron.d/vzdump";
 569
 570     if ((-f $filename) && (! -l $link_fn)) {
 571         rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
 572         symlink($filename, $link_fn);
 573     }
 574 }
 575
 576 sub gen_pve_vzdump_files {
 577
 578     my $filename = "/etc/pve/vzdump.cron";
 579
 580     PVE::Tools::file_set_contents($filename, $vzdump_cron_dummy)
 581         if ! -f $filename;
 582
 583     gen_pve_vzdump_symlink();
 584 };
 585
 586 # join helpers
 587
 588 sub assert_joinable {
 589     my ($local_addr, $links, $force) = @_;
 590
 591     my $errors = '';
 592     my $error = sub { $errors .= "* $_[0]\n"; };
 593
 594     if (-f $authfile) {
 595         $error->("authentication key '$authfile' already exists");
 596     }
 597
 598     if (-f $clusterconf)  {
 599         $error->("cluster config '$clusterconf' already exists");
 600     }
 601
 602     my $vmlist = PVE::Cluster::get_vmlist();
 603     if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
 604         $error->("this host already contains virtual guests");
 605     }
 606
 607     if (PVE::Tools::run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
 608         $error->("corosync is already running, is this node already in a cluster?!");
 609     }
 610
 611     # check if corosync ring IPs are configured on the current nodes interfaces
 612     my $check_ip = sub {
 613         my $ip = shift // return;
 614         my $logid = shift;
 615         if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
 616             my $host = $ip;
 617             eval { $ip = PVE::Network::get_ip_from_hostname($host); };
 618             if ($@) {
 619                 $error->("$logid: cannot use '$host': $@\n") ;
 620                 return;
 621             }
 622         }
 623
 624         my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
 625         my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
 626
 627         $error->("$logid: cannot use IP '$ip', not found on local node!\n")
 628             if scalar(@$configured_ips) < 1;
 629     };
 630
 631     $check_ip->($local_addr, 'local node address');
 632
 633     foreach my $link (keys %$links) {
 634         $check_ip->($links->{$link}->{address}, "link$link");
 635     }
 636
 637     if ($errors) {
 638         warn "detected the following error(s):\n$errors";
 639         die "Check if node may join a cluster failed!\n" if !$force;
 640         warn "\nWARNING : detected error but forced to continue!\n\n";
 641     }
 642 }
 643
 644 sub join {
 645     my ($param) = @_;
 646
 647     my $nodename = PVE::INotify::nodename();
 648     my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 649
 650     my $links = PVE::Corosync::extract_corosync_link_args($param);
 651
 652     # check if we can join with the given parameters and current node state
 653     assert_joinable($local_ip_address, $links, $param->{force});
 654
 655     setup_sshd_config();
 656     setup_rootsshconfig();
 657     setup_ssh_keys();
 658
 659     # make sure known_hosts is on local filesystem
 660     ssh_unmerge_known_hosts();
 661
 662     my $host = $param->{hostname};
 663     my $conn_args = {
 664         username => 'root@pam',
 665         password => $param->{password},
 666         cookie_name => 'PVEAuthCookie',
 667         protocol => 'https',
 668         host => $host,
 669         port => 8006,
 670     };
 671
 672     if (my $fp = $param->{fingerprint}) {
 673         $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
 674     } else {
 675         # API schema ensures that we can only get here from CLI handler
 676         $conn_args->{manual_verification} = 1;
 677     }
 678
 679     print "Establishing API connection with host '$host'\n";
 680
 681     my $conn = PVE::APIClient::LWP->new(%$conn_args);
 682     $conn->login();
 683
 684     # login raises an exception on failure, so if we get here we're good
 685     print "Login succeeded.\n";
 686
 687     # check cluster join API version
 688     my $apiver = eval { $conn->get("/cluster/config/apiversion") } // 0;
 689
 690     if ($apiver < (JOIN_API_VERSION - JOIN_API_AGE_AS_JOINEE)) {
 691         die "error: incompatible join API version on cluster ($apiver, local has "
 692             . JOIN_API_VERSION . "). Make sure all nodes are up-to-date.\n";
 693     }
 694
 695     my $args = {};
 696     $args->{force} = $param->{force} if defined($param->{force});
 697     $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
 698     $args->{votes} = $param->{votes} if defined($param->{votes});
 699     foreach my $link (keys %$links) {
 700         $args->{"link$link"} = PVE::Corosync::print_corosync_link($links->{$link});
 701     }
 702
 703     # this will be used as fallback if no links are specified
 704     if (!%$links) {
 705         $args->{link0} = $local_ip_address if $apiver == 0;
 706         $args->{new_node_ip} = $local_ip_address if $apiver >= 1;
 707
 708         print "No cluster network links passed explicitly, fallback to local node"
 709             . " IP '$local_ip_address'\n";
 710     }
 711
 712     if ($apiver >= 1) {
 713         $args->{apiversion} = JOIN_API_VERSION;
 714     }
 715
 716     print "Request addition of this node\n";
 717     my $res = eval { $conn->post("/cluster/config/nodes/$nodename", $args); };
 718     if (my $err = $@) {
 719         if (ref($err) && $err->isa('PVE::APIClient::Exception')) {
 720             # we received additional info about the error, show the user
 721             chomp $err->{msg};
 722             warn "An error occured on the cluster node: $err->{msg}\n";
 723             foreach my $key (sort keys %{$err->{errors}}) {
 724                 my $symbol = ($key =~ m/^warning/) ? '*' : '!';
 725                 warn "$symbol $err->{errors}->{$key}\n";
 726             }
 727
 728             die "Cluster join aborted!\n";
 729         }
 730
 731         die $@;
 732     }
 733
 734     if (defined($res->{warnings})) {
 735         foreach my $warn (@{$res->{warnings}}) {
 736             warn "cluster: $warn\n";
 737         }
 738     }
 739
 740     print "Join request OK, finishing setup locally\n";
 741
 742     # added successfuly - now prepare local node
 743     finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
 744 }
 745
 746 sub finish_join {
 747     my ($nodename, $corosync_conf, $corosync_authkey) = @_;
 748
 749     mkdir "$localclusterdir";
 750     PVE::Tools::file_set_contents($authfile, $corosync_authkey);
 751     PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
 752
 753     print "stopping pve-cluster service\n";
 754     my $cmd = ['systemctl', 'stop', 'pve-cluster'];
 755     PVE::Tools::run_command($cmd, errmsg => "can't stop pve-cluster service");
 756
 757     my $dbfile = PVE::Cluster::cfs_backup_database();
 758     unlink $dbfile;
 759
 760     $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
 761     PVE::Tools::run_command($cmd, errmsg => "starting pve-cluster failed");
 762
 763     # wait for quorum
 764     my $printqmsg = 1;
 765     while (!PVE::Cluster::check_cfs_quorum(1)) {
 766         if ($printqmsg) {
 767             print "waiting for quorum...";
 768             STDOUT->flush();
 769             $printqmsg = 0;
 770         }
 771         sleep(1);
 772     }
 773     print "OK\n" if !$printqmsg;
 774
 775     updatecerts_and_ssh(1);
 776
 777     print "generated new node certificate, restart pveproxy and pvedaemon services\n";
 778     PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
 779
 780     print "successfully added node '$nodename' to cluster.\n";
 781 }
 782
 783 sub updatecerts_and_ssh {
 784     my ($force_new_cert, $silent) = @_;
 785
 786     my $p = sub { print "$_[0]\n" if !$silent };
 787
 788     setup_rootsshconfig();
 789
 790     gen_pve_vzdump_symlink();
 791
 792     if (!PVE::Cluster::check_cfs_quorum(1)) {
 793         return undef if $silent;
 794         die "no quorum - unable to update files\n";
 795     }
 796
 797     setup_ssh_keys();
 798
 799     my $nodename = PVE::INotify::nodename();
 800     my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 801
 802     $p->("(re)generate node files");
 803     $p->("generate new node certificate") if $force_new_cert;
 804     gen_pve_node_files($nodename, $local_ip_address, $force_new_cert);
 805
 806     $p->("merge authorized SSH keys and known hosts");
 807     ssh_merge_keys();
 808     ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 809     gen_pve_vzdump_files();
 810 }
 811
 812 1;