data/PVE/Cluster/Setup.pm

   1 package PVE::Cluster::Setup;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Digest::HMAC_SHA1;
   7 use Digest::SHA;
   8 use IO::File;
   9 use MIME::Base64;
  10 use Net::IP;
  11 use UUID;
  12 use POSIX qw(EEXIST);
  13
  14 use PVE::APIClient::LWP;
  15 use PVE::Cluster;
  16 use PVE::Corosync;
  17 use PVE::INotify;
  18 use PVE::JSONSchema;
  19 use PVE::Network;
  20 use PVE::Tools;
  21 use PVE::Certificate;
  22
  23 my $pmxcfs_base_dir = PVE::Cluster::base_dir();
  24 my $pmxcfs_auth_dir = PVE::Cluster::auth_dir();
  25
  26 # only write output if something fails
  27 sub run_silent_cmd {
  28     my ($cmd) = @_;
  29
  30     my $outbuf = '';
  31     my $record = sub { $outbuf .= shift . "\n"; };
  32
  33     eval { PVE::Tools::run_command($cmd, outfunc => $record, errfunc => $record) };
  34
  35     if (my $err = $@) {
  36         print STDERR $outbuf;
  37         die $err;
  38     }
  39 }
  40
  41 # Corosync related files
  42 my $localclusterdir = "/etc/corosync";
  43 my $localclusterconf = "$localclusterdir/corosync.conf";
  44 my $authfile = "$localclusterdir/authkey";
  45 my $clusterconf = "$pmxcfs_base_dir/corosync.conf";
  46
  47 # CA/certificate related files
  48 my $pveca_key_fn = "$pmxcfs_auth_dir/pve-root-ca.key";
  49 my $pveca_srl_fn = "$pmxcfs_auth_dir/pve-root-ca.srl";
  50 my $pveca_cert_fn = "$pmxcfs_base_dir/pve-root-ca.pem";
  51 # this is just a secret accessable by the web browser
  52 # and is used for CSRF prevention
  53 my $pvewww_key_fn = "$pmxcfs_base_dir/pve-www.key";
  54
  55 # ssh related files
  56 my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
  57 my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
  58 my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
  59 my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
  60 my $sshknownhosts = "$pmxcfs_auth_dir/known_hosts";
  61 my $sshauthkeys = "$pmxcfs_auth_dir/authorized_keys";
  62 my $sshd_config_fn = "/etc/ssh/sshd_config";
  63 my $rootsshauthkeys = "/root/.ssh/authorized_keys";
  64 my $rootsshauthkeysbackup = "${rootsshauthkeys}.org";
  65 my $rootsshconfig = "/root/.ssh/config";
  66
  67 # ssh related utility functions
  68
  69 sub ssh_merge_keys {
  70     # remove duplicate keys in $sshauthkeys
  71     # ssh-copy-id simply add keys, so the file can grow to large
  72
  73     my $data = '';
  74     if (-f $sshauthkeys) {
  75         $data = PVE::Tools::file_get_contents($sshauthkeys, 128*1024);
  76         chomp($data);
  77     }
  78
  79     my $found_backup;
  80     if (-f $rootsshauthkeysbackup) {
  81         $data .= "\n";
  82         $data .= PVE::Tools::file_get_contents($rootsshauthkeysbackup, 128*1024);
  83         chomp($data);
  84         $found_backup = 1;
  85     }
  86
  87     # always add ourself
  88     if (-f $ssh_rsa_id) {
  89         my $pub = PVE::Tools::file_get_contents($ssh_rsa_id);
  90         chomp($pub);
  91         $data .= "\n$pub\n";
  92     }
  93
  94     my $newdata = "";
  95     my $vhash = {};
  96     my @lines = split(/\n/, $data);
  97     foreach my $line (@lines) {
  98         if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
  99             next if $vhash->{$3}++;
 100         }
 101         $newdata .= "$line\n";
 102     }
 103
 104     PVE::Tools::file_set_contents($sshauthkeys, $newdata, 0600);
 105
 106     if ($found_backup && -l $rootsshauthkeys) {
 107         # everything went well, so we can remove the backup
 108         unlink $rootsshauthkeysbackup;
 109     }
 110 }
 111
 112 sub setup_sshd_config {
 113     my () = @_;
 114
 115     my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
 116
 117     return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
 118
 119     if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
 120         chomp $conf;
 121         $conf .= "\nPermitRootLogin yes\n";
 122     }
 123
 124     PVE::Tools::file_set_contents($sshd_config_fn, $conf);
 125
 126     PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
 127 }
 128
 129 sub setup_rootsshconfig {
 130
 131     # create ssh key if it does not exist
 132     if (! -f $ssh_rsa_id) {
 133         mkdir '/root/.ssh/';
 134         system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
 135     }
 136
 137     # create ssh config if it does not exist
 138     if (! -f $rootsshconfig) {
 139         mkdir '/root/.ssh';
 140         if (my $fh = IO::File->new($rootsshconfig, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
 141             # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k  26 Jan 2017)
 142             # changed order to put AES before Chacha20 (most hardware has AESNI)
 143             print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
 144             close($fh);
 145         }
 146     }
 147 }
 148
 149 sub setup_ssh_keys {
 150
 151     mkdir $pmxcfs_auth_dir;
 152
 153     my $import_ok;
 154
 155     if (! -f $sshauthkeys) {
 156         my $old;
 157         if (-f $rootsshauthkeys) {
 158             $old = PVE::Tools::file_get_contents($rootsshauthkeys, 128*1024);
 159         }
 160         if (my $fh = IO::File->new ($sshauthkeys, O_CREAT|O_WRONLY|O_EXCL, 0400)) {
 161             PVE::Tools::safe_print($sshauthkeys, $fh, $old) if $old;
 162             close($fh);
 163             $import_ok = 1;
 164         }
 165     }
 166
 167     warn "can't create shared ssh key database '$sshauthkeys'\n"
 168         if ! -f $sshauthkeys;
 169
 170     if (-f $rootsshauthkeys && ! -l $rootsshauthkeys) {
 171         if (!rename($rootsshauthkeys , $rootsshauthkeysbackup)) {
 172             warn "rename $rootsshauthkeys failed - $!\n";
 173         }
 174     }
 175
 176     if (! -l $rootsshauthkeys) {
 177         symlink $sshauthkeys, $rootsshauthkeys;
 178     }
 179
 180     if (! -l $rootsshauthkeys) {
 181         warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n";
 182     } else {
 183         unlink $rootsshauthkeysbackup if $import_ok;
 184     }
 185 }
 186
 187 sub ssh_unmerge_known_hosts {
 188     return if ! -l $sshglobalknownhosts;
 189
 190     my $old = '';
 191     $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024)
 192         if -f $sshknownhosts;
 193
 194     PVE::Tools::file_set_contents($sshglobalknownhosts, $old);
 195 }
 196
 197 sub ssh_merge_known_hosts {
 198     my ($nodename, $ip_address, $createLink) = @_;
 199
 200     die "no node name specified" if !$nodename;
 201     die "no ip address specified" if !$ip_address;
 202
 203     # ssh lowercases hostnames (aliases) before comparision, so we need too
 204     $nodename = lc($nodename);
 205     $ip_address = lc($ip_address);
 206
 207     mkdir $pmxcfs_auth_dir;
 208
 209     if (! -f $sshknownhosts) {
 210         if (my $fh = IO::File->new($sshknownhosts, O_CREAT|O_WRONLY|O_EXCL, 0600)) {
 211             close($fh);
 212         }
 213     }
 214
 215     my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
 216
 217     my $new = '';
 218
 219     if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
 220         $new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
 221     }
 222
 223     my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
 224     # Note: file sometimes containe emty lines at start, so we use multiline match
 225     die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
 226     $hostkey = $1;
 227
 228     my $data = '';
 229     my $vhash = {};
 230
 231     my $found_nodename;
 232     my $found_local_ip;
 233
 234     my $merge_line = sub {
 235         my ($line, $all) = @_;
 236
 237         return if $line =~ m/^\s*$/; # skip empty lines
 238         return if $line =~ m/^#/; # skip comments
 239
 240         if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
 241             my $key = $1;
 242             my $rsakey = $2;
 243             if (!$vhash->{$key}) {
 244                 $vhash->{$key} = 1;
 245                 if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
 246                     my $salt = decode_base64($1);
 247                     my $digest = $2;
 248                     my $hmac = Digest::HMAC_SHA1->new($salt);
 249                     $hmac->add($nodename);
 250                     my $hd = $hmac->b64digest . '=';
 251                     if ($digest eq $hd) {
 252                         if ($rsakey eq $hostkey) {
 253                             $found_nodename = 1;
 254                             $data .= $line;
 255                         }
 256                         return;
 257                     }
 258                     $hmac = Digest::HMAC_SHA1->new($salt);
 259                     $hmac->add($ip_address);
 260                     $hd = $hmac->b64digest . '=';
 261                     if ($digest eq $hd) {
 262                         if ($rsakey eq $hostkey) {
 263                             $found_local_ip = 1;
 264                             $data .= $line;
 265                         }
 266                         return;
 267                     }
 268                 } else {
 269                     $key = lc($key); # avoid duplicate entries, ssh compares lowercased
 270                     if ($key eq $ip_address) {
 271                         $found_local_ip = 1 if $rsakey eq $hostkey;
 272                     } elsif ($key eq $nodename) {
 273                         $found_nodename = 1 if $rsakey eq $hostkey;
 274                     }
 275                 }
 276                 $data .= $line;
 277             }
 278         } elsif ($all) {
 279             $data .= $line;
 280         }
 281     };
 282
 283     while ($old && $old =~ s/^((.*?)(\n|$))//) {
 284         my $line = "$2\n";
 285         &$merge_line($line, 1);
 286     }
 287
 288     while ($new && $new =~ s/^((.*?)(\n|$))//) {
 289         my $line = "$2\n";
 290         &$merge_line($line);
 291     }
 292
 293     # add our own key if not already there
 294     $data .= "$nodename $hostkey\n" if !$found_nodename;
 295     $data .= "$ip_address $hostkey\n" if !$found_local_ip;
 296
 297     PVE::Tools::file_set_contents($sshknownhosts, $data);
 298
 299     return if !$createLink;
 300
 301     unlink $sshglobalknownhosts;
 302     symlink $sshknownhosts, $sshglobalknownhosts;
 303
 304     warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
 305         if ! -l $sshglobalknownhosts;
 306
 307 }
 308
 309 # directory and file creation
 310
 311 sub gen_local_dirs {
 312     my ($nodename) = @_;
 313
 314     PVE::Cluster::check_cfs_is_mounted();
 315
 316     my @required_dirs = (
 317         "$pmxcfs_base_dir/priv",
 318         "$pmxcfs_base_dir/nodes",
 319         "$pmxcfs_base_dir/nodes/$nodename",
 320         "$pmxcfs_base_dir/nodes/$nodename/lxc",
 321         "$pmxcfs_base_dir/nodes/$nodename/qemu-server",
 322         "$pmxcfs_base_dir/nodes/$nodename/openvz",
 323         "$pmxcfs_base_dir/nodes/$nodename/priv");
 324
 325     foreach my $dir (@required_dirs) {
 326         if (! -d $dir) {
 327             mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
 328         }
 329     }
 330 }
 331
 332 sub gen_auth_key {
 333     my $authprivkeyfn = "$pmxcfs_auth_dir/authkey.key";
 334     my $authpubkeyfn = "$pmxcfs_base_dir/authkey.pub";
 335
 336     return if -f "$authprivkeyfn";
 337
 338     PVE::Cluster::check_cfs_is_mounted();
 339
 340     PVE::Cluster::cfs_lock_authkey(undef, sub {
 341         mkdir $pmxcfs_auth_dir || $! == EEXIST || die "unable to create dir '$pmxcfs_auth_dir' - $!\n";
 342
 343         run_silent_cmd(['openssl', 'genrsa', '-out', $authprivkeyfn, '2048']);
 344
 345         run_silent_cmd(['openssl', 'rsa', '-in', $authprivkeyfn, '-pubout', '-out', $authpubkeyfn]);
 346     });
 347
 348     die "$@\n" if $@;
 349 }
 350
 351 sub gen_pveca_key {
 352
 353     return if -f $pveca_key_fn;
 354
 355     eval {
 356         run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']);
 357     };
 358
 359     die "unable to generate pve ca key:\n$@" if $@;
 360 }
 361
 362 sub gen_pveca_cert {
 363
 364     if (-f $pveca_key_fn && -f $pveca_cert_fn) {
 365         return 0;
 366     }
 367
 368     gen_pveca_key();
 369
 370     # we try to generate an unique 'subject' to avoid browser problems
 371     # (reused serial numbers, ..)
 372     my $uuid;
 373     UUID::generate($uuid);
 374     my $uuid_str;
 375     UUID::unparse($uuid, $uuid_str);
 376
 377     eval {
 378         # wrap openssl with faketime to prevent bug #904
 379         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'req', '-batch',
 380                         '-days', '3650', '-new', '-x509', '-nodes', '-key',
 381                         $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
 382                         "/CN=Proxmox Virtual Environment/OU=$uuid_str/O=PVE Cluster Manager CA/"]);
 383     };
 384
 385     die "generating pve root certificate failed:\n$@" if $@;
 386
 387     return 1;
 388 }
 389
 390 sub gen_pve_ssl_key {
 391     my ($nodename) = @_;
 392
 393     die "no node name specified" if !$nodename;
 394
 395     my $pvessl_key_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.key";
 396
 397     return if -f $pvessl_key_fn;
 398
 399     eval {
 400         run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
 401     };
 402
 403     die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
 404 }
 405
 406 sub gen_pve_www_key {
 407
 408     return if -f $pvewww_key_fn;
 409
 410     eval {
 411         run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
 412     };
 413
 414     die "unable to generate pve www key:\n$@" if $@;
 415 }
 416
 417 sub update_serial {
 418     my ($serial) = @_;
 419
 420     PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
 421 }
 422
 423 sub gen_pve_ssl_cert {
 424     my ($force, $nodename, $ip) = @_;
 425
 426     die "no node name specified" if !$nodename;
 427     die "no IP specified" if !$ip;
 428
 429     my $pvessl_cert_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.pem";
 430
 431     return if !$force && -f $pvessl_cert_fn;
 432
 433     my $names = "IP:127.0.0.1,IP:::1,DNS:localhost";
 434
 435     my $rc = PVE::INotify::read_file('resolvconf');
 436
 437     $names .= ",IP:$ip";
 438
 439     my $fqdn = $nodename;
 440
 441     $names .= ",DNS:$nodename";
 442
 443     if ($rc && $rc->{search}) {
 444         $fqdn = $nodename . "." . $rc->{search};
 445         $names .= ",DNS:$fqdn";
 446     }
 447
 448     my $sslconf = <<__EOD;
 449 RANDFILE = /root/.rnd
 450 extensions = v3_req
 451
 452 [ req ]
 453 default_bits = 2048
 454 distinguished_name = req_distinguished_name
 455 req_extensions = v3_req
 456 prompt = no
 457 string_mask = nombstr
 458
 459 [ req_distinguished_name ]
 460 organizationalUnitName = PVE Cluster Node
 461 organizationName = Proxmox Virtual Environment
 462 commonName = $fqdn
 463
 464 [ v3_req ]
 465 basicConstraints = CA:FALSE
 466 extendedKeyUsage = serverAuth
 467 subjectAltName = $names
 468 __EOD
 469
 470     my $cfgfn = "/tmp/pvesslconf-$$.tmp";
 471     my $fh = IO::File->new ($cfgfn, "w");
 472     print $fh $sslconf;
 473     close ($fh);
 474
 475     my $reqfn = "/tmp/pvecertreq-$$.tmp";
 476     unlink $reqfn;
 477
 478     my $pvessl_key_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.key";
 479     eval {
 480         run_silent_cmd(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
 481                         '-key', $pvessl_key_fn, '-out', $reqfn]);
 482     };
 483
 484     if (my $err = $@) {
 485         unlink $reqfn;
 486         unlink $cfgfn;
 487         die "unable to generate pve certificate request:\n$err";
 488     }
 489
 490     update_serial("0000000000000000") if ! -f $pveca_srl_fn;
 491
 492     # get ca expiry
 493     my $cainfo = PVE::Certificate::get_certificate_info($pveca_cert_fn);
 494     my $daysleft = int(($cainfo->{notafter} - time())/(24*60*60));
 495
 496     if ($daysleft < 14) {
 497         die "CA expires in less than 2 weeks, unable to generate certificate.\n";
 498     }
 499
 500     # let the certificate expire a little sooner that the ca, so subtract 2 days
 501     $daysleft -= 2;
 502
 503     # we want the certificates to only last 2 years, since some browsers
 504     # do not accept certificates with very long expiry time
 505     if ($daysleft >= 2*365) {
 506         $daysleft = 2*365;
 507     }
 508
 509     eval {
 510         # wrap openssl with faketime to prevent bug #904
 511         run_silent_cmd(['faketime', 'yesterday', 'openssl', 'x509', '-req',
 512                         '-in', $reqfn, '-days', $daysleft, '-out', $pvessl_cert_fn,
 513                         '-CAkey', $pveca_key_fn, '-CA', $pveca_cert_fn,
 514                         '-CAserial', $pveca_srl_fn, '-extfile', $cfgfn]);
 515     };
 516
 517     if (my $err = $@) {
 518         unlink $reqfn;
 519         unlink $cfgfn;
 520         die "unable to generate pve ssl certificate:\n$err";
 521     }
 522
 523     unlink $cfgfn;
 524     unlink $reqfn;
 525 }
 526
 527 sub gen_pve_node_files {
 528     my ($nodename, $ip, $opt_force) = @_;
 529
 530     gen_local_dirs($nodename);
 531
 532     gen_auth_key();
 533
 534     # make sure we have a (cluster wide) secret
 535     # for CSRFR prevention
 536     gen_pve_www_key();
 537
 538     # make sure we have a (per node) private key
 539     gen_pve_ssl_key($nodename);
 540
 541     # make sure we have a CA
 542     my $force = gen_pveca_cert();
 543
 544     $force = 1 if $opt_force;
 545
 546     gen_pve_ssl_cert($force, $nodename, $ip);
 547 }
 548
 549 my $vzdump_cron_dummy = <<__EOD;
 550 # cluster wide vzdump cron schedule
 551 # Atomatically generated file - do not edit
 552
 553 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
 554
 555 __EOD
 556
 557 sub gen_pve_vzdump_symlink {
 558
 559     my $filename = "/etc/pve/vzdump.cron";
 560
 561     my $link_fn = "/etc/cron.d/vzdump";
 562
 563     if ((-f $filename) && (! -l $link_fn)) {
 564         rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
 565         symlink($filename, $link_fn);
 566     }
 567 }
 568
 569 sub gen_pve_vzdump_files {
 570
 571     my $filename = "/etc/pve/vzdump.cron";
 572
 573     PVE::Tools::file_set_contents($filename, $vzdump_cron_dummy)
 574         if ! -f $filename;
 575
 576     gen_pve_vzdump_symlink();
 577 };
 578
 579 # join helpers
 580
 581 sub assert_joinable {
 582     my ($local_addr, $link0, $link1, $force) = @_;
 583
 584     my $errors = '';
 585     my $error = sub { $errors .= "* $_[0]\n"; };
 586
 587     if (-f $authfile) {
 588         $error->("authentication key '$authfile' already exists");
 589     }
 590
 591     if (-f $clusterconf)  {
 592         $error->("cluster config '$clusterconf' already exists");
 593     }
 594
 595     my $vmlist = PVE::Cluster::get_vmlist();
 596     if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
 597         $error->("this host already contains virtual guests");
 598     }
 599
 600     if (PVE::Tools::run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
 601         $error->("corosync is already running, is this node already in a cluster?!");
 602     }
 603
 604     # check if corosync ring IPs are configured on the current nodes interfaces
 605     my $check_ip = sub {
 606         my $ip = shift // return;
 607         my $logid = shift;
 608         if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
 609             my $host = $ip;
 610             eval { $ip = PVE::Network::get_ip_from_hostname($host); };
 611             if ($@) {
 612                 $error->("$logid: cannot use '$host': $@\n") ;
 613                 return;
 614             }
 615         }
 616
 617         my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
 618         my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
 619
 620         $error->("$logid: cannot use IP '$ip', not found on local node!\n")
 621             if scalar(@$configured_ips) < 1;
 622     };
 623
 624     $check_ip->($local_addr, 'local node address');
 625     $check_ip->($link0->{address}, 'ring0') if defined($link0);
 626     $check_ip->($link1->{address}, 'ring1') if defined($link1);
 627
 628     if ($errors) {
 629         warn "detected the following error(s):\n$errors";
 630         die "Check if node may join a cluster failed!\n" if !$force;
 631     }
 632 }
 633
 634 sub join {
 635     my ($param) = @_;
 636
 637     my $nodename = PVE::INotify::nodename();
 638     my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 639
 640     my $link0 = PVE::Corosync::parse_corosync_link($param->{link0});
 641     my $link1 = PVE::Corosync::parse_corosync_link($param->{link1});
 642
 643     # check if we can join with the given parameters and current node state
 644     assert_joinable($local_ip_address, $link0, $link1, $param->{force});
 645
 646     setup_sshd_config();
 647     setup_rootsshconfig();
 648     setup_ssh_keys();
 649
 650     # make sure known_hosts is on local filesystem
 651     ssh_unmerge_known_hosts();
 652
 653     my $host = $param->{hostname};
 654     my $conn_args = {
 655         username => 'root@pam',
 656         password => $param->{password},
 657         cookie_name => 'PVEAuthCookie',
 658         protocol => 'https',
 659         host => $host,
 660         port => 8006,
 661     };
 662
 663     if (my $fp = $param->{fingerprint}) {
 664         $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
 665     } else {
 666         # API schema ensures that we can only get here from CLI handler
 667         $conn_args->{manual_verification} = 1;
 668     }
 669
 670     print "Establishing API connection with host '$host'\n";
 671
 672     my $conn = PVE::APIClient::LWP->new(%$conn_args);
 673     $conn->login();
 674
 675     # login raises an exception on failure, so if we get here we're good
 676     print "Login succeeded.\n";
 677
 678     my $args = {};
 679     $args->{force} = $param->{force} if defined($param->{force});
 680     $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
 681     $args->{votes} = $param->{votes} if defined($param->{votes});
 682     # just pass the un-parsed string through, or as we've address as the
 683     # default_key, we can just pass the fallback directly too
 684     $args->{link0} = $param->{link0} // $local_ip_address;
 685     $args->{link1} = $param->{link1} if defined($param->{link1});
 686
 687     print "Request addition of this node\n";
 688     my $res = $conn->post("/cluster/config/nodes/$nodename", $args);
 689
 690     print "Join request OK, finishing setup locally\n";
 691
 692     # added successfuly - now prepare local node
 693     finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
 694 }
 695
 696 sub finish_join {
 697     my ($nodename, $corosync_conf, $corosync_authkey) = @_;
 698
 699     mkdir "$localclusterdir";
 700     PVE::Tools::file_set_contents($authfile, $corosync_authkey);
 701     PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
 702
 703     print "stopping pve-cluster service\n";
 704     my $cmd = ['systemctl', 'stop', 'pve-cluster'];
 705     PVE::Tools::run_command($cmd, errmsg => "can't stop pve-cluster service");
 706
 707     my $dbfile = PVE::Cluster::cfs_backup_database();
 708     unlink $dbfile;
 709
 710     $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
 711     PVE::Tools::run_command($cmd, errmsg => "starting pve-cluster failed");
 712
 713     # wait for quorum
 714     my $printqmsg = 1;
 715     while (!PVE::Cluster::check_cfs_quorum(1)) {
 716         if ($printqmsg) {
 717             print "waiting for quorum...";
 718             STDOUT->flush();
 719             $printqmsg = 0;
 720         }
 721         sleep(1);
 722     }
 723     print "OK\n" if !$printqmsg;
 724
 725     updatecerts_and_ssh(1);
 726
 727     print "generated new node certificate, restart pveproxy and pvedaemon services\n";
 728     PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
 729
 730     print "successfully added node '$nodename' to cluster.\n";
 731 }
 732
 733 sub updatecerts_and_ssh {
 734     my ($force_new_cert, $silent) = @_;
 735
 736     my $p = sub { print "$_[0]\n" if !$silent };
 737
 738     setup_rootsshconfig();
 739
 740     gen_pve_vzdump_symlink();
 741
 742     if (!PVE::Cluster::check_cfs_quorum(1)) {
 743         return undef if $silent;
 744         die "no quorum - unable to update files\n";
 745     }
 746
 747     setup_ssh_keys();
 748
 749     my $nodename = PVE::INotify::nodename();
 750     my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 751
 752     $p->("(re)generate node files");
 753     $p->("generate new node certificate") if $force_new_cert;
 754     gen_pve_node_files($nodename, $local_ip_address, $force_new_cert);
 755
 756     $p->("merge authorized SSH keys and known hosts");
 757     ssh_merge_keys();
 758     ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 759     gen_pve_vzdump_files();
 760 }
 761
 762 1;