5 use POSIX
qw(EEXIST ENOENT);
8 use Storable
qw(dclone);
12 use Digest
::HMAC_SHA1
;
14 use PVE
::Tools
qw(run_command);
20 use PVE
::Cluster
::IPCConst
;
33 use Data
::Dumper
; # fixme: remove
35 # x509 certificate utils
37 my $basedir = "/etc/pve";
38 my $authdir = "$basedir/priv";
39 my $lockdir = "/etc/pve/priv/lock";
41 # cfs and corosync files
42 my $dbfile = "/var/lib/pve-cluster/config.db";
43 my $dbbackupdir = "/var/lib/pve-cluster/backup";
44 my $localclusterdir = "/etc/corosync";
45 my $localclusterconf = "$localclusterdir/corosync.conf";
46 my $authfile = "$localclusterdir/authkey";
47 my $clusterconf = "$basedir/corosync.conf";
49 my $authprivkeyfn = "$authdir/authkey.key";
50 my $authpubkeyfn = "$basedir/authkey.pub";
51 my $pveca_key_fn = "$authdir/pve-root-ca.key";
52 my $pveca_srl_fn = "$authdir/pve-root-ca.srl";
53 my $pveca_cert_fn = "$basedir/pve-root-ca.pem";
54 # this is just a secret accessable by the web browser
55 # and is used for CSRF prevention
56 my $pvewww_key_fn = "$basedir/pve-www.key";
59 my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
60 my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
61 my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
62 my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
63 my $sshknownhosts = "/etc/pve/priv/known_hosts";
64 my $sshauthkeys = "/etc/pve/priv/authorized_keys";
65 my $sshd_config_fn = "/etc/ssh/sshd_config";
66 my $rootsshauthkeys = "/root/.ssh/authorized_keys";
67 my $rootsshauthkeysbackup = "${rootsshauthkeys}.org";
68 my $rootsshconfig = "/root/.ssh/config";
70 # this is just a readonly copy, the relevant one is in status.c from pmxcfs
71 # observed files are the one we can get directly through IPCC, they are cached
72 # using a computed version and only those can be used by the cfs_*_file methods
76 'datacenter.cfg' => 1,
77 'replication.cfg' => 1,
79 'corosync.conf.new' => 1,
82 'priv/shadow.cfg' => 1,
86 'ha/crm_commands' => 1,
87 'ha/manager_status' => 1,
88 'ha/resources.cfg' => 1,
95 # only write output if something fails
100 my $record = sub { $outbuf .= shift . "\n"; };
102 eval { run_command
($cmd, outfunc
=> $record, errfunc
=> $record) };
105 print STDERR
$outbuf;
110 sub check_cfs_quorum
{
113 # note: -w filename always return 1 for root, so wee need
114 # to use File::lstat here
115 my $st = File
::stat::lstat("$basedir/local");
116 my $quorate = ($st && (($st->mode & 0200) != 0));
118 die "cluster not ready - no quorum?\n" if !$quorate && !$noerr;
123 sub check_cfs_is_mounted
{
126 my $res = -l
"$basedir/local";
128 die "pve configuration filesystem not mounted\n"
137 check_cfs_is_mounted
();
139 my @required_dirs = (
142 "$basedir/nodes/$nodename",
143 "$basedir/nodes/$nodename/lxc",
144 "$basedir/nodes/$nodename/qemu-server",
145 "$basedir/nodes/$nodename/openvz",
146 "$basedir/nodes/$nodename/priv");
148 foreach my $dir (@required_dirs) {
150 mkdir($dir) || $! == EEXIST
|| die "unable to create directory '$dir' - $!\n";
157 return if -f
"$authprivkeyfn";
159 check_cfs_is_mounted
();
161 mkdir $authdir || $! == EEXIST
|| die "unable to create dir '$authdir' - $!\n";
163 run_silent_cmd
(['openssl', 'genrsa', '-out', $authprivkeyfn, '2048']);
165 run_silent_cmd
(['openssl', 'rsa', '-in', $authprivkeyfn, '-pubout', '-out', $authpubkeyfn]);
170 return if -f
$pveca_key_fn;
173 run_silent_cmd
(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']);
176 die "unable to generate pve ca key:\n$@" if $@;
181 if (-f
$pveca_key_fn && -f
$pveca_cert_fn) {
187 # we try to generate an unique 'subject' to avoid browser problems
188 # (reused serial numbers, ..)
190 UUID
::generate
($uuid);
192 UUID
::unparse
($uuid, $uuid_str);
195 # wrap openssl with faketime to prevent bug #904
196 run_silent_cmd
(['faketime', 'yesterday', 'openssl', 'req', '-batch',
197 '-days', '3650', '-new', '-x509', '-nodes', '-key',
198 $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
199 "/CN=Proxmox Virtual Environment/OU=$uuid_str/O=PVE Cluster Manager CA/"]);
202 die "generating pve root certificate failed:\n$@" if $@;
207 sub gen_pve_ssl_key
{
210 die "no node name specified" if !$nodename;
212 my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
214 return if -f
$pvessl_key_fn;
217 run_silent_cmd
(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
220 die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
223 sub gen_pve_www_key
{
225 return if -f
$pvewww_key_fn;
228 run_silent_cmd
(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
231 die "unable to generate pve www key:\n$@" if $@;
237 PVE
::Tools
::file_set_contents
($pveca_srl_fn, $serial);
240 sub gen_pve_ssl_cert
{
241 my ($force, $nodename, $ip) = @_;
243 die "no node name specified" if !$nodename;
244 die "no IP specified" if !$ip;
246 my $pvessl_cert_fn = "$basedir/nodes/$nodename/pve-ssl.pem";
248 return if !$force && -f
$pvessl_cert_fn;
250 my $names = "IP:127.0.0.1,IP:::1,DNS:localhost";
252 my $rc = PVE
::INotify
::read_file
('resolvconf');
256 my $fqdn = $nodename;
258 $names .= ",DNS:$nodename";
260 if ($rc && $rc->{search
}) {
261 $fqdn = $nodename . "." . $rc->{search
};
262 $names .= ",DNS:$fqdn";
265 my $sslconf = <<__EOD;
266 RANDFILE = /root/.rnd
271 distinguished_name = req_distinguished_name
272 req_extensions = v3_req
274 string_mask = nombstr
276 [ req_distinguished_name ]
277 organizationalUnitName = PVE Cluster Node
278 organizationName = Proxmox Virtual Environment
282 basicConstraints = CA:FALSE
283 extendedKeyUsage = serverAuth
284 subjectAltName = $names
287 my $cfgfn = "/tmp/pvesslconf-$$.tmp";
288 my $fh = IO
::File-
>new ($cfgfn, "w");
292 my $reqfn = "/tmp/pvecertreq-$$.tmp";
295 my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
297 run_silent_cmd
(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
298 '-key', $pvessl_key_fn, '-out', $reqfn]);
304 die "unable to generate pve certificate request:\n$err";
307 update_serial
("0000000000000000") if ! -f
$pveca_srl_fn;
310 # wrap openssl with faketime to prevent bug #904
311 run_silent_cmd
(['faketime', 'yesterday', 'openssl', 'x509', '-req',
312 '-in', $reqfn, '-days', '3650', '-out', $pvessl_cert_fn,
313 '-CAkey', $pveca_key_fn, '-CA', $pveca_cert_fn,
314 '-CAserial', $pveca_srl_fn, '-extfile', $cfgfn]);
320 die "unable to generate pve ssl certificate:\n$err";
327 sub gen_pve_node_files
{
328 my ($nodename, $ip, $opt_force) = @_;
330 gen_local_dirs
($nodename);
334 # make sure we have a (cluster wide) secret
335 # for CSRFR prevention
338 # make sure we have a (per node) private key
339 gen_pve_ssl_key
($nodename);
341 # make sure we have a CA
342 my $force = gen_pveca_cert
();
344 $force = 1 if $opt_force;
346 gen_pve_ssl_cert
($force, $nodename, $ip);
349 my $vzdump_cron_dummy = <<__EOD;
350 # cluster wide vzdump cron schedule
351 # Atomatically generated file - do not edit
353 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
357 sub gen_pve_vzdump_symlink
{
359 my $filename = "/etc/pve/vzdump.cron";
361 my $link_fn = "/etc/cron.d/vzdump";
363 if ((-f
$filename) && (! -l
$link_fn)) {
364 rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
365 symlink($filename, $link_fn);
369 sub gen_pve_vzdump_files
{
371 my $filename = "/etc/pve/vzdump.cron";
373 PVE
::Tools
::file_set_contents
($filename, $vzdump_cron_dummy)
376 gen_pve_vzdump_symlink
();
383 my $ipcc_send_rec = sub {
384 my ($msgid, $data) = @_;
386 my $res = PVE
::IPCC
::ipcc_send_rec
($msgid, $data);
388 die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
393 my $ipcc_send_rec_json = sub {
394 my ($msgid, $data) = @_;
396 my $res = PVE
::IPCC
::ipcc_send_rec
($msgid, $data);
398 die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
400 return decode_json
($res);
403 my $ipcc_get_config = sub {
406 my $bindata = pack "Z*", $path;
407 my $res = PVE
::IPCC
::ipcc_send_rec
(CFS_IPC_GET_CONFIG
, $bindata);
408 if (!defined($res)) {
410 return undef if $! == ENOENT
;
419 my $ipcc_get_status = sub {
420 my ($name, $nodename) = @_;
422 my $bindata = pack "Z[256]Z[256]", $name, ($nodename || "");
423 return PVE
::IPCC
::ipcc_send_rec
(CFS_IPC_GET_STATUS
, $bindata);
426 my $ipcc_update_status = sub {
427 my ($name, $data) = @_;
429 my $raw = ref($data) ? encode_json
($data) : $data;
431 my $bindata = pack "Z[256]Z*", $name, $raw;
433 return &$ipcc_send_rec(CFS_IPC_SET_STATUS
, $bindata);
437 my ($priority, $ident, $tag, $msg) = @_;
439 my $bindata = pack "CCCZ*Z*Z*", $priority, bytes
::length($ident) + 1,
440 bytes
::length($tag) + 1, $ident, $tag, $msg;
442 return &$ipcc_send_rec(CFS_IPC_LOG_CLUSTER_MSG
, $bindata);
445 my $ipcc_get_cluster_log = sub {
446 my ($user, $max) = @_;
448 $max = 0 if !defined($max);
450 my $bindata = pack "VVVVZ*", $max, 0, 0, 0, ($user || "");
451 return &$ipcc_send_rec(CFS_IPC_GET_CLUSTER_LOG
, $bindata);
459 my $res = &$ipcc_send_rec_json(CFS_IPC_GET_FS_VERSION
);
460 #warn "GOT1: " . Dumper($res);
461 die "no starttime\n" if !$res->{starttime
};
463 if (!$res->{starttime
} || !$versions->{starttime
} ||
464 $res->{starttime
} != $versions->{starttime
}) {
465 #print "detected changed starttime\n";
484 if (!$clinfo->{version
} || $clinfo->{version
} != $versions->{clinfo
}) {
485 #warn "detected new clinfo\n";
486 $clinfo = &$ipcc_send_rec_json(CFS_IPC_GET_CLUSTER_INFO
);
497 if (!$vmlist->{version
} || $vmlist->{version
} != $versions->{vmlist
}) {
498 #warn "detected new vmlist1\n";
499 $vmlist = &$ipcc_send_rec_json(CFS_IPC_GET_GUEST_LIST
);
519 return $clinfo->{nodelist
};
524 my $nodelist = $clinfo->{nodelist
};
528 my $nodename = PVE
::INotify
::nodename
();
530 if (!$nodelist || !$nodelist->{$nodename}) {
531 return [ $nodename ];
534 return [ keys %$nodelist ];
537 # $data must be a chronological descending ordered array of tasks
538 sub broadcast_tasklist
{
541 # the serialized list may not get bigger than 32kb (CFS_MAX_STATUS_SIZE
542 # from pmxcfs) - drop older items until we satisfy this constraint
543 my $size = length(encode_json
($data));
544 while ($size >= (32 * 1024)) {
546 $size = length(encode_json
($data));
550 &$ipcc_update_status("tasklist", $data);
556 my $tasklistcache = {};
561 my $kvstore = $versions->{kvstore
} || {};
563 my $nodelist = get_nodelist
();
566 foreach my $node (@$nodelist) {
567 next if $nodename && ($nodename ne $node);
569 my $ver = $kvstore->{$node}->{tasklist
} if $kvstore->{$node};
570 my $cd = $tasklistcache->{$node};
571 if (!$cd || !$ver || !$cd->{version
} ||
572 ($cd->{version
} != $ver)) {
573 my $raw = &$ipcc_get_status("tasklist", $node) || '[]';
574 my $data = decode_json
($raw);
576 $cd = $tasklistcache->{$node} = {
580 } elsif ($cd && $cd->{data
}) {
581 push @$res, @{$cd->{data
}};
585 syslog
('err', $err) if $err;
592 my ($rrdid, $data) = @_;
595 &$ipcc_update_status("rrd/$rrdid", $data);
602 my $last_rrd_dump = 0;
603 my $last_rrd_data = "";
609 my $diff = $ctime - $last_rrd_dump;
611 return $last_rrd_data;
616 $raw = &$ipcc_send_rec(CFS_IPC_GET_RRD_DUMP
);
628 while ($raw =~ s/^(.*)\n//) {
629 my ($key, @ela) = split(/:/, $1);
631 next if !(scalar(@ela) > 1);
632 $res->{$key} = [ map { $_ eq 'U' ?
undef : $_ } @ela ];
636 $last_rrd_dump = $ctime;
637 $last_rrd_data = $res;
642 sub create_rrd_data
{
643 my ($rrdname, $timeframe, $cf) = @_;
645 my $rrddir = "/var/lib/rrdcached/db";
647 my $rrd = "$rrddir/$rrdname";
651 day
=> [ 60*30, 70 ],
652 week
=> [ 60*180, 70 ],
653 month
=> [ 60*720, 70 ],
654 year
=> [ 60*10080, 70 ],
657 my ($reso, $count) = @{$setup->{$timeframe}};
658 my $ctime = $reso*int(time()/$reso);
659 my $req_start = $ctime - $reso*$count;
661 $cf = "AVERAGE" if !$cf;
669 my $socket = "/var/run/rrdcached.sock";
670 push @args, "--daemon" => "unix:$socket" if -S
$socket;
672 my ($start, $step, $names, $data) = RRDs
::fetch
($rrd, $cf, @args);
674 my $err = RRDs
::error
;
675 die "RRD error: $err\n" if $err;
677 die "got wrong time resolution ($step != $reso)\n"
681 my $fields = scalar(@$names);
682 for my $line (@$data) {
683 my $entry = { 'time' => $start };
685 for (my $i = 0; $i < $fields; $i++) {
686 my $name = $names->[$i];
687 if (defined(my $val = $line->[$i])) {
688 $entry->{$name} = $val;
690 # leave empty fields undefined
691 # maybe make this configurable?
700 sub create_rrd_graph
{
701 my ($rrdname, $timeframe, $ds, $cf) = @_;
703 # Using RRD graph is clumsy - maybe it
704 # is better to simply fetch the data, and do all display
705 # related things with javascript (new extjs html5 graph library).
707 my $rrddir = "/var/lib/rrdcached/db";
709 my $rrd = "$rrddir/$rrdname";
711 my @ids = PVE
::Tools
::split_list
($ds);
713 my $ds_txt = join('_', @ids);
715 my $filename = "${rrd}_${ds_txt}.png";
719 day
=> [ 60*30, 70 ],
720 week
=> [ 60*180, 70 ],
721 month
=> [ 60*720, 70 ],
722 year
=> [ 60*10080, 70 ],
725 my ($reso, $count) = @{$setup->{$timeframe}};
728 "--imgformat" => "PNG",
732 "--start" => - $reso*$count,
734 "--lower-limit" => 0,
737 my $socket = "/var/run/rrdcached.sock";
738 push @args, "--daemon" => "unix:$socket" if -S
$socket;
740 my @coldef = ('#00ddff', '#ff0000');
742 $cf = "AVERAGE" if !$cf;
745 foreach my $id (@ids) {
746 my $col = $coldef[$i++] || die "fixme: no color definition";
747 push @args, "DEF:${id}=$rrd:${id}:$cf";
749 if ($id eq 'cpu' || $id eq 'iowait') {
750 push @args, "CDEF:${id}_per=${id},100,*";
751 $dataid = "${id}_per";
753 push @args, "LINE2:${dataid}${col}:${id}";
756 push @args, '--full-size-mode';
758 # we do not really store data into the file
759 my $res = RRDs
::graphv
('-', @args);
761 my $err = RRDs
::error
;
762 die "RRD error: $err\n" if $err;
764 return { filename
=> $filename, image
=> $res->{image
} };
767 # a fast way to read files (avoid fuse overhead)
771 return &$ipcc_get_config($path);
774 sub get_cluster_log
{
775 my ($user, $max) = @_;
777 return &$ipcc_get_cluster_log($user, $max);
782 sub cfs_register_file
{
783 my ($filename, $parser, $writer) = @_;
785 $observed->{$filename} || die "unknown file '$filename'";
787 die "file '$filename' already registered" if $file_info->{$filename};
789 $file_info->{$filename} = {
795 my $ccache_read = sub {
796 my ($filename, $parser, $version) = @_;
798 $ccache->{$filename} = {} if !$ccache->{$filename};
800 my $ci = $ccache->{$filename};
802 if (!$ci->{version
} || !$version || $ci->{version
} != $version) {
803 # we always call the parser, even when the file does not exists
804 # (in that case $data is undef)
805 my $data = get_config
($filename);
806 $ci->{data
} = &$parser("/etc/pve/$filename", $data);
807 $ci->{version
} = $version;
810 my $res = ref($ci->{data
}) ? dclone
($ci->{data
}) : $ci->{data
};
815 sub cfs_file_version
{
820 if ($filename =~ m!^nodes/[^/]+/(openvz|lxc|qemu-server)/(\d+)\.conf$!) {
821 my ($type, $vmid) = ($1, $2);
822 if ($vmlist && $vmlist->{ids
} && $vmlist->{ids
}->{$vmid}) {
823 $version = $vmlist->{ids
}->{$vmid}->{version
};
825 $infotag = "/$type/";
827 $infotag = $filename;
828 $version = $versions->{$filename};
831 my $info = $file_info->{$infotag} ||
832 die "unknown file type '$filename'\n";
834 return wantarray ?
($version, $info) : $version;
840 my ($version, $info) = cfs_file_version
($filename);
841 my $parser = $info->{parser
};
843 return &$ccache_read($filename, $parser, $version);
847 my ($filename, $data) = @_;
849 my ($version, $info) = cfs_file_version
($filename);
851 my $writer = $info->{writer
} || die "no writer defined";
853 my $fsname = "/etc/pve/$filename";
855 my $raw = &$writer($fsname, $data);
857 if (my $ci = $ccache->{$filename}) {
858 $ci->{version
} = undef;
861 PVE
::Tools
::file_set_contents
($fsname, $raw);
865 my ($lockid, $timeout, $code, @param) = @_;
867 my $prev_alarm = alarm(0); # suspend outer alarm early
872 # this timeout is for acquire the lock
873 $timeout = 10 if !$timeout;
875 my $filename = "$lockdir/$lockid";
882 die "pve cluster filesystem not online.\n";
885 my $timeout_err = sub { die "got lock request timeout\n"; };
886 local $SIG{ALRM
} = $timeout_err;
890 $got_lock = mkdir($filename);
891 $timeout = alarm(0) - 1; # we'll sleep for 1s, see down below
895 $timeout_err->() if $timeout <= 0;
897 print STDERR
"trying to acquire cfs lock '$lockid' ...\n";
898 utime (0, 0, $filename); # cfs unlock request
902 # fixed command timeout: cfs locks have a timeout of 120
903 # using 60 gives us another 60 seconds to abort the task
904 local $SIG{ALRM
} = sub { die "got lock timeout - aborting command\n"; };
907 cfs_update
(); # make sure we read latest versions inside code()
909 $res = &$code(@param);
916 $err = "no quorum!\n" if !$got_lock && !check_cfs_quorum
(1);
918 rmdir $filename if $got_lock; # if we held the lock always unlock again
923 $@ = "error with cfs lock '$lockid': $err";
933 my ($filename, $timeout, $code, @param) = @_;
935 my $info = $observed->{$filename} || die "unknown file '$filename'";
937 my $lockid = "file-$filename";
938 $lockid =~ s/[.\/]/_
/g
;
940 &$cfs_lock($lockid, $timeout, $code, @param);
943 sub cfs_lock_storage
{
944 my ($storeid, $timeout, $code, @param) = @_;
946 my $lockid = "storage-$storeid";
948 &$cfs_lock($lockid, $timeout, $code, @param);
951 sub cfs_lock_domain
{
952 my ($domainname, $timeout, $code, @param) = @_;
954 my $lockid = "domain-$domainname";
956 &$cfs_lock($lockid, $timeout, $code, @param);
960 my ($account, $timeout, $code, @param) = @_;
962 my $lockid = "acme-$account";
964 &$cfs_lock($lockid, $timeout, $code, @param);
982 my ($priority, $ident, $msg) = @_;
984 if (my $tmp = $log_levels->{$priority}) {
988 die "need numeric log priority" if $priority !~ /^\d+$/;
990 my $tag = PVE
::SafeSyslog
::tag
();
992 $msg = "empty message" if !$msg;
994 $ident = "" if !$ident;
995 $ident = encode
("ascii", $ident,
996 sub { sprintf "\\u%04x", shift });
998 my $ascii = encode
("ascii", $msg, sub { sprintf "\\u%04x", shift });
1001 syslog
($priority, "<%s> %s", $ident, $ascii);
1003 syslog
($priority, "%s", $ascii);
1006 eval { &$ipcc_log($priority, $ident, $tag, $ascii); };
1008 syslog
("err", "writing cluster log failed: $@") if $@;
1011 sub check_vmid_unused
{
1012 my ($vmid, $noerr) = @_;
1014 my $vmlist = get_vmlist
();
1016 my $d = $vmlist->{ids
}->{$vmid};
1017 return 1 if !defined($d);
1019 return undef if $noerr;
1021 my $vmtypestr = $d->{type
} eq 'qemu' ?
'VM' : 'CT';
1022 die "$vmtypestr $vmid already exists on node '$d->{node}'\n";
1025 sub check_node_exists
{
1026 my ($nodename, $noerr) = @_;
1028 my $nodelist = $clinfo->{nodelist
};
1029 return 1 if $nodelist && $nodelist->{$nodename};
1031 return undef if $noerr;
1033 die "no such cluster node '$nodename'\n";
1036 # this is also used to get the IP of the local node
1037 sub remote_node_ip
{
1038 my ($nodename, $noerr) = @_;
1040 my $nodelist = $clinfo->{nodelist
};
1041 if ($nodelist && $nodelist->{$nodename}) {
1042 if (my $ip = $nodelist->{$nodename}->{ip
}) {
1043 return $ip if !wantarray;
1044 my $family = $nodelist->{$nodename}->{address_family
};
1046 $nodelist->{$nodename}->{address_family
} =
1048 PVE
::Tools
::get_host_address_family
($ip);
1050 return wantarray ?
($ip, $family) : $ip;
1054 # fallback: try to get IP by other means
1055 return PVE
::Network
::get_ip_from_hostname
($nodename, $noerr);
1058 sub get_local_migration_ip
{
1059 my ($migration_network, $noerr) = @_;
1061 my $cidr = $migration_network;
1063 if (!defined($cidr)) {
1064 my $dc_conf = cfs_read_file
('datacenter.cfg');
1065 $cidr = $dc_conf->{migration
}->{network
}
1066 if defined($dc_conf->{migration
}->{network
});
1069 if (defined($cidr)) {
1070 my $ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
1072 die "could not get migration ip: no IP address configured on local " .
1073 "node for network '$cidr'\n" if !$noerr && (scalar(@$ips) == 0);
1075 die "could not get migration ip: multiple IP address configured for " .
1076 "network '$cidr'\n" if !$noerr && (scalar(@$ips) > 1);
1084 # ssh related utility functions
1086 sub ssh_merge_keys
{
1087 # remove duplicate keys in $sshauthkeys
1088 # ssh-copy-id simply add keys, so the file can grow to large
1091 if (-f
$sshauthkeys) {
1092 $data = PVE
::Tools
::file_get_contents
($sshauthkeys, 128*1024);
1097 if (-f
$rootsshauthkeysbackup) {
1099 $data .= PVE
::Tools
::file_get_contents
($rootsshauthkeysbackup, 128*1024);
1104 # always add ourself
1105 if (-f
$ssh_rsa_id) {
1106 my $pub = PVE
::Tools
::file_get_contents
($ssh_rsa_id);
1108 $data .= "\n$pub\n";
1113 my @lines = split(/\n/, $data);
1114 foreach my $line (@lines) {
1115 if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
1116 next if $vhash->{$3}++;
1118 $newdata .= "$line\n";
1121 PVE
::Tools
::file_set_contents
($sshauthkeys, $newdata, 0600);
1123 if ($found_backup && -l
$rootsshauthkeys) {
1124 # everything went well, so we can remove the backup
1125 unlink $rootsshauthkeysbackup;
1129 sub setup_sshd_config
{
1132 my $conf = PVE
::Tools
::file_get_contents
($sshd_config_fn);
1134 return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
1136 if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
1138 $conf .= "\nPermitRootLogin yes\n";
1141 PVE
::Tools
::file_set_contents
($sshd_config_fn, $conf);
1143 PVE
::Tools
::run_command
(['systemctl', 'reload-or-restart', 'sshd']);
1146 sub setup_rootsshconfig
{
1148 # create ssh key if it does not exist
1149 if (! -f
$ssh_rsa_id) {
1150 mkdir '/root/.ssh/';
1151 system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
1154 # create ssh config if it does not exist
1155 if (! -f
$rootsshconfig) {
1157 if (my $fh = IO
::File-
>new($rootsshconfig, O_CREAT
|O_WRONLY
|O_EXCL
, 0640)) {
1158 # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k 26 Jan 2017)
1159 # changed order to put AES before Chacha20 (most hardware has AESNI)
1160 print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
1166 sub setup_ssh_keys
{
1172 if (! -f
$sshauthkeys) {
1174 if (-f
$rootsshauthkeys) {
1175 $old = PVE
::Tools
::file_get_contents
($rootsshauthkeys, 128*1024);
1177 if (my $fh = IO
::File-
>new ($sshauthkeys, O_CREAT
|O_WRONLY
|O_EXCL
, 0400)) {
1178 PVE
::Tools
::safe_print
($sshauthkeys, $fh, $old) if $old;
1184 warn "can't create shared ssh key database '$sshauthkeys'\n"
1185 if ! -f
$sshauthkeys;
1187 if (-f
$rootsshauthkeys && ! -l
$rootsshauthkeys) {
1188 if (!rename($rootsshauthkeys , $rootsshauthkeysbackup)) {
1189 warn "rename $rootsshauthkeys failed - $!\n";
1193 if (! -l
$rootsshauthkeys) {
1194 symlink $sshauthkeys, $rootsshauthkeys;
1197 if (! -l
$rootsshauthkeys) {
1198 warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n";
1200 unlink $rootsshauthkeysbackup if $import_ok;
1204 sub ssh_unmerge_known_hosts
{
1205 return if ! -l
$sshglobalknownhosts;
1208 $old = PVE
::Tools
::file_get_contents
($sshknownhosts, 128*1024)
1209 if -f
$sshknownhosts;
1211 PVE
::Tools
::file_set_contents
($sshglobalknownhosts, $old);
1214 sub ssh_merge_known_hosts
{
1215 my ($nodename, $ip_address, $createLink) = @_;
1217 die "no node name specified" if !$nodename;
1218 die "no ip address specified" if !$ip_address;
1220 # ssh lowercases hostnames (aliases) before comparision, so we need too
1221 $nodename = lc($nodename);
1222 $ip_address = lc($ip_address);
1226 if (! -f
$sshknownhosts) {
1227 if (my $fh = IO
::File-
>new($sshknownhosts, O_CREAT
|O_WRONLY
|O_EXCL
, 0600)) {
1232 my $old = PVE
::Tools
::file_get_contents
($sshknownhosts, 128*1024);
1236 if ((! -l
$sshglobalknownhosts) && (-f
$sshglobalknownhosts)) {
1237 $new = PVE
::Tools
::file_get_contents
($sshglobalknownhosts, 128*1024);
1240 my $hostkey = PVE
::Tools
::file_get_contents
($ssh_host_rsa_id);
1241 # Note: file sometimes containe emty lines at start, so we use multiline match
1242 die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
1251 my $merge_line = sub {
1252 my ($line, $all) = @_;
1254 return if $line =~ m/^\s*$/; # skip empty lines
1255 return if $line =~ m/^#/; # skip comments
1257 if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
1260 if (!$vhash->{$key}) {
1262 if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
1263 my $salt = decode_base64
($1);
1265 my $hmac = Digest
::HMAC_SHA1-
>new($salt);
1266 $hmac->add($nodename);
1267 my $hd = $hmac->b64digest . '=';
1268 if ($digest eq $hd) {
1269 if ($rsakey eq $hostkey) {
1270 $found_nodename = 1;
1275 $hmac = Digest
::HMAC_SHA1-
>new($salt);
1276 $hmac->add($ip_address);
1277 $hd = $hmac->b64digest . '=';
1278 if ($digest eq $hd) {
1279 if ($rsakey eq $hostkey) {
1280 $found_local_ip = 1;
1286 $key = lc($key); # avoid duplicate entries, ssh compares lowercased
1287 if ($key eq $ip_address) {
1288 $found_local_ip = 1 if $rsakey eq $hostkey;
1289 } elsif ($key eq $nodename) {
1290 $found_nodename = 1 if $rsakey eq $hostkey;
1300 while ($old && $old =~ s/^((.*?)(\n|$))//) {
1302 &$merge_line($line, 1);
1305 while ($new && $new =~ s/^((.*?)(\n|$))//) {
1307 &$merge_line($line);
1310 # add our own key if not already there
1311 $data .= "$nodename $hostkey\n" if !$found_nodename;
1312 $data .= "$ip_address $hostkey\n" if !$found_local_ip;
1314 PVE
::Tools
::file_set_contents
($sshknownhosts, $data);
1316 return if !$createLink;
1318 unlink $sshglobalknownhosts;
1319 symlink $sshknownhosts, $sshglobalknownhosts;
1321 warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
1322 if ! -l
$sshglobalknownhosts;
1326 my $migration_format = {
1330 enum
=> ['secure', 'insecure'],
1331 description
=> "Migration traffic is encrypted using an SSH tunnel by " .
1332 "default. On secure, completely private networks this can be " .
1333 "disabled to increase performance.",
1334 default => 'secure',
1338 type
=> 'string', format
=> 'CIDR',
1339 format_description
=> 'CIDR',
1340 description
=> "CIDR of the (sub) network that is used for migration."
1344 my $datacenter_schema = {
1346 additionalProperties
=> 0,
1351 description
=> "Default keybord layout for vnc server.",
1352 enum
=> PVE
::Tools
::kvmkeymaplist
(),
1357 description
=> "Default GUI language.",
1383 description
=> "Specify external http proxy which is used for downloads (example: 'http://username:password\@host:port/')",
1384 pattern
=> "http://.*",
1386 migration_unsecure
=> {
1389 description
=> "Migration is secure using SSH tunnel by default. " .
1390 "For secure private networks you can disable it to speed up " .
1391 "migration. Deprecated, use the 'migration' property instead!",
1395 type
=> 'string', format
=> $migration_format,
1396 description
=> "For cluster wide migration settings.",
1401 description
=> "Select the default Console viewer. You can either use the builtin java applet (VNC; deprecated and maps to html5), an external virt-viewer comtatible application (SPICE), an HTML5 based vnc viewer (noVNC), or an HTML5 based console client (xtermjs). If the selected viewer is not available (e.g. SPICE not activated for the VM), the fallback is noVNC.",
1402 enum
=> ['applet', 'vv', 'html5', 'xtermjs'],
1407 format
=> 'email-opt',
1408 description
=> "Specify email address to send notification from (default is root@\$hostname)",
1414 description
=> "Defines how many workers (per node) are maximal started ".
1415 " on actions like 'stopall VMs' or task from the ha-manager.",
1420 default => 'watchdog',
1421 enum
=> [ 'watchdog', 'hardware', 'both' ],
1422 description
=> "Set the fencing mode of the HA cluster. Hardware mode " .
1423 "needs a valid configuration of fence devices in /etc/pve/ha/fence.cfg." .
1424 " With both all two modes are used." .
1425 "\n\nWARNING: 'hardware' and 'both' are EXPERIMENTAL & WIP",
1430 pattern
=> qr/[a-f0-9]{2}(?::[a-f0-9]{2}){0,2}:?/i,
1431 description
=> 'Prefix for autogenerated MAC addresses.',
1433 bwlimit
=> PVE
::JSONSchema
::get_standard_option
('bwlimit'),
1437 # make schema accessible from outside (for documentation)
1438 sub get_datacenter_schema
{ return $datacenter_schema };
1440 sub parse_datacenter_config
{
1441 my ($filename, $raw) = @_;
1443 my $res = PVE
::JSONSchema
::parse_config
($datacenter_schema, $filename, $raw // '');
1445 if (my $migration = $res->{migration
}) {
1446 $res->{migration
} = PVE
::JSONSchema
::parse_property_string
($migration_format, $migration);
1449 # for backwards compatibility only, new migration property has precedence
1450 if (defined($res->{migration_unsecure
})) {
1451 if (defined($res->{migration
}->{type
})) {
1452 warn "deprecated setting 'migration_unsecure' and new 'migration: type' " .
1453 "set at same time! Ignore 'migration_unsecure'\n";
1455 $res->{migration
}->{type
} = ($res->{migration_unsecure
}) ?
'insecure' : 'secure';
1459 # for backwards compatibility only, applet maps to html5
1460 if (defined($res->{console
}) && $res->{console
} eq 'applet') {
1461 $res->{console
} = 'html5';
1467 sub write_datacenter_config
{
1468 my ($filename, $cfg) = @_;
1470 # map deprecated setting to new one
1471 if (defined($cfg->{migration_unsecure
}) && !defined($cfg->{migration
})) {
1472 my $migration_unsecure = delete $cfg->{migration_unsecure
};
1473 $cfg->{migration
}->{type
} = ($migration_unsecure) ?
'insecure' : 'secure';
1476 # map deprecated applet setting to html5
1477 if (defined($cfg->{console
}) && $cfg->{console
} eq 'applet') {
1478 $cfg->{console
} = 'html5';
1481 if (my $migration = $cfg->{migration
}) {
1482 $cfg->{migration
} = PVE
::JSONSchema
::print_property_string
($migration, $migration_format);
1485 return PVE
::JSONSchema
::dump_config
($datacenter_schema, $filename, $cfg);
1488 cfs_register_file
('datacenter.cfg',
1489 \
&parse_datacenter_config
,
1490 \
&write_datacenter_config
);
1492 # X509 Certificate cache helper
1494 my $cert_cache_nodes = {};
1495 my $cert_cache_timestamp = time();
1496 my $cert_cache_fingerprints = {};
1498 sub update_cert_cache
{
1499 my ($update_node, $clear) = @_;
1501 syslog
('info', "Clearing outdated entries from certificate cache")
1504 $cert_cache_timestamp = time() if !defined($update_node);
1506 my $node_list = defined($update_node) ?
1507 [ $update_node ] : [ keys %$cert_cache_nodes ];
1509 foreach my $node (@$node_list) {
1510 my $clear_old = sub {
1511 if (my $old_fp = $cert_cache_nodes->{$node}) {
1512 # distrust old fingerprint
1513 delete $cert_cache_fingerprints->{$old_fp};
1514 # ensure reload on next proxied request
1515 delete $cert_cache_nodes->{$node};
1519 my $fp = eval { get_node_fingerprint
($node) };
1522 &$clear_old() if $clear;
1526 my $old_fp = $cert_cache_nodes->{$node};
1527 $cert_cache_fingerprints->{$fp} = 1;
1528 $cert_cache_nodes->{$node} = $fp;
1530 if (defined($old_fp) && $fp ne $old_fp) {
1531 delete $cert_cache_fingerprints->{$old_fp};
1536 # load and cache cert fingerprint once
1537 sub initialize_cert_cache
{
1540 update_cert_cache
($node)
1541 if defined($node) && !defined($cert_cache_nodes->{$node});
1544 sub read_ssl_cert_fingerprint
{
1545 my ($cert_path) = @_;
1547 my $bio = Net
::SSLeay
::BIO_new_file
($cert_path, 'r')
1548 or die "unable to read '$cert_path' - $!\n";
1550 my $cert = Net
::SSLeay
::PEM_read_bio_X509
($bio);
1551 Net
::SSLeay
::BIO_free
($bio);
1553 die "unable to read certificate from '$cert_path'\n" if !$cert;
1555 my $fp = Net
::SSLeay
::X509_get_fingerprint
($cert, 'sha256');
1556 Net
::SSLeay
::X509_free
($cert);
1558 die "unable to get fingerprint for '$cert_path' - got empty value\n"
1559 if !defined($fp) || $fp eq '';
1564 sub get_node_fingerprint
{
1567 my $cert_path = "/etc/pve/nodes/$node/pve-ssl.pem";
1568 my $custom_cert_path = "/etc/pve/nodes/$node/pveproxy-ssl.pem";
1570 $cert_path = $custom_cert_path if -f
$custom_cert_path;
1572 return read_ssl_cert_fingerprint
($cert_path);
1576 sub check_cert_fingerprint
{
1579 # clear cache every 30 minutes at least
1580 update_cert_cache
(undef, 1) if time() - $cert_cache_timestamp >= 60*30;
1582 # get fingerprint of server certificate
1583 my $fp = Net
::SSLeay
::X509_get_fingerprint
($cert, 'sha256');
1584 return 0 if !defined($fp) || $fp eq ''; # error
1587 for my $expected (keys %$cert_cache_fingerprints) {
1588 return 1 if $fp eq $expected;
1593 return 1 if &$check();
1595 # clear cache and retry at most once every minute
1596 if (time() - $cert_cache_timestamp >= 60) {
1597 syslog
('info', "Could not verify remote node certificate '$fp' with list of pinned certificates, refreshing cache");
1598 update_cert_cache
();
1605 # bash completion helpers
1607 sub complete_next_vmid
{
1609 my $vmlist = get_vmlist
() || {};
1610 my $idlist = $vmlist->{ids
} || {};
1612 for (my $i = 100; $i < 10000; $i++) {
1613 return [$i] if !defined($idlist->{$i});
1621 my $vmlist = get_vmlist
();
1622 my $ids = $vmlist->{ids
} || {};
1624 return [ keys %$ids ];
1627 sub complete_local_vmid
{
1629 my $vmlist = get_vmlist
();
1630 my $ids = $vmlist->{ids
} || {};
1632 my $nodename = PVE
::INotify
::nodename
();
1635 foreach my $vmid (keys %$ids) {
1636 my $d = $ids->{$vmid};
1637 next if !$d->{node
} || $d->{node
} ne $nodename;
1644 sub complete_migration_target
{
1648 my $nodename = PVE
::INotify
::nodename
();
1650 my $nodelist = get_nodelist
();
1651 foreach my $node (@$nodelist) {
1652 next if $node eq $nodename;
1660 my ($node, $network_cidr) = @_;
1663 if (defined($network_cidr)) {
1664 # Use mtunnel via to get the remote node's ip inside $network_cidr.
1665 # This goes over the regular network (iow. uses get_ssh_info() with
1666 # $network_cidr undefined.
1667 # FIXME: Use the REST API client for this after creating an API entry
1668 # for get_migration_ip.
1669 my $default_remote = get_ssh_info
($node, undef);
1670 my $default_ssh = ssh_info_to_command
($default_remote);
1671 my $cmd =[@$default_ssh, 'pvecm', 'mtunnel',
1672 '-migration_network', $network_cidr,
1675 PVE
::Tools
::run_command
($cmd, outfunc
=> sub {
1678 die "internal error: unexpected output from mtunnel\n"
1680 if ($line =~ /^ip: '(.*)'$/) {
1683 die "internal error: bad output from mtunnel\n"
1687 die "failed to get ip for node '$node' in network '$network_cidr'\n"
1690 $ip = remote_node_ip
($node);
1696 network
=> $network_cidr,
1700 sub ssh_info_to_command_base
{
1701 my ($info, @extra_options) = @_;
1705 '-o', 'BatchMode=yes',
1706 '-o', 'HostKeyAlias='.$info->{name
},
1711 sub ssh_info_to_command
{
1712 my ($info, @extra_options) = @_;
1713 my $cmd = ssh_info_to_command_base
($info, @extra_options);
1714 push @$cmd, "root\@$info->{ip}";
1718 sub assert_joinable
{
1719 my ($ring0_addr, $ring1_addr, $force) = @_;
1722 my $error = sub { $errors .= "* $_[0]\n"; };
1725 $error->("authentication key '$authfile' already exists");
1728 if (-f
$clusterconf) {
1729 $error->("cluster config '$clusterconf' already exists");
1732 my $vmlist = get_vmlist
();
1733 if ($vmlist && $vmlist->{ids
} && scalar(keys %{$vmlist->{ids
}})) {
1734 $error->("this host already contains virtual guests");
1737 if (run_command
(['corosync-quorumtool', '-l'], noerr
=> 1, quiet
=> 1) == 0) {
1738 $error->("corosync is already running, is this node already in a cluster?!");
1741 # check if corosync ring IPs are configured on the current nodes interfaces
1742 my $check_ip = sub {
1743 my $ip = shift // return;
1744 if (!PVE
::JSONSchema
::pve_verify_ip
($ip, 1)) {
1746 eval { $ip = PVE
::Network
::get_ip_from_hostname
($host); };
1748 $error->("cannot use '$host': $@\n") ;
1753 my $cidr = (Net
::IP
::ip_is_ipv6
($ip)) ?
"$ip/128" : "$ip/32";
1754 my $configured_ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
1756 $error->("cannot use IP '$ip', it must be configured exactly once on local node!\n")
1757 if (scalar(@$configured_ips) != 1);
1760 $check_ip->($ring0_addr);
1761 $check_ip->($ring1_addr);
1764 warn "detected the following error(s):\n$errors";
1765 die "Check if node may join a cluster failed!\n" if !$force;
1769 # NOTE: filesystem must be offline here, no DB changes allowed
1770 my $backup_cfs_database = sub {
1776 my $backup_fn = "$dbbackupdir/config-$ctime.sql.gz";
1778 print "backup old database to '$backup_fn'\n";
1780 my $cmd = [ ['sqlite3', $dbfile, '.dump'], ['gzip', '-', \
">${backup_fn}"] ];
1781 run_command
($cmd, 'errmsg' => "cannot backup old database\n");
1783 my $maxfiles = 10; # purge older backup
1784 my $backups = [ sort { $b cmp $a } <$dbbackupdir/config-*.sql
.gz
> ];
1786 if ((my $count = scalar(@$backups)) > $maxfiles) {
1787 foreach my $f (@$backups[$maxfiles..$count-1]) {
1788 next if $f !~ m/^(\S+)$/; # untaint
1789 print "delete old backup '$1'\n";
1798 my $nodename = PVE
::INotify
::nodename
();
1800 setup_sshd_config
();
1801 setup_rootsshconfig
();
1804 # check if we can join with the given parameters and current node state
1805 my ($ring0_addr, $ring1_addr) = $param->@{'ring0_addr', 'ring1_addr'};
1806 assert_joinable
($ring0_addr, $ring1_addr, $param->{force
});
1808 # make sure known_hosts is on local filesystem
1809 ssh_unmerge_known_hosts
();
1811 my $host = $param->{hostname
};
1812 my $local_ip_address = remote_node_ip
($nodename);
1815 username
=> 'root@pam',
1816 password
=> $param->{password
},
1817 cookie_name
=> 'PVEAuthCookie',
1818 protocol
=> 'https',
1823 if (my $fp = $param->{fingerprint
}) {
1824 $conn_args->{cached_fingerprints
} = { uc($fp) => 1 };
1826 # API schema ensures that we can only get here from CLI handler
1827 $conn_args->{manual_verification
} = 1;
1830 print "Establishing API connection with host '$host'\n";
1832 my $conn = PVE
::APIClient
::LWP-
>new(%$conn_args);
1835 # login raises an exception on failure, so if we get here we're good
1836 print "Login succeeded.\n";
1839 $args->{force
} = $param->{force
} if defined($param->{force
});
1840 $args->{nodeid
} = $param->{nodeid
} if $param->{nodeid
};
1841 $args->{votes
} = $param->{votes
} if defined($param->{votes
});
1842 $args->{ring0_addr
} = $ring0_addr // $local_ip_address;
1843 $args->{ring1_addr
} = $ring1_addr if defined($ring1_addr);
1845 print "Request addition of this node\n";
1846 my $res = $conn->post("/cluster/config/nodes/$nodename", $args);
1848 print "Join request OK, finishing setup locally\n";
1850 # added successfuly - now prepare local node
1851 finish_join
($nodename, $res->{corosync_conf
}, $res->{corosync_authkey
});
1855 my ($nodename, $corosync_conf, $corosync_authkey) = @_;
1857 mkdir "$localclusterdir";
1858 PVE
::Tools
::file_set_contents
($authfile, $corosync_authkey);
1859 PVE
::Tools
::file_set_contents
($localclusterconf, $corosync_conf);
1861 print "stopping pve-cluster service\n";
1862 my $cmd = ['systemctl', 'stop', 'pve-cluster'];
1863 run_command
($cmd, errmsg
=> "can't stop pve-cluster service");
1865 $backup_cfs_database->($dbfile);
1868 $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
1869 run_command
($cmd, errmsg
=> "starting pve-cluster failed");
1873 while (!check_cfs_quorum
(1)) {
1875 print "waiting for quorum...";
1881 print "OK\n" if !$printqmsg;
1883 updatecerts_and_ssh
(1);
1885 print "generated new node certificate, restart pveproxy and pvedaemon services\n";
1886 run_command
(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
1888 print "successfully added node '$nodename' to cluster.\n";
1891 sub updatecerts_and_ssh
{
1892 my ($force_new_cert, $silent) = @_;
1894 my $p = sub { print "$_[0]\n" if !$silent };
1896 setup_rootsshconfig
();
1898 gen_pve_vzdump_symlink
();
1900 if (!check_cfs_quorum
(1)) {
1901 return undef if $silent;
1902 die "no quorum - unable to update files\n";
1907 my $nodename = PVE
::INotify
::nodename
();
1908 my $local_ip_address = remote_node_ip
($nodename);
1910 $p->("(re)generate node files");
1911 $p->("generate new node certificate") if $force_new_cert;
1912 gen_pve_node_files
($nodename, $local_ip_address, $force_new_cert);
1914 $p->("merge authorized SSH keys and known hosts");
1916 ssh_merge_known_hosts
($nodename, $local_ip_address, 1);
1917 gen_pve_vzdump_files
();