14 use POSIX
qw(EEXIST ENOENT);
17 use Storable
qw(dclone);
25 use PVE
::Tools
qw(run_command);
27 use PVE
::Cluster
::IPCConst
;
37 use Data
::Dumper
; # fixme: remove
39 # x509 certificate utils
41 my $basedir = "/etc/pve";
42 my $authdir = "$basedir/priv";
43 my $lockdir = "/etc/pve/priv/lock";
45 # cfs and corosync files
46 my $dbfile = "/var/lib/pve-cluster/config.db";
47 my $dbbackupdir = "/var/lib/pve-cluster/backup";
48 my $localclusterdir = "/etc/corosync";
49 my $localclusterconf = "$localclusterdir/corosync.conf";
50 my $authfile = "$localclusterdir/authkey";
51 my $clusterconf = "$basedir/corosync.conf";
53 my $authprivkeyfn = "$authdir/authkey.key";
54 my $authpubkeyfn = "$basedir/authkey.pub";
55 my $pveca_key_fn = "$authdir/pve-root-ca.key";
56 my $pveca_srl_fn = "$authdir/pve-root-ca.srl";
57 my $pveca_cert_fn = "$basedir/pve-root-ca.pem";
58 # this is just a secret accessable by the web browser
59 # and is used for CSRF prevention
60 my $pvewww_key_fn = "$basedir/pve-www.key";
63 my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
64 my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
65 my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
66 my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
67 my $sshknownhosts = "/etc/pve/priv/known_hosts";
68 my $sshauthkeys = "/etc/pve/priv/authorized_keys";
69 my $sshd_config_fn = "/etc/ssh/sshd_config";
70 my $rootsshauthkeys = "/root/.ssh/authorized_keys";
71 my $rootsshauthkeysbackup = "${rootsshauthkeys}.org";
72 my $rootsshconfig = "/root/.ssh/config";
74 # this is just a readonly copy, the relevant one is in status.c from pmxcfs
75 # observed files are the one we can get directly through IPCC, they are cached
76 # using a computed version and only those can be used by the cfs_*_file methods
80 'datacenter.cfg' => 1,
81 'replication.cfg' => 1,
83 'corosync.conf.new' => 1,
86 'priv/shadow.cfg' => 1,
91 'ha/crm_commands' => 1,
92 'ha/manager_status' => 1,
93 'ha/resources.cfg' => 1,
100 # only write output if something fails
105 my $record = sub { $outbuf .= shift . "\n"; };
107 eval { run_command
($cmd, outfunc
=> $record, errfunc
=> $record) };
110 print STDERR
$outbuf;
115 sub check_cfs_quorum
{
118 # note: -w filename always return 1 for root, so wee need
119 # to use File::lstat here
120 my $st = File
::stat::lstat("$basedir/local");
121 my $quorate = ($st && (($st->mode & 0200) != 0));
123 die "cluster not ready - no quorum?\n" if !$quorate && !$noerr;
128 sub check_cfs_is_mounted
{
131 my $res = -l
"$basedir/local";
133 die "pve configuration filesystem not mounted\n"
142 check_cfs_is_mounted
();
144 my @required_dirs = (
147 "$basedir/nodes/$nodename",
148 "$basedir/nodes/$nodename/lxc",
149 "$basedir/nodes/$nodename/qemu-server",
150 "$basedir/nodes/$nodename/openvz",
151 "$basedir/nodes/$nodename/priv");
153 foreach my $dir (@required_dirs) {
155 mkdir($dir) || $! == EEXIST
|| die "unable to create directory '$dir' - $!\n";
162 return if -f
"$authprivkeyfn";
164 check_cfs_is_mounted
();
166 cfs_lock_authkey
(undef, sub {
167 mkdir $authdir || $! == EEXIST
|| die "unable to create dir '$authdir' - $!\n";
169 run_silent_cmd
(['openssl', 'genrsa', '-out', $authprivkeyfn, '2048']);
171 run_silent_cmd
(['openssl', 'rsa', '-in', $authprivkeyfn, '-pubout', '-out', $authpubkeyfn]);
179 return if -f
$pveca_key_fn;
182 run_silent_cmd
(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']);
185 die "unable to generate pve ca key:\n$@" if $@;
190 if (-f
$pveca_key_fn && -f
$pveca_cert_fn) {
196 # we try to generate an unique 'subject' to avoid browser problems
197 # (reused serial numbers, ..)
199 UUID
::generate
($uuid);
201 UUID
::unparse
($uuid, $uuid_str);
204 # wrap openssl with faketime to prevent bug #904
205 run_silent_cmd
(['faketime', 'yesterday', 'openssl', 'req', '-batch',
206 '-days', '3650', '-new', '-x509', '-nodes', '-key',
207 $pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
208 "/CN=Proxmox Virtual Environment/OU=$uuid_str/O=PVE Cluster Manager CA/"]);
211 die "generating pve root certificate failed:\n$@" if $@;
216 sub gen_pve_ssl_key
{
219 die "no node name specified" if !$nodename;
221 my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
223 return if -f
$pvessl_key_fn;
226 run_silent_cmd
(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
229 die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
232 sub gen_pve_www_key
{
234 return if -f
$pvewww_key_fn;
237 run_silent_cmd
(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
240 die "unable to generate pve www key:\n$@" if $@;
246 PVE
::Tools
::file_set_contents
($pveca_srl_fn, $serial);
249 sub gen_pve_ssl_cert
{
250 my ($force, $nodename, $ip) = @_;
252 die "no node name specified" if !$nodename;
253 die "no IP specified" if !$ip;
255 my $pvessl_cert_fn = "$basedir/nodes/$nodename/pve-ssl.pem";
257 return if !$force && -f
$pvessl_cert_fn;
259 my $names = "IP:127.0.0.1,IP:::1,DNS:localhost";
261 my $rc = PVE
::INotify
::read_file
('resolvconf');
265 my $fqdn = $nodename;
267 $names .= ",DNS:$nodename";
269 if ($rc && $rc->{search
}) {
270 $fqdn = $nodename . "." . $rc->{search
};
271 $names .= ",DNS:$fqdn";
274 my $sslconf = <<__EOD;
275 RANDFILE = /root/.rnd
280 distinguished_name = req_distinguished_name
281 req_extensions = v3_req
283 string_mask = nombstr
285 [ req_distinguished_name ]
286 organizationalUnitName = PVE Cluster Node
287 organizationName = Proxmox Virtual Environment
291 basicConstraints = CA:FALSE
292 extendedKeyUsage = serverAuth
293 subjectAltName = $names
296 my $cfgfn = "/tmp/pvesslconf-$$.tmp";
297 my $fh = IO
::File-
>new ($cfgfn, "w");
301 my $reqfn = "/tmp/pvecertreq-$$.tmp";
304 my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
306 run_silent_cmd
(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
307 '-key', $pvessl_key_fn, '-out', $reqfn]);
313 die "unable to generate pve certificate request:\n$err";
316 update_serial
("0000000000000000") if ! -f
$pveca_srl_fn;
319 # wrap openssl with faketime to prevent bug #904
320 run_silent_cmd
(['faketime', 'yesterday', 'openssl', 'x509', '-req',
321 '-in', $reqfn, '-days', '3650', '-out', $pvessl_cert_fn,
322 '-CAkey', $pveca_key_fn, '-CA', $pveca_cert_fn,
323 '-CAserial', $pveca_srl_fn, '-extfile', $cfgfn]);
329 die "unable to generate pve ssl certificate:\n$err";
336 sub gen_pve_node_files
{
337 my ($nodename, $ip, $opt_force) = @_;
339 gen_local_dirs
($nodename);
343 # make sure we have a (cluster wide) secret
344 # for CSRFR prevention
347 # make sure we have a (per node) private key
348 gen_pve_ssl_key
($nodename);
350 # make sure we have a CA
351 my $force = gen_pveca_cert
();
353 $force = 1 if $opt_force;
355 gen_pve_ssl_cert
($force, $nodename, $ip);
358 my $vzdump_cron_dummy = <<__EOD;
359 # cluster wide vzdump cron schedule
360 # Atomatically generated file - do not edit
362 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
366 sub gen_pve_vzdump_symlink
{
368 my $filename = "/etc/pve/vzdump.cron";
370 my $link_fn = "/etc/cron.d/vzdump";
372 if ((-f
$filename) && (! -l
$link_fn)) {
373 rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
374 symlink($filename, $link_fn);
378 sub gen_pve_vzdump_files
{
380 my $filename = "/etc/pve/vzdump.cron";
382 PVE
::Tools
::file_set_contents
($filename, $vzdump_cron_dummy)
385 gen_pve_vzdump_symlink
();
392 my $ipcc_send_rec = sub {
393 my ($msgid, $data) = @_;
395 my $res = PVE
::IPCC
::ipcc_send_rec
($msgid, $data);
397 die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
402 my $ipcc_send_rec_json = sub {
403 my ($msgid, $data) = @_;
405 my $res = PVE
::IPCC
::ipcc_send_rec
($msgid, $data);
407 die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
409 return decode_json
($res);
412 my $ipcc_get_config = sub {
415 my $bindata = pack "Z*", $path;
416 my $res = PVE
::IPCC
::ipcc_send_rec
(CFS_IPC_GET_CONFIG
, $bindata);
417 if (!defined($res)) {
419 return undef if $! == ENOENT
;
428 my $ipcc_get_status = sub {
429 my ($name, $nodename) = @_;
431 my $bindata = pack "Z[256]Z[256]", $name, ($nodename || "");
432 return PVE
::IPCC
::ipcc_send_rec
(CFS_IPC_GET_STATUS
, $bindata);
435 my $ipcc_update_status = sub {
436 my ($name, $data) = @_;
438 my $raw = ref($data) ? encode_json
($data) : $data;
440 my $bindata = pack "Z[256]Z*", $name, $raw;
442 return &$ipcc_send_rec(CFS_IPC_SET_STATUS
, $bindata);
446 my ($priority, $ident, $tag, $msg) = @_;
448 my $bindata = pack "CCCZ*Z*Z*", $priority, bytes
::length($ident) + 1,
449 bytes
::length($tag) + 1, $ident, $tag, $msg;
451 return &$ipcc_send_rec(CFS_IPC_LOG_CLUSTER_MSG
, $bindata);
454 my $ipcc_get_cluster_log = sub {
455 my ($user, $max) = @_;
457 $max = 0 if !defined($max);
459 my $bindata = pack "VVVVZ*", $max, 0, 0, 0, ($user || "");
460 return &$ipcc_send_rec(CFS_IPC_GET_CLUSTER_LOG
, $bindata);
468 my $res = &$ipcc_send_rec_json(CFS_IPC_GET_FS_VERSION
);
469 #warn "GOT1: " . Dumper($res);
470 die "no starttime\n" if !$res->{starttime
};
472 if (!$res->{starttime
} || !$versions->{starttime
} ||
473 $res->{starttime
} != $versions->{starttime
}) {
474 #print "detected changed starttime\n";
493 if (!$clinfo->{version
} || $clinfo->{version
} != $versions->{clinfo
}) {
494 #warn "detected new clinfo\n";
495 $clinfo = &$ipcc_send_rec_json(CFS_IPC_GET_CLUSTER_INFO
);
506 if (!$vmlist->{version
} || $vmlist->{version
} != $versions->{vmlist
}) {
507 #warn "detected new vmlist1\n";
508 $vmlist = &$ipcc_send_rec_json(CFS_IPC_GET_GUEST_LIST
);
528 return $clinfo->{nodelist
};
532 my $nodelist = $clinfo->{nodelist
};
534 my $nodename = PVE
::INotify
::nodename
();
536 if (!$nodelist || !$nodelist->{$nodename}) {
537 return [ $nodename ];
540 return [ keys %$nodelist ];
543 # $data must be a chronological descending ordered array of tasks
544 sub broadcast_tasklist
{
547 # the serialized list may not get bigger than 32kb (CFS_MAX_STATUS_SIZE
548 # from pmxcfs) - drop older items until we satisfy this constraint
549 my $size = length(encode_json
($data));
550 while ($size >= (32 * 1024)) {
552 $size = length(encode_json
($data));
556 &$ipcc_update_status("tasklist", $data);
562 my $tasklistcache = {};
567 my $kvstore = $versions->{kvstore
} || {};
569 my $nodelist = get_nodelist
();
572 foreach my $node (@$nodelist) {
573 next if $nodename && ($nodename ne $node);
575 my $ver = $kvstore->{$node}->{tasklist
} if $kvstore->{$node};
576 my $cd = $tasklistcache->{$node};
577 if (!$cd || !$ver || !$cd->{version
} ||
578 ($cd->{version
} != $ver)) {
579 my $raw = &$ipcc_get_status("tasklist", $node) || '[]';
580 my $data = decode_json
($raw);
582 $cd = $tasklistcache->{$node} = {
586 } elsif ($cd && $cd->{data
}) {
587 push @$res, @{$cd->{data
}};
591 syslog
('err', $err) if $err;
598 my ($rrdid, $data) = @_;
601 &$ipcc_update_status("rrd/$rrdid", $data);
608 my $last_rrd_dump = 0;
609 my $last_rrd_data = "";
615 my $diff = $ctime - $last_rrd_dump;
617 return $last_rrd_data;
622 $raw = &$ipcc_send_rec(CFS_IPC_GET_RRD_DUMP
);
634 while ($raw =~ s/^(.*)\n//) {
635 my ($key, @ela) = split(/:/, $1);
637 next if !(scalar(@ela) > 1);
638 $res->{$key} = [ map { $_ eq 'U' ?
undef : $_ } @ela ];
642 $last_rrd_dump = $ctime;
643 $last_rrd_data = $res;
648 sub create_rrd_data
{
649 my ($rrdname, $timeframe, $cf) = @_;
651 my $rrddir = "/var/lib/rrdcached/db";
653 my $rrd = "$rrddir/$rrdname";
657 day
=> [ 60*30, 70 ],
658 week
=> [ 60*180, 70 ],
659 month
=> [ 60*720, 70 ],
660 year
=> [ 60*10080, 70 ],
663 my ($reso, $count) = @{$setup->{$timeframe}};
664 my $ctime = $reso*int(time()/$reso);
665 my $req_start = $ctime - $reso*$count;
667 $cf = "AVERAGE" if !$cf;
675 my $socket = "/var/run/rrdcached.sock";
676 push @args, "--daemon" => "unix:$socket" if -S
$socket;
678 my ($start, $step, $names, $data) = RRDs
::fetch
($rrd, $cf, @args);
680 my $err = RRDs
::error
;
681 die "RRD error: $err\n" if $err;
683 die "got wrong time resolution ($step != $reso)\n"
687 my $fields = scalar(@$names);
688 for my $line (@$data) {
689 my $entry = { 'time' => $start };
691 for (my $i = 0; $i < $fields; $i++) {
692 my $name = $names->[$i];
693 if (defined(my $val = $line->[$i])) {
694 $entry->{$name} = $val;
696 # leave empty fields undefined
697 # maybe make this configurable?
706 sub create_rrd_graph
{
707 my ($rrdname, $timeframe, $ds, $cf) = @_;
709 # Using RRD graph is clumsy - maybe it
710 # is better to simply fetch the data, and do all display
711 # related things with javascript (new extjs html5 graph library).
713 my $rrddir = "/var/lib/rrdcached/db";
715 my $rrd = "$rrddir/$rrdname";
717 my @ids = PVE
::Tools
::split_list
($ds);
719 my $ds_txt = join('_', @ids);
721 my $filename = "${rrd}_${ds_txt}.png";
725 day
=> [ 60*30, 70 ],
726 week
=> [ 60*180, 70 ],
727 month
=> [ 60*720, 70 ],
728 year
=> [ 60*10080, 70 ],
731 my ($reso, $count) = @{$setup->{$timeframe}};
734 "--imgformat" => "PNG",
738 "--start" => - $reso*$count,
740 "--lower-limit" => 0,
743 my $socket = "/var/run/rrdcached.sock";
744 push @args, "--daemon" => "unix:$socket" if -S
$socket;
746 my @coldef = ('#00ddff', '#ff0000');
748 $cf = "AVERAGE" if !$cf;
751 foreach my $id (@ids) {
752 my $col = $coldef[$i++] || die "fixme: no color definition";
753 push @args, "DEF:${id}=$rrd:${id}:$cf";
755 if ($id eq 'cpu' || $id eq 'iowait') {
756 push @args, "CDEF:${id}_per=${id},100,*";
757 $dataid = "${id}_per";
759 push @args, "LINE2:${dataid}${col}:${id}";
762 push @args, '--full-size-mode';
764 # we do not really store data into the file
765 my $res = RRDs
::graphv
('-', @args);
767 my $err = RRDs
::error
;
768 die "RRD error: $err\n" if $err;
770 return { filename
=> $filename, image
=> $res->{image
} };
773 # a fast way to read files (avoid fuse overhead)
777 return &$ipcc_get_config($path);
780 sub get_cluster_log
{
781 my ($user, $max) = @_;
783 return &$ipcc_get_cluster_log($user, $max);
788 sub cfs_register_file
{
789 my ($filename, $parser, $writer) = @_;
791 $observed->{$filename} || die "unknown file '$filename'";
793 die "file '$filename' already registered" if $file_info->{$filename};
795 $file_info->{$filename} = {
801 my $ccache_read = sub {
802 my ($filename, $parser, $version) = @_;
804 $ccache->{$filename} = {} if !$ccache->{$filename};
806 my $ci = $ccache->{$filename};
808 if (!$ci->{version
} || !$version || $ci->{version
} != $version) {
809 # we always call the parser, even when the file does not exists
810 # (in that case $data is undef)
811 my $data = get_config
($filename);
812 $ci->{data
} = &$parser("/etc/pve/$filename", $data);
813 $ci->{version
} = $version;
816 my $res = ref($ci->{data
}) ? dclone
($ci->{data
}) : $ci->{data
};
821 sub cfs_file_version
{
826 if ($filename =~ m!^nodes/[^/]+/(openvz|lxc|qemu-server)/(\d+)\.conf$!) {
827 my ($type, $vmid) = ($1, $2);
828 if ($vmlist && $vmlist->{ids
} && $vmlist->{ids
}->{$vmid}) {
829 $version = $vmlist->{ids
}->{$vmid}->{version
};
831 $infotag = "/$type/";
833 $infotag = $filename;
834 $version = $versions->{$filename};
837 my $info = $file_info->{$infotag} ||
838 die "unknown file type '$filename'\n";
840 return wantarray ?
($version, $info) : $version;
846 my ($version, $info) = cfs_file_version
($filename);
847 my $parser = $info->{parser
};
849 return &$ccache_read($filename, $parser, $version);
853 my ($filename, $data) = @_;
855 my ($version, $info) = cfs_file_version
($filename);
857 my $writer = $info->{writer
} || die "no writer defined";
859 my $fsname = "/etc/pve/$filename";
861 my $raw = &$writer($fsname, $data);
863 if (my $ci = $ccache->{$filename}) {
864 $ci->{version
} = undef;
867 PVE
::Tools
::file_set_contents
($fsname, $raw);
871 my ($lockid, $timeout, $code, @param) = @_;
873 my $prev_alarm = alarm(0); # suspend outer alarm early
878 # this timeout is for acquire the lock
879 $timeout = 10 if !$timeout;
881 my $filename = "$lockdir/$lockid";
888 die "pve cluster filesystem not online.\n";
891 my $timeout_err = sub { die "got lock request timeout\n"; };
892 local $SIG{ALRM
} = $timeout_err;
896 $got_lock = mkdir($filename);
897 $timeout = alarm(0) - 1; # we'll sleep for 1s, see down below
901 $timeout_err->() if $timeout <= 0;
903 print STDERR
"trying to acquire cfs lock '$lockid' ...\n";
904 utime (0, 0, $filename); # cfs unlock request
908 # fixed command timeout: cfs locks have a timeout of 120
909 # using 60 gives us another 60 seconds to abort the task
910 local $SIG{ALRM
} = sub { die "got lock timeout - aborting command\n"; };
913 cfs_update
(); # make sure we read latest versions inside code()
915 $res = &$code(@param);
922 $err = "no quorum!\n" if !$got_lock && !check_cfs_quorum
(1);
924 rmdir $filename if $got_lock; # if we held the lock always unlock again
929 $@ = "error with cfs lock '$lockid': $err";
939 my ($filename, $timeout, $code, @param) = @_;
941 my $info = $observed->{$filename} || die "unknown file '$filename'";
943 my $lockid = "file-$filename";
944 $lockid =~ s/[.\/]/_
/g
;
946 &$cfs_lock($lockid, $timeout, $code, @param);
949 sub cfs_lock_storage
{
950 my ($storeid, $timeout, $code, @param) = @_;
952 my $lockid = "storage-$storeid";
954 &$cfs_lock($lockid, $timeout, $code, @param);
957 sub cfs_lock_domain
{
958 my ($domainname, $timeout, $code, @param) = @_;
960 my $lockid = "domain-$domainname";
962 &$cfs_lock($lockid, $timeout, $code, @param);
966 my ($account, $timeout, $code, @param) = @_;
968 my $lockid = "acme-$account";
970 &$cfs_lock($lockid, $timeout, $code, @param);
973 sub cfs_lock_authkey
{
974 my ($timeout, $code, @param) = @_;
976 $cfs_lock->('authkey', $timeout, $code, @param);
994 my ($priority, $ident, $msg) = @_;
996 if (my $tmp = $log_levels->{$priority}) {
1000 die "need numeric log priority" if $priority !~ /^\d+$/;
1002 my $tag = PVE
::SafeSyslog
::tag
();
1004 $msg = "empty message" if !$msg;
1006 $ident = "" if !$ident;
1007 $ident = encode
("ascii", $ident,
1008 sub { sprintf "\\u%04x", shift });
1010 my $ascii = encode
("ascii", $msg, sub { sprintf "\\u%04x", shift });
1013 syslog
($priority, "<%s> %s", $ident, $ascii);
1015 syslog
($priority, "%s", $ascii);
1018 eval { &$ipcc_log($priority, $ident, $tag, $ascii); };
1020 syslog
("err", "writing cluster log failed: $@") if $@;
1023 sub check_vmid_unused
{
1024 my ($vmid, $noerr) = @_;
1026 my $vmlist = get_vmlist
();
1028 my $d = $vmlist->{ids
}->{$vmid};
1029 return 1 if !defined($d);
1031 return undef if $noerr;
1033 my $vmtypestr = $d->{type
} eq 'qemu' ?
'VM' : 'CT';
1034 die "$vmtypestr $vmid already exists on node '$d->{node}'\n";
1037 sub check_node_exists
{
1038 my ($nodename, $noerr) = @_;
1040 my $nodelist = $clinfo->{nodelist
};
1041 return 1 if $nodelist && $nodelist->{$nodename};
1043 return undef if $noerr;
1045 die "no such cluster node '$nodename'\n";
1048 # this is also used to get the IP of the local node
1049 sub remote_node_ip
{
1050 my ($nodename, $noerr) = @_;
1052 my $nodelist = $clinfo->{nodelist
};
1053 if ($nodelist && $nodelist->{$nodename}) {
1054 if (my $ip = $nodelist->{$nodename}->{ip
}) {
1055 return $ip if !wantarray;
1056 my $family = $nodelist->{$nodename}->{address_family
};
1058 $nodelist->{$nodename}->{address_family
} =
1060 PVE
::Tools
::get_host_address_family
($ip);
1062 return wantarray ?
($ip, $family) : $ip;
1066 # fallback: try to get IP by other means
1067 return PVE
::Network
::get_ip_from_hostname
($nodename, $noerr);
1070 sub get_local_migration_ip
{
1071 my ($migration_network, $noerr) = @_;
1073 my $cidr = $migration_network;
1075 if (!defined($cidr)) {
1076 my $dc_conf = cfs_read_file
('datacenter.cfg');
1077 $cidr = $dc_conf->{migration
}->{network
}
1078 if defined($dc_conf->{migration
}->{network
});
1081 if (defined($cidr)) {
1082 my $ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
1084 die "could not get migration ip: no IP address configured on local " .
1085 "node for network '$cidr'\n" if !$noerr && (scalar(@$ips) == 0);
1087 die "could not get migration ip: multiple IP address configured for " .
1088 "network '$cidr'\n" if !$noerr && (scalar(@$ips) > 1);
1096 # ssh related utility functions
1098 sub ssh_merge_keys
{
1099 # remove duplicate keys in $sshauthkeys
1100 # ssh-copy-id simply add keys, so the file can grow to large
1103 if (-f
$sshauthkeys) {
1104 $data = PVE
::Tools
::file_get_contents
($sshauthkeys, 128*1024);
1109 if (-f
$rootsshauthkeysbackup) {
1111 $data .= PVE
::Tools
::file_get_contents
($rootsshauthkeysbackup, 128*1024);
1116 # always add ourself
1117 if (-f
$ssh_rsa_id) {
1118 my $pub = PVE
::Tools
::file_get_contents
($ssh_rsa_id);
1120 $data .= "\n$pub\n";
1125 my @lines = split(/\n/, $data);
1126 foreach my $line (@lines) {
1127 if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
1128 next if $vhash->{$3}++;
1130 $newdata .= "$line\n";
1133 PVE
::Tools
::file_set_contents
($sshauthkeys, $newdata, 0600);
1135 if ($found_backup && -l
$rootsshauthkeys) {
1136 # everything went well, so we can remove the backup
1137 unlink $rootsshauthkeysbackup;
1141 sub setup_sshd_config
{
1144 my $conf = PVE
::Tools
::file_get_contents
($sshd_config_fn);
1146 return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
1148 if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
1150 $conf .= "\nPermitRootLogin yes\n";
1153 PVE
::Tools
::file_set_contents
($sshd_config_fn, $conf);
1155 PVE
::Tools
::run_command
(['systemctl', 'reload-or-restart', 'sshd']);
1158 sub setup_rootsshconfig
{
1160 # create ssh key if it does not exist
1161 if (! -f
$ssh_rsa_id) {
1162 mkdir '/root/.ssh/';
1163 system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
1166 # create ssh config if it does not exist
1167 if (! -f
$rootsshconfig) {
1169 if (my $fh = IO
::File-
>new($rootsshconfig, O_CREAT
|O_WRONLY
|O_EXCL
, 0640)) {
1170 # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k 26 Jan 2017)
1171 # changed order to put AES before Chacha20 (most hardware has AESNI)
1172 print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
1178 sub setup_ssh_keys
{
1184 if (! -f
$sshauthkeys) {
1186 if (-f
$rootsshauthkeys) {
1187 $old = PVE
::Tools
::file_get_contents
($rootsshauthkeys, 128*1024);
1189 if (my $fh = IO
::File-
>new ($sshauthkeys, O_CREAT
|O_WRONLY
|O_EXCL
, 0400)) {
1190 PVE
::Tools
::safe_print
($sshauthkeys, $fh, $old) if $old;
1196 warn "can't create shared ssh key database '$sshauthkeys'\n"
1197 if ! -f
$sshauthkeys;
1199 if (-f
$rootsshauthkeys && ! -l
$rootsshauthkeys) {
1200 if (!rename($rootsshauthkeys , $rootsshauthkeysbackup)) {
1201 warn "rename $rootsshauthkeys failed - $!\n";
1205 if (! -l
$rootsshauthkeys) {
1206 symlink $sshauthkeys, $rootsshauthkeys;
1209 if (! -l
$rootsshauthkeys) {
1210 warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n";
1212 unlink $rootsshauthkeysbackup if $import_ok;
1216 sub ssh_unmerge_known_hosts
{
1217 return if ! -l
$sshglobalknownhosts;
1220 $old = PVE
::Tools
::file_get_contents
($sshknownhosts, 128*1024)
1221 if -f
$sshknownhosts;
1223 PVE
::Tools
::file_set_contents
($sshglobalknownhosts, $old);
1226 sub ssh_merge_known_hosts
{
1227 my ($nodename, $ip_address, $createLink) = @_;
1229 die "no node name specified" if !$nodename;
1230 die "no ip address specified" if !$ip_address;
1232 # ssh lowercases hostnames (aliases) before comparision, so we need too
1233 $nodename = lc($nodename);
1234 $ip_address = lc($ip_address);
1238 if (! -f
$sshknownhosts) {
1239 if (my $fh = IO
::File-
>new($sshknownhosts, O_CREAT
|O_WRONLY
|O_EXCL
, 0600)) {
1244 my $old = PVE
::Tools
::file_get_contents
($sshknownhosts, 128*1024);
1248 if ((! -l
$sshglobalknownhosts) && (-f
$sshglobalknownhosts)) {
1249 $new = PVE
::Tools
::file_get_contents
($sshglobalknownhosts, 128*1024);
1252 my $hostkey = PVE
::Tools
::file_get_contents
($ssh_host_rsa_id);
1253 # Note: file sometimes containe emty lines at start, so we use multiline match
1254 die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
1263 my $merge_line = sub {
1264 my ($line, $all) = @_;
1266 return if $line =~ m/^\s*$/; # skip empty lines
1267 return if $line =~ m/^#/; # skip comments
1269 if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
1272 if (!$vhash->{$key}) {
1274 if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
1275 my $salt = decode_base64
($1);
1277 my $hmac = Digest
::HMAC_SHA1-
>new($salt);
1278 $hmac->add($nodename);
1279 my $hd = $hmac->b64digest . '=';
1280 if ($digest eq $hd) {
1281 if ($rsakey eq $hostkey) {
1282 $found_nodename = 1;
1287 $hmac = Digest
::HMAC_SHA1-
>new($salt);
1288 $hmac->add($ip_address);
1289 $hd = $hmac->b64digest . '=';
1290 if ($digest eq $hd) {
1291 if ($rsakey eq $hostkey) {
1292 $found_local_ip = 1;
1298 $key = lc($key); # avoid duplicate entries, ssh compares lowercased
1299 if ($key eq $ip_address) {
1300 $found_local_ip = 1 if $rsakey eq $hostkey;
1301 } elsif ($key eq $nodename) {
1302 $found_nodename = 1 if $rsakey eq $hostkey;
1312 while ($old && $old =~ s/^((.*?)(\n|$))//) {
1314 &$merge_line($line, 1);
1317 while ($new && $new =~ s/^((.*?)(\n|$))//) {
1319 &$merge_line($line);
1322 # add our own key if not already there
1323 $data .= "$nodename $hostkey\n" if !$found_nodename;
1324 $data .= "$ip_address $hostkey\n" if !$found_local_ip;
1326 PVE
::Tools
::file_set_contents
($sshknownhosts, $data);
1328 return if !$createLink;
1330 unlink $sshglobalknownhosts;
1331 symlink $sshknownhosts, $sshglobalknownhosts;
1333 warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
1334 if ! -l
$sshglobalknownhosts;
1338 my $migration_format = {
1342 enum
=> ['secure', 'insecure'],
1343 description
=> "Migration traffic is encrypted using an SSH tunnel by " .
1344 "default. On secure, completely private networks this can be " .
1345 "disabled to increase performance.",
1346 default => 'secure',
1350 type
=> 'string', format
=> 'CIDR',
1351 format_description
=> 'CIDR',
1352 description
=> "CIDR of the (sub) network that is used for migration."
1357 shutdown_policy
=> {
1359 enum
=> ['freeze', 'failover', 'conditional'],
1360 description
=> "The policy for HA services on node shutdown. 'freeze' disables auto-recovery, 'failover' ensures recovery, 'conditional' recovers on poweroff and freezes on reboot. Running HA Services will always get stopped first on shutdown.",
1361 verbose_description
=> "Describes the policy for handling HA services on poweroff or reboot of a node. Freeze will always freeze services which are still located on the node on shutdown, those services won't be recovered by the HA manager. Failover will not mark the services as frozen and thus the services will get recovered to other nodes, if the shutdown node does not come up again quickly (< 1min). 'conditional' chooses automatically depending on the type of shutdown, i.e., on a reboot the service will be frozen but on a poweroff the service will stay as is, and thus get recovered after about 2 minutes.",
1362 default => 'conditional',
1366 PVE
::JSONSchema
::register_format
('mac-prefix', \
&pve_verify_mac_prefix
);
1367 sub pve_verify_mac_prefix
{
1368 my ($mac_prefix, $noerr) = @_;
1370 if ($mac_prefix !~ m/^[a-f0-9][02468ace](?::[a-f0-9]{2}){0,2}:?$/i) {
1371 return undef if $noerr;
1372 die "value is not a valid unicast MAC address prefix\n";
1380 description
=> "U2F AppId URL override. Defaults to the origin.",
1381 format_description
=> 'APPID',
1386 description
=> "U2F Origin override. Mostly useful for single nodes with a single URL.",
1387 format_description
=> 'URL',
1392 my $datacenter_schema = {
1394 additionalProperties
=> 0,
1399 description
=> "Default keybord layout for vnc server.",
1400 enum
=> PVE
::Tools
::kvmkeymaplist
(),
1405 description
=> "Default GUI language.",
1431 description
=> "Specify external http proxy which is used for downloads (example: 'http://username:password\@host:port/')",
1432 pattern
=> "http://.*",
1434 migration_unsecure
=> {
1437 description
=> "Migration is secure using SSH tunnel by default. " .
1438 "For secure private networks you can disable it to speed up " .
1439 "migration. Deprecated, use the 'migration' property instead!",
1443 type
=> 'string', format
=> $migration_format,
1444 description
=> "For cluster wide migration settings.",
1449 description
=> "Select the default Console viewer. You can either use the builtin java applet (VNC; deprecated and maps to html5), an external virt-viewer comtatible application (SPICE), an HTML5 based vnc viewer (noVNC), or an HTML5 based console client (xtermjs). If the selected viewer is not available (e.g. SPICE not activated for the VM), the fallback is noVNC.",
1450 enum
=> ['applet', 'vv', 'html5', 'xtermjs'],
1455 format
=> 'email-opt',
1456 description
=> "Specify email address to send notification from (default is root@\$hostname)",
1462 description
=> "Defines how many workers (per node) are maximal started ".
1463 " on actions like 'stopall VMs' or task from the ha-manager.",
1468 default => 'watchdog',
1469 enum
=> [ 'watchdog', 'hardware', 'both' ],
1470 description
=> "Set the fencing mode of the HA cluster. Hardware mode " .
1471 "needs a valid configuration of fence devices in /etc/pve/ha/fence.cfg." .
1472 " With both all two modes are used." .
1473 "\n\nWARNING: 'hardware' and 'both' are EXPERIMENTAL & WIP",
1477 type
=> 'string', format
=> $ha_format,
1478 description
=> "Cluster wide HA settings.",
1483 format
=> 'mac-prefix',
1484 description
=> 'Prefix for autogenerated MAC addresses.',
1486 bwlimit
=> PVE
::JSONSchema
::get_standard_option
('bwlimit'),
1490 format
=> $u2f_format,
1491 description
=> 'u2f',
1496 # make schema accessible from outside (for documentation)
1497 sub get_datacenter_schema
{ return $datacenter_schema };
1499 sub parse_datacenter_config
{
1500 my ($filename, $raw) = @_;
1502 my $res = PVE
::JSONSchema
::parse_config
($datacenter_schema, $filename, $raw // '');
1504 if (my $migration = $res->{migration
}) {
1505 $res->{migration
} = PVE
::JSONSchema
::parse_property_string
($migration_format, $migration);
1508 if (my $ha = $res->{ha
}) {
1509 $res->{ha
} = PVE
::JSONSchema
::parse_property_string
($ha_format, $ha);
1512 # for backwards compatibility only, new migration property has precedence
1513 if (defined($res->{migration_unsecure
})) {
1514 if (defined($res->{migration
}->{type
})) {
1515 warn "deprecated setting 'migration_unsecure' and new 'migration: type' " .
1516 "set at same time! Ignore 'migration_unsecure'\n";
1518 $res->{migration
}->{type
} = ($res->{migration_unsecure
}) ?
'insecure' : 'secure';
1522 # for backwards compatibility only, applet maps to html5
1523 if (defined($res->{console
}) && $res->{console
} eq 'applet') {
1524 $res->{console
} = 'html5';
1530 sub write_datacenter_config
{
1531 my ($filename, $cfg) = @_;
1533 # map deprecated setting to new one
1534 if (defined($cfg->{migration_unsecure
}) && !defined($cfg->{migration
})) {
1535 my $migration_unsecure = delete $cfg->{migration_unsecure
};
1536 $cfg->{migration
}->{type
} = ($migration_unsecure) ?
'insecure' : 'secure';
1539 # map deprecated applet setting to html5
1540 if (defined($cfg->{console
}) && $cfg->{console
} eq 'applet') {
1541 $cfg->{console
} = 'html5';
1544 if (ref($cfg->{migration
})) {
1545 my $migration = $cfg->{migration
};
1546 $cfg->{migration
} = PVE
::JSONSchema
::print_property_string
($migration, $migration_format);
1549 if (ref($cfg->{ha
})) {
1550 my $ha = $cfg->{ha
};
1551 $cfg->{ha
} = PVE
::JSONSchema
::print_property_string
($ha, $ha_format);
1554 return PVE
::JSONSchema
::dump_config
($datacenter_schema, $filename, $cfg);
1557 cfs_register_file
('datacenter.cfg',
1558 \
&parse_datacenter_config
,
1559 \
&write_datacenter_config
);
1561 # X509 Certificate cache helper
1563 my $cert_cache_nodes = {};
1564 my $cert_cache_timestamp = time();
1565 my $cert_cache_fingerprints = {};
1567 sub update_cert_cache
{
1568 my ($update_node, $clear) = @_;
1570 syslog
('info', "Clearing outdated entries from certificate cache")
1573 $cert_cache_timestamp = time() if !defined($update_node);
1575 my $node_list = defined($update_node) ?
1576 [ $update_node ] : [ keys %$cert_cache_nodes ];
1578 foreach my $node (@$node_list) {
1579 my $clear_old = sub {
1580 if (my $old_fp = $cert_cache_nodes->{$node}) {
1581 # distrust old fingerprint
1582 delete $cert_cache_fingerprints->{$old_fp};
1583 # ensure reload on next proxied request
1584 delete $cert_cache_nodes->{$node};
1588 my $fp = eval { get_node_fingerprint
($node) };
1591 &$clear_old() if $clear;
1595 my $old_fp = $cert_cache_nodes->{$node};
1596 $cert_cache_fingerprints->{$fp} = 1;
1597 $cert_cache_nodes->{$node} = $fp;
1599 if (defined($old_fp) && $fp ne $old_fp) {
1600 delete $cert_cache_fingerprints->{$old_fp};
1605 # load and cache cert fingerprint once
1606 sub initialize_cert_cache
{
1609 update_cert_cache
($node)
1610 if defined($node) && !defined($cert_cache_nodes->{$node});
1613 sub read_ssl_cert_fingerprint
{
1614 my ($cert_path) = @_;
1616 my $bio = Net
::SSLeay
::BIO_new_file
($cert_path, 'r')
1617 or die "unable to read '$cert_path' - $!\n";
1619 my $cert = Net
::SSLeay
::PEM_read_bio_X509
($bio);
1620 Net
::SSLeay
::BIO_free
($bio);
1622 die "unable to read certificate from '$cert_path'\n" if !$cert;
1624 my $fp = Net
::SSLeay
::X509_get_fingerprint
($cert, 'sha256');
1625 Net
::SSLeay
::X509_free
($cert);
1627 die "unable to get fingerprint for '$cert_path' - got empty value\n"
1628 if !defined($fp) || $fp eq '';
1633 sub get_node_fingerprint
{
1636 my $cert_path = "/etc/pve/nodes/$node/pve-ssl.pem";
1637 my $custom_cert_path = "/etc/pve/nodes/$node/pveproxy-ssl.pem";
1639 $cert_path = $custom_cert_path if -f
$custom_cert_path;
1641 return read_ssl_cert_fingerprint
($cert_path);
1645 sub check_cert_fingerprint
{
1648 # clear cache every 30 minutes at least
1649 update_cert_cache
(undef, 1) if time() - $cert_cache_timestamp >= 60*30;
1651 # get fingerprint of server certificate
1652 my $fp = Net
::SSLeay
::X509_get_fingerprint
($cert, 'sha256');
1653 return 0 if !defined($fp) || $fp eq ''; # error
1656 for my $expected (keys %$cert_cache_fingerprints) {
1657 return 1 if $fp eq $expected;
1662 return 1 if &$check();
1664 # clear cache and retry at most once every minute
1665 if (time() - $cert_cache_timestamp >= 60) {
1666 syslog
('info', "Could not verify remote node certificate '$fp' with list of pinned certificates, refreshing cache");
1667 update_cert_cache
();
1674 # bash completion helpers
1676 sub complete_next_vmid
{
1678 my $vmlist = get_vmlist
() || {};
1679 my $idlist = $vmlist->{ids
} || {};
1681 for (my $i = 100; $i < 10000; $i++) {
1682 return [$i] if !defined($idlist->{$i});
1690 my $vmlist = get_vmlist
();
1691 my $ids = $vmlist->{ids
} || {};
1693 return [ keys %$ids ];
1696 sub complete_local_vmid
{
1698 my $vmlist = get_vmlist
();
1699 my $ids = $vmlist->{ids
} || {};
1701 my $nodename = PVE
::INotify
::nodename
();
1704 foreach my $vmid (keys %$ids) {
1705 my $d = $ids->{$vmid};
1706 next if !$d->{node
} || $d->{node
} ne $nodename;
1713 sub complete_migration_target
{
1717 my $nodename = PVE
::INotify
::nodename
();
1719 my $nodelist = get_nodelist
();
1720 foreach my $node (@$nodelist) {
1721 next if $node eq $nodename;
1729 my ($node, $network_cidr) = @_;
1732 if (defined($network_cidr)) {
1733 # Use mtunnel via to get the remote node's ip inside $network_cidr.
1734 # This goes over the regular network (iow. uses get_ssh_info() with
1735 # $network_cidr undefined.
1736 # FIXME: Use the REST API client for this after creating an API entry
1737 # for get_migration_ip.
1738 my $default_remote = get_ssh_info
($node, undef);
1739 my $default_ssh = ssh_info_to_command
($default_remote);
1740 my $cmd =[@$default_ssh, 'pvecm', 'mtunnel',
1741 '-migration_network', $network_cidr,
1744 PVE
::Tools
::run_command
($cmd, outfunc
=> sub {
1747 die "internal error: unexpected output from mtunnel\n"
1749 if ($line =~ /^ip: '(.*)'$/) {
1752 die "internal error: bad output from mtunnel\n"
1756 die "failed to get ip for node '$node' in network '$network_cidr'\n"
1759 $ip = remote_node_ip
($node);
1765 network
=> $network_cidr,
1769 sub ssh_info_to_command_base
{
1770 my ($info, @extra_options) = @_;
1774 '-o', 'BatchMode=yes',
1775 '-o', 'HostKeyAlias='.$info->{name
},
1780 sub ssh_info_to_command
{
1781 my ($info, @extra_options) = @_;
1782 my $cmd = ssh_info_to_command_base
($info, @extra_options);
1783 push @$cmd, "root\@$info->{ip}";
1787 sub assert_joinable
{
1788 my ($ring0_addr, $ring1_addr, $force) = @_;
1791 my $error = sub { $errors .= "* $_[0]\n"; };
1794 $error->("authentication key '$authfile' already exists");
1797 if (-f
$clusterconf) {
1798 $error->("cluster config '$clusterconf' already exists");
1801 my $vmlist = get_vmlist
();
1802 if ($vmlist && $vmlist->{ids
} && scalar(keys %{$vmlist->{ids
}})) {
1803 $error->("this host already contains virtual guests");
1806 if (run_command
(['corosync-quorumtool', '-l'], noerr
=> 1, quiet
=> 1) == 0) {
1807 $error->("corosync is already running, is this node already in a cluster?!");
1810 # check if corosync ring IPs are configured on the current nodes interfaces
1811 my $check_ip = sub {
1812 my $ip = shift // return;
1814 if (!PVE
::JSONSchema
::pve_verify_ip
($ip, 1)) {
1816 eval { $ip = PVE
::Network
::get_ip_from_hostname
($host); };
1818 $error->("$logid: cannot use '$host': $@\n") ;
1823 my $cidr = (Net
::IP
::ip_is_ipv6
($ip)) ?
"$ip/128" : "$ip/32";
1824 my $configured_ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
1826 $error->("$logid: cannot use IP '$ip', it must be configured exactly once on local node!\n")
1827 if (scalar(@$configured_ips) != 1);
1830 $check_ip->($ring0_addr, 'ring0');
1831 $check_ip->($ring1_addr, 'ring1');
1834 warn "detected the following error(s):\n$errors";
1835 die "Check if node may join a cluster failed!\n" if !$force;
1839 # NOTE: filesystem must be offline here, no DB changes allowed
1840 my $backup_cfs_database = sub {
1846 my $backup_fn = "$dbbackupdir/config-$ctime.sql.gz";
1848 print "backup old database to '$backup_fn'\n";
1850 my $cmd = [ ['sqlite3', $dbfile, '.dump'], ['gzip', '-', \
">${backup_fn}"] ];
1851 run_command
($cmd, 'errmsg' => "cannot backup old database\n");
1853 my $maxfiles = 10; # purge older backup
1854 my $backups = [ sort { $b cmp $a } <$dbbackupdir/config-*.sql
.gz
> ];
1856 if ((my $count = scalar(@$backups)) > $maxfiles) {
1857 foreach my $f (@$backups[$maxfiles..$count-1]) {
1858 next if $f !~ m/^(\S+)$/; # untaint
1859 print "delete old backup '$1'\n";
1868 my $nodename = PVE
::INotify
::nodename
();
1870 setup_sshd_config
();
1871 setup_rootsshconfig
();
1874 # check if we can join with the given parameters and current node state
1875 my ($ring0_addr, $ring1_addr) = $param->@{'ring0_addr', 'ring1_addr'};
1876 assert_joinable
($ring0_addr, $ring1_addr, $param->{force
});
1878 # make sure known_hosts is on local filesystem
1879 ssh_unmerge_known_hosts
();
1881 my $host = $param->{hostname
};
1882 my $local_ip_address = remote_node_ip
($nodename);
1885 username
=> 'root@pam',
1886 password
=> $param->{password
},
1887 cookie_name
=> 'PVEAuthCookie',
1888 protocol
=> 'https',
1893 if (my $fp = $param->{fingerprint
}) {
1894 $conn_args->{cached_fingerprints
} = { uc($fp) => 1 };
1896 # API schema ensures that we can only get here from CLI handler
1897 $conn_args->{manual_verification
} = 1;
1900 print "Establishing API connection with host '$host'\n";
1902 my $conn = PVE
::APIClient
::LWP-
>new(%$conn_args);
1905 # login raises an exception on failure, so if we get here we're good
1906 print "Login succeeded.\n";
1909 $args->{force
} = $param->{force
} if defined($param->{force
});
1910 $args->{nodeid
} = $param->{nodeid
} if $param->{nodeid
};
1911 $args->{votes
} = $param->{votes
} if defined($param->{votes
});
1912 $args->{ring0_addr
} = $ring0_addr // $local_ip_address;
1913 $args->{ring1_addr
} = $ring1_addr if defined($ring1_addr);
1915 print "Request addition of this node\n";
1916 my $res = $conn->post("/cluster/config/nodes/$nodename", $args);
1918 print "Join request OK, finishing setup locally\n";
1920 # added successfuly - now prepare local node
1921 finish_join
($nodename, $res->{corosync_conf
}, $res->{corosync_authkey
});
1925 my ($nodename, $corosync_conf, $corosync_authkey) = @_;
1927 mkdir "$localclusterdir";
1928 PVE
::Tools
::file_set_contents
($authfile, $corosync_authkey);
1929 PVE
::Tools
::file_set_contents
($localclusterconf, $corosync_conf);
1931 print "stopping pve-cluster service\n";
1932 my $cmd = ['systemctl', 'stop', 'pve-cluster'];
1933 run_command
($cmd, errmsg
=> "can't stop pve-cluster service");
1935 $backup_cfs_database->($dbfile);
1938 $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
1939 run_command
($cmd, errmsg
=> "starting pve-cluster failed");
1943 while (!check_cfs_quorum
(1)) {
1945 print "waiting for quorum...";
1951 print "OK\n" if !$printqmsg;
1953 updatecerts_and_ssh
(1);
1955 print "generated new node certificate, restart pveproxy and pvedaemon services\n";
1956 run_command
(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
1958 print "successfully added node '$nodename' to cluster.\n";
1961 sub updatecerts_and_ssh
{
1962 my ($force_new_cert, $silent) = @_;
1964 my $p = sub { print "$_[0]\n" if !$silent };
1966 setup_rootsshconfig
();
1968 gen_pve_vzdump_symlink
();
1970 if (!check_cfs_quorum
(1)) {
1971 return undef if $silent;
1972 die "no quorum - unable to update files\n";
1977 my $nodename = PVE
::INotify
::nodename
();
1978 my $local_ip_address = remote_node_ip
($nodename);
1980 $p->("(re)generate node files");
1981 $p->("generate new node certificate") if $force_new_cert;
1982 gen_pve_node_files
($nodename, $local_ip_address, $force_new_cert);
1984 $p->("merge authorized SSH keys and known hosts");
1986 ssh_merge_known_hosts
($nodename, $local_ip_address, 1);
1987 gen_pve_vzdump_files
();