use strict;
use warnings;
-use POSIX qw(EEXIST);
+use POSIX qw(EEXIST ENOENT);
use File::stat qw();
use Socket;
use Storable qw(dclone);
use IO::File;
use MIME::Base64;
-use XML::Parser;
use Digest::SHA;
use Digest::HMAC_SHA1;
-use PVE::Tools;
+use Net::SSLeay;
+use PVE::Tools qw(run_command);
use PVE::INotify;
use PVE::IPCC;
use PVE::SafeSyslog;
use PVE::JSONSchema;
use PVE::Network;
+use PVE::Cluster::IPCConst;
use JSON;
use RRDs;
use Encode;
my $authdir = "$basedir/priv";
my $lockdir = "/etc/pve/priv/lock";
+# cfs and corosync files
+my $dbfile = "/var/lib/pve-cluster/config.db";
+my $dbbackupdir = "/var/lib/pve-cluster/backup";
+my $localclusterdir = "/etc/corosync";
+my $localclusterconf = "$localclusterdir/corosync.conf";
+my $authfile = "$localclusterdir/authkey";
+my $clusterconf = "$basedir/corosync.conf";
+
my $authprivkeyfn = "$authdir/authkey.key";
my $authpubkeyfn = "$basedir/authkey.pub";
my $pveca_key_fn = "$authdir/pve-root-ca.key";
'vzdump.cron' => 1,
'storage.cfg' => 1,
'datacenter.cfg' => 1,
+ 'replication.cfg' => 1,
'corosync.conf' => 1,
'corosync.conf.new' => 1,
'user.cfg' => 1,
my ($cmd) = @_;
my $outbuf = '';
+ my $record = sub { $outbuf .= shift . "\n"; };
- my $record_output = sub {
- $outbuf .= shift;
- $outbuf .= "\n";
- };
-
- eval {
- PVE::Tools::run_command($cmd, outfunc => $record_output,
- errfunc => $record_output);
- };
-
- my $err = $@;
+ eval { run_command($cmd, outfunc => $record, errfunc => $record) };
- if ($err) {
+ if (my $err = $@) {
print STDERR $outbuf;
die $err;
}
my @required_dirs = (
"$basedir/priv",
- "$basedir/nodes",
+ "$basedir/nodes",
"$basedir/nodes/$nodename",
"$basedir/nodes/$nodename/lxc",
"$basedir/nodes/$nodename/qemu-server",
"$basedir/nodes/$nodename/openvz",
"$basedir/nodes/$nodename/priv");
-
+
foreach my $dir (@required_dirs) {
if (! -d $dir) {
mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
my $rc = PVE::INotify::read_file('resolvconf');
$names .= ",IP:$ip";
-
+
my $fqdn = $nodename;
$names .= ",DNS:$nodename";
my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
- die "ipcc_send_rec failed: $!\n" if !defined($res) && ($! != 0);
+ die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
return $res;
};
my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
- die "ipcc_send_rec failed: $!\n" if !defined($res) && ($! != 0);
+ die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
return decode_json($res);
};
my ($path) = @_;
my $bindata = pack "Z*", $path;
- my $res = PVE::IPCC::ipcc_send_rec(6, $bindata);
+ my $res = PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_CONFIG, $bindata);
if (!defined($res)) {
- return undef if ($! != 0);
+ if ($! != 0) {
+ return undef if $! == ENOENT;
+ die "$!\n";
+ }
return '';
}
my ($name, $nodename) = @_;
my $bindata = pack "Z[256]Z[256]", $name, ($nodename || "");
- return PVE::IPCC::ipcc_send_rec(5, $bindata);
+ return PVE::IPCC::ipcc_send_rec(CFS_IPC_GET_STATUS, $bindata);
};
my $ipcc_update_status = sub {
# update status
my $bindata = pack "Z[256]Z*", $name, $raw;
- return &$ipcc_send_rec(4, $bindata);
+ return &$ipcc_send_rec(CFS_IPC_SET_STATUS, $bindata);
};
my $ipcc_log = sub {
my $bindata = pack "CCCZ*Z*Z*", $priority, bytes::length($ident) + 1,
bytes::length($tag) + 1, $ident, $tag, $msg;
- return &$ipcc_send_rec(7, $bindata);
+ return &$ipcc_send_rec(CFS_IPC_LOG_CLUSTER_MSG, $bindata);
};
my $ipcc_get_cluster_log = sub {
$max = 0 if !defined($max);
my $bindata = pack "VVVVZ*", $max, 0, 0, 0, ($user || "");
- return &$ipcc_send_rec(8, $bindata);
+ return &$ipcc_send_rec(CFS_IPC_GET_CLUSTER_LOG, $bindata);
};
my $ccache = {};
sub cfs_update {
+ my ($fail) = @_;
eval {
- my $res = &$ipcc_send_rec_json(1);
+ my $res = &$ipcc_send_rec_json(CFS_IPC_GET_FS_VERSION);
#warn "GOT1: " . Dumper($res);
die "no starttime\n" if !$res->{starttime};
$vmlist = {};
$clinfo = {};
$ccache = {};
+ die $err if $fail;
warn $err;
}
eval {
if (!$clinfo->{version} || $clinfo->{version} != $versions->{clinfo}) {
#warn "detected new clinfo\n";
- $clinfo = &$ipcc_send_rec_json(2);
+ $clinfo = &$ipcc_send_rec_json(CFS_IPC_GET_CLUSTER_INFO);
}
};
$err = $@;
if ($err) {
$clinfo = {};
+ die $err if $fail;
warn $err;
}
eval {
if (!$vmlist->{version} || $vmlist->{version} != $versions->{vmlist}) {
#warn "detected new vmlist1\n";
- $vmlist = &$ipcc_send_rec_json(3);
+ $vmlist = &$ipcc_send_rec_json(CFS_IPC_GET_GUEST_LIST);
}
};
$err = $@;
if ($err) {
$vmlist = {};
+ die $err if $fail;
warn $err;
}
}
return [ keys %$nodelist ];
}
+# $data must be a chronological descending ordered array of tasks
sub broadcast_tasklist {
my ($data) = @_;
+ # the serialized list may not get bigger than 32kb (CFS_MAX_STATUS_SIZE
+ # from pmxcfs) - drop older items until we satisfy this constraint
+ my $size = length(encode_json($data));
+ while ($size >= (32 * 1024)) {
+ pop @$data;
+ $size = length(encode_json($data));
+ }
+
eval {
&$ipcc_update_status("tasklist", $data);
};
eval {
my $ver = $kvstore->{$node}->{tasklist} if $kvstore->{$node};
my $cd = $tasklistcache->{$node};
- if (!$cd || !$ver || !$cd->{version} ||
+ if (!$cd || !$ver || !$cd->{version} ||
($cd->{version} != $ver)) {
my $raw = &$ipcc_get_status("tasklist", $node) || '[]';
my $data = decode_json($raw);
my $raw;
eval {
- $raw = &$ipcc_send_rec(10);
+ $raw = &$ipcc_send_rec(CFS_IPC_GET_RRD_DUMP);
};
my $err = $@;
my ($key, @ela) = split(/:/, $1);
next if !$key;
next if !(scalar(@ela) > 1);
- $res->{$key} = \@ela;
+ $res->{$key} = [ map { $_ eq 'U' ? undef : $_ } @ela ];
}
}
my $err = RRDs::error;
die "RRD error: $err\n" if $err;
-
- die "got wrong time resolution ($step != $reso)\n"
+
+ die "got wrong time resolution ($step != $reso)\n"
if $step != $reso;
my $res = [];
# Using RRD graph is clumsy - maybe it
# is better to simply fetch the data, and do all display
# related things with javascript (new extjs html5 graph library).
-
+
my $rrddir = "/var/lib/rrdcached/db";
my $rrd = "$rrddir/$rrdname";
push @args, '--full-size-mode';
# we do not really store data into the file
- my $res = RRDs::graphv('', @args);
+ my $res = RRDs::graphv('-', @args);
my $err = RRDs::error;
die "RRD error: $err\n" if $err;
sub cfs_read_file {
my ($filename) = @_;
- my ($version, $info) = cfs_file_version($filename);
+ my ($version, $info) = cfs_file_version($filename);
my $parser = $info->{parser};
return &$ccache_read($filename, $parser, $version);
sub cfs_write_file {
my ($filename, $data) = @_;
- my ($version, $info) = cfs_file_version($filename);
+ my ($version, $info) = cfs_file_version($filename);
my $writer = $info->{writer} || die "no writer defined";
my $cfs_lock = sub {
my ($lockid, $timeout, $code, @param) = @_;
+ my $prev_alarm = alarm(0); # suspend outer alarm early
+
my $res;
+ my $got_lock = 0;
- # this timeout is for aquire the lock
+ # this timeout is for acquire the lock
$timeout = 10 if !$timeout;
my $filename = "$lockdir/$lockid";
- my $msg = "can't aquire cfs lock '$lockid'";
-
eval {
mkdir $lockdir;
if (! -d $lockdir) {
- die "$msg: pve cluster filesystem not online.\n";
+ die "pve cluster filesystem not online.\n";
}
- local $SIG{ALRM} = sub { die "got lock request timeout\n"; };
+ my $timeout_err = sub { die "got lock request timeout\n"; };
+ local $SIG{ALRM} = $timeout_err;
- alarm ($timeout);
+ while (1) {
+ alarm ($timeout);
+ $got_lock = mkdir($filename);
+ $timeout = alarm(0) - 1; # we'll sleep for 1s, see down below
- if (!(mkdir $filename)) {
- print STDERR "trying to aquire cfs lock '$lockid' ...";
- while (1) {
- if (!(mkdir $filename)) {
- (utime 0, 0, $filename); # cfs unlock request
- } else {
- print STDERR " OK\n";
- last;
- }
- sleep(1);
- }
+ last if $got_lock;
+
+ $timeout_err->() if $timeout <= 0;
+
+ print STDERR "trying to acquire cfs lock '$lockid' ...\n";
+ utime (0, 0, $filename); # cfs unlock request
+ sleep(1);
}
# fixed command timeout: cfs locks have a timeout of 120
# using 60 gives us another 60 seconds to abort the task
- alarm(60);
local $SIG{ALRM} = sub { die "got lock timeout - aborting command\n"; };
+ alarm(60);
cfs_update(); # make sure we read latest versions inside code()
my $err = $@;
- alarm(0);
+ $err = "no quorum!\n" if !$got_lock && !check_cfs_quorum(1);
- if ($err && ($err eq "got lock request timeout\n") &&
- !check_cfs_quorum()){
- $err = "$msg: no quorum!\n";
- }
+ rmdir $filename if $got_lock; # if we held the lock always unlock again
- if (!$err || $err !~ /^got lock timeout -/) {
- rmdir $filename; # cfs unlock
- }
+ alarm($prev_alarm);
if ($err) {
- $@ = $err;
+ $@ = "error with cfs lock '$lockid': $err";
return undef;
}
&$cfs_lock($lockid, $timeout, $code, @param);
}
+sub cfs_lock_acme {
+ my ($account, $timeout, $code, @param) = @_;
+
+ my $lockid = "acme-$account";
+
+ &$cfs_lock($lockid, $timeout, $code, @param);
+}
+
my $log_levels = {
"emerg" => 0,
"alert" => 1,
sub check_vmid_unused {
my ($vmid, $noerr) = @_;
-
+
my $vmlist = get_vmlist();
my $d = $vmlist->{ids}->{$vmid};
return 1 if !defined($d);
-
+
return undef if $noerr;
my $vmtypestr = $d->{type} eq 'qemu' ? 'VM' : 'CT';
$family =
PVE::Tools::get_host_address_family($ip);
}
- return ($ip, $family);
+ return wantarray ? ($ip, $family) : $ip;
}
}
# fallback: try to get IP by other means
- my ($family, $packed_ip);
-
- eval {
- my @res = PVE::Tools::getaddrinfo_all($nodename);
- $family = $res[0]->{family};
- $packed_ip = (PVE::Tools::unpack_sockaddr_in46($res[0]->{addr}))[2];
- };
-
- if ($@) {
- die "hostname lookup failed:\n$@" if !$noerr;
- return undef;
- }
-
- my $ip = Socket::inet_ntop($family, $packed_ip);
- if ($ip =~ m/^127\.|^::1$/) {
- die "hostname lookup failed - got local IP address ($nodename = $ip)\n" if !$noerr;
- return undef;
- }
-
- return wantarray ? ($ip, $family) : $ip;
+ return PVE::Network::get_ip_from_hostname($nodename, $noerr);
}
sub get_local_migration_ip {
if (defined($cidr)) {
my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
- die "no IP address configured on local node for network '$cidr'\n"
- if !$noerr && (scalar(@$ips) == 0);
+ die "could not get migration ip: no IP address configured on local " .
+ "node for network '$cidr'\n" if !$noerr && (scalar(@$ips) == 0);
- die "multiple IP address configured for network '$cidr'\n"
- if !$noerr && (scalar(@$ips) > 1);
+ die "could not get migration ip: multiple IP address configured for " .
+ "network '$cidr'\n" if !$noerr && (scalar(@$ips) > 1);
return @$ips[0];
}
}
sub setup_sshd_config {
+ my () = @_;
my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
-
+
return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
chomp $conf;
$conf .= "\nPermitRootLogin yes\n";
- }
+ }
PVE::Tools::file_set_contents($sshd_config_fn, $conf);
if (! -f $rootsshconfig) {
mkdir '/root/.ssh';
if (my $fh = IO::File->new($rootsshconfig, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
- # this is the default ciphers list from debian openssl0.9.8 except blowfish is added as prefered
- print $fh "Ciphers blowfish-cbc,aes128-ctr,aes192-ctr,aes256-ctr,arcfour256,arcfour128,aes128-cbc,3des-cbc\n";
+ # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k 26 Jan 2017)
+ # changed order to put AES before Chacha20 (most hardware has AESNI)
+ print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
close($fh);
}
}
}
}
- warn "can't create shared ssh key database '$sshauthkeys'\n"
+ warn "can't create shared ssh key database '$sshauthkeys'\n"
if ! -f $sshauthkeys;
if (-f $rootsshauthkeys && ! -l $rootsshauthkeys) {
die "no node name specified" if !$nodename;
die "no ip address specified" if !$ip_address;
-
+
+ # ssh lowercases hostnames (aliases) before comparision, so we need too
+ $nodename = lc($nodename);
+ $ip_address = lc($ip_address);
+
mkdir $authdir;
if (! -f $sshknownhosts) {
}
}
- my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
-
+ my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
+
my $new = '';
-
+
if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
$new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
}
my $merge_line = sub {
my ($line, $all) = @_;
+ return if $line =~ m/^\s*$/; # skip empty lines
+ return if $line =~ m/^#/; # skip comments
+
if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
my $key = $1;
my $rsakey = $2;
}
return;
}
+ } else {
+ $key = lc($key); # avoid duplicate entries, ssh compares lowercased
+ if ($key eq $ip_address) {
+ $found_local_ip = 1 if $rsakey eq $hostkey;
+ } elsif ($key eq $nodename) {
+ $found_nodename = 1 if $rsakey eq $hostkey;
+ }
}
$data .= $line;
}
while ($old && $old =~ s/^((.*?)(\n|$))//) {
my $line = "$2\n";
- next if $line =~ m/^\s*$/; # skip empty lines
- next if $line =~ m/^#/; # skip comments
&$merge_line($line, 1);
}
while ($new && $new =~ s/^((.*?)(\n|$))//) {
my $line = "$2\n";
- next if $line =~ m/^\s*$/; # skip empty lines
- next if $line =~ m/^#/; # skip comments
&$merge_line($line);
}
- my $addIndex = $$;
- my $add_known_hosts_entry = sub {
- my ($name, $hostkey) = @_;
- $addIndex++;
- my $hmac = Digest::HMAC_SHA1->new("$addIndex" . time());
- my $b64salt = $hmac->b64digest . '=';
- $hmac = Digest::HMAC_SHA1->new(decode_base64($b64salt));
- $hmac->add($name);
- my $digest = $hmac->b64digest . '=';
- $data .= "|1|$b64salt|$digest $hostkey\n";
- };
-
- if (!$found_nodename || !$found_local_ip) {
- &$add_known_hosts_entry($nodename, $hostkey) if !$found_nodename;
- &$add_known_hosts_entry($ip_address, $hostkey) if !$found_local_ip;
- }
+ # add our own key if not already there
+ $data .= "$nodename $hostkey\n" if !$found_nodename;
+ $data .= "$ip_address $hostkey\n" if !$found_local_ip;
PVE::Tools::file_set_contents($sshknownhosts, $data);
unlink $sshglobalknownhosts;
symlink $sshknownhosts, $sshglobalknownhosts;
-
- warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
+
+ warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
if ! -l $sshglobalknownhosts;
}
+my $migration_format = {
+ type => {
+ default_key => 1,
+ type => 'string',
+ enum => ['secure', 'insecure'],
+ description => "Migration traffic is encrypted using an SSH tunnel by " .
+ "default. On secure, completely private networks this can be " .
+ "disabled to increase performance.",
+ default => 'secure',
+ },
+ network => {
+ optional => 1,
+ type => 'string', format => 'CIDR',
+ format_description => 'CIDR',
+ description => "CIDR of the (sub) network that is used for migration."
+ },
+};
+
my $datacenter_schema = {
type => "object",
additionalProperties => 0,
optional => 1,
type => 'string',
description => "Default GUI language.",
- enum => [ 'en', 'de' ],
+ enum => [
+ 'zh_CN',
+ 'zh_TW',
+ 'ca',
+ 'en',
+ 'eu',
+ 'fr',
+ 'de',
+ 'it',
+ 'es',
+ 'ja',
+ 'nb',
+ 'nn',
+ 'fa',
+ 'pl',
+ 'pt_BR',
+ 'ru',
+ 'sl',
+ 'sv',
+ 'tr',
+ ],
},
http_proxy => {
optional => 1,
migration_unsecure => {
optional => 1,
type => 'boolean',
- description => "Migration is secure using SSH tunnel by default. For secure private networks you can disable it to speed up migration.",
+ description => "Migration is secure using SSH tunnel by default. " .
+ "For secure private networks you can disable it to speed up " .
+ "migration. Deprecated, use the 'migration' property instead!",
+ },
+ migration => {
+ optional => 1,
+ type => 'string', format => $migration_format,
+ description => "For cluster wide migration settings.",
},
console => {
optional => 1,
type => 'string',
- description => "Select the default Console viewer. You can either use the builtin java applet (VNC), an external virt-viewer comtatible application (SPICE), or an HTML5 based viewer (noVNC).",
- enum => ['applet', 'vv', 'html5'],
+ description => "Select the default Console viewer. You can either use the builtin java applet (VNC; deprecated and maps to html5), an external virt-viewer comtatible application (SPICE), an HTML5 based vnc viewer (noVNC), or an HTML5 based console client (xtermjs). If the selected viewer is not available (e.g. SPICE not activated for the VM), the fallback is noVNC.",
+ enum => ['applet', 'vv', 'html5', 'xtermjs'],
},
email_from => {
optional => 1,
pattern => qr/[a-f0-9]{2}(?::[a-f0-9]{2}){0,2}:?/i,
description => 'Prefix for autogenerated MAC addresses.',
},
+ bwlimit => PVE::JSONSchema::get_standard_option('bwlimit'),
},
};
sub parse_datacenter_config {
my ($filename, $raw) = @_;
- return PVE::JSONSchema::parse_config($datacenter_schema, $filename, $raw // '');
+ my $res = PVE::JSONSchema::parse_config($datacenter_schema, $filename, $raw // '');
+
+ if (my $migration = $res->{migration}) {
+ $res->{migration} = PVE::JSONSchema::parse_property_string($migration_format, $migration);
+ }
+
+ # for backwards compatibility only, new migration property has precedence
+ if (defined($res->{migration_unsecure})) {
+ if (defined($res->{migration}->{type})) {
+ warn "deprecated setting 'migration_unsecure' and new 'migration: type' " .
+ "set at same time! Ignore 'migration_unsecure'\n";
+ } else {
+ $res->{migration}->{type} = ($res->{migration_unsecure}) ? 'insecure' : 'secure';
+ }
+ }
+
+ # for backwards compatibility only, applet maps to html5
+ if (defined($res->{console}) && $res->{console} eq 'applet') {
+ $res->{console} = 'html5';
+ }
+
+ return $res;
}
sub write_datacenter_config {
my ($filename, $cfg) = @_;
-
+
+ # map deprecated setting to new one
+ if (defined($cfg->{migration_unsecure}) && !defined($cfg->{migration})) {
+ my $migration_unsecure = delete $cfg->{migration_unsecure};
+ $cfg->{migration}->{type} = ($migration_unsecure) ? 'insecure' : 'secure';
+ }
+
+ # map deprecated applet setting to html5
+ if (defined($cfg->{console}) && $cfg->{console} eq 'applet') {
+ $cfg->{console} = 'html5';
+ }
+
+ if (my $migration = $cfg->{migration}) {
+ $cfg->{migration} = PVE::JSONSchema::print_property_string($migration, $migration_format);
+ }
+
return PVE::JSONSchema::dump_config($datacenter_schema, $filename, $cfg);
}
-cfs_register_file('datacenter.cfg',
- \&parse_datacenter_config,
+cfs_register_file('datacenter.cfg',
+ \&parse_datacenter_config,
\&write_datacenter_config);
-# a very simply parser ...
-sub parse_corosync_conf {
- my ($filename, $raw) = @_;
+# X509 Certificate cache helper
- return {} if !$raw;
-
- my $digest = Digest::SHA::sha1_hex(defined($raw) ? $raw : '');
-
- $raw =~ s/#.*$//mg;
- $raw =~ s/\r?\n/ /g;
- $raw =~ s/\s+/ /g;
- $raw =~ s/^\s+//;
- $raw =~ s/\s*$//;
-
- my @tokens = split(/\s/, $raw);
-
- my $conf = { section => 'main', children => [] };
-
- my $stack = [];
- my $section = $conf;
-
- while (defined(my $token = shift @tokens)) {
- my $nexttok = $tokens[0];
-
- if ($nexttok && ($nexttok eq '{')) {
- shift @tokens; # skip '{'
- my $new_section = {
- section => $token,
- children => [],
- };
- push @{$section->{children}}, $new_section;
- push @$stack, $section;
- $section = $new_section;
- next;
- }
+my $cert_cache_nodes = {};
+my $cert_cache_timestamp = time();
+my $cert_cache_fingerprints = {};
+
+sub update_cert_cache {
+ my ($update_node, $clear) = @_;
+
+ syslog('info', "Clearing outdated entries from certificate cache")
+ if $clear;
+
+ $cert_cache_timestamp = time() if !defined($update_node);
+
+ my $node_list = defined($update_node) ?
+ [ $update_node ] : [ keys %$cert_cache_nodes ];
+
+ foreach my $node (@$node_list) {
+ my $clear_old = sub {
+ if (my $old_fp = $cert_cache_nodes->{$node}) {
+ # distrust old fingerprint
+ delete $cert_cache_fingerprints->{$old_fp};
+ # ensure reload on next proxied request
+ delete $cert_cache_nodes->{$node};
+ }
+ };
- if ($token eq '}') {
- $section = pop @$stack;
- die "parse error - uncexpected '}'\n" if !$section;
+ my $fp = eval { get_node_fingerprint($node) };
+ if (my $err = $@) {
+ warn "$err\n";
+ &$clear_old() if $clear;
next;
}
- my $key = $token;
- die "missing ':' after key '$key'\n" if ! ($key =~ s/:$//);
-
- die "parse error - no value for '$key'\n" if !defined($nexttok);
- my $value = shift @tokens;
+ my $old_fp = $cert_cache_nodes->{$node};
+ $cert_cache_fingerprints->{$fp} = 1;
+ $cert_cache_nodes->{$node} = $fp;
- push @{$section->{children}}, { key => $key, value => $value };
+ if (defined($old_fp) && $fp ne $old_fp) {
+ delete $cert_cache_fingerprints->{$old_fp};
+ }
}
+}
- $conf->{digest} = $digest;
+# load and cache cert fingerprint once
+sub initialize_cert_cache {
+ my ($node) = @_;
- return $conf;
+ update_cert_cache($node)
+ if defined($node) && !defined($cert_cache_nodes->{$node});
}
-my $dump_corosync_section;
-$dump_corosync_section = sub {
- my ($section, $prefix) = @_;
+sub read_ssl_cert_fingerprint {
+ my ($cert_path) = @_;
- my $raw = $prefix . $section->{section} . " {\n";
-
- my @list = grep { defined($_->{key}) } @{$section->{children}};
- foreach my $child (sort {$a->{key} cmp $b->{key}} @list) {
- $raw .= $prefix . " $child->{key}: $child->{value}\n";
- }
-
- @list = grep { defined($_->{section}) } @{$section->{children}};
- foreach my $child (sort {$a->{section} cmp $b->{section}} @list) {
- $raw .= &$dump_corosync_section($child, "$prefix ");
- }
+ my $bio = Net::SSLeay::BIO_new_file($cert_path, 'r')
+ or die "unable to read '$cert_path' - $!\n";
- $raw .= $prefix . "}\n\n";
-
- return $raw;
-
-};
+ my $cert = Net::SSLeay::PEM_read_bio_X509($bio);
+ Net::SSLeay::BIO_free($bio);
-sub write_corosync_conf {
- my ($filename, $conf) = @_;
+ die "unable to read certificate from '$cert_path'\n" if !$cert;
- my $raw = '';
+ my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
+ Net::SSLeay::X509_free($cert);
- my $prefix = '';
-
- die "no main section" if $conf->{section} ne 'main';
+ die "unable to get fingerprint for '$cert_path' - got empty value\n"
+ if !defined($fp) || $fp eq '';
- my @list = grep { defined($_->{key}) } @{$conf->{children}};
- foreach my $child (sort {$a->{key} cmp $b->{key}} @list) {
- $raw .= "$child->{key}: $child->{value}\n";
- }
+ return $fp;
+}
- @list = grep { defined($_->{section}) } @{$conf->{children}};
- foreach my $child (sort {$a->{section} cmp $b->{section}} @list) {
- $raw .= &$dump_corosync_section($child, $prefix);
- }
+sub get_node_fingerprint {
+ my ($node) = @_;
- return $raw;
-}
+ my $cert_path = "/etc/pve/nodes/$node/pve-ssl.pem";
+ my $custom_cert_path = "/etc/pve/nodes/$node/pveproxy-ssl.pem";
-sub corosync_conf_version {
- my ($conf, $noerr, $new_value) = @_;
-
- foreach my $child (@{$conf->{children}}) {
- next if !defined($child->{section});
- if ($child->{section} eq 'totem') {
- foreach my $e (@{$child->{children}}) {
- next if !defined($e->{key});
- if ($e->{key} eq 'config_version') {
- if ($new_value) {
- $e->{value} = $new_value;
- return $new_value;
- } elsif (my $version = int($e->{value})) {
- return $version;
- }
- last;
- }
- }
- }
- }
-
- return undef if $noerr;
+ $cert_path = $custom_cert_path if -f $custom_cert_path;
- die "invalid corosync config - unable to read version\n";
+ return read_ssl_cert_fingerprint($cert_path);
}
-# read only - use "rename corosync.conf.new corosync.conf" to write
-PVE::Cluster::cfs_register_file('corosync.conf', \&parse_corosync_conf);
-# this is read/write
-PVE::Cluster::cfs_register_file('corosync.conf.new', \&parse_corosync_conf,
- \&write_corosync_conf);
-sub check_corosync_conf_exists {
- my ($silent) = @_;
+sub check_cert_fingerprint {
+ my ($cert) = @_;
+
+ # clear cache every 30 minutes at least
+ update_cert_cache(undef, 1) if time() - $cert_cache_timestamp >= 60*30;
- $silent = $silent // 0;
+ # get fingerprint of server certificate
+ my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
+ return 0 if !defined($fp) || $fp eq ''; # error
- my $exists = -f "$basedir/corosync.conf";
+ my $check = sub {
+ for my $expected (keys %$cert_cache_fingerprints) {
+ return 1 if $fp eq $expected;
+ }
+ return 0;
+ };
- warn "Corosync config '$basedir/corosync.conf' does not exist - is this node part of a cluster?\n"
- if !$silent && !$exists;
+ return 1 if &$check();
- return $exists;
+ # clear cache and retry at most once every minute
+ if (time() - $cert_cache_timestamp >= 60) {
+ syslog ('info', "Could not verify remote node certificate '$fp' with list of pinned certificates, refreshing cache");
+ update_cert_cache();
+ return &$check();
+ }
+
+ return 0;
}
# bash completion helpers
return $res;
}
+sub get_ssh_info {
+ my ($node, $network_cidr) = @_;
+
+ my $ip;
+ if (defined($network_cidr)) {
+ # Use mtunnel via to get the remote node's ip inside $network_cidr.
+ # This goes over the regular network (iow. uses get_ssh_info() with
+ # $network_cidr undefined.
+ # FIXME: Use the REST API client for this after creating an API entry
+ # for get_migration_ip.
+ my $default_remote = get_ssh_info($node, undef);
+ my $default_ssh = ssh_info_to_command($default_remote);
+ my $cmd =[@$default_ssh, 'pvecm', 'mtunnel',
+ '-migration_network', $network_cidr,
+ '-get_migration_ip'
+ ];
+ PVE::Tools::run_command($cmd, outfunc => sub {
+ my ($line) = @_;
+ chomp $line;
+ die "internal error: unexpected output from mtunnel\n"
+ if defined($ip);
+ if ($line =~ /^ip: '(.*)'$/) {
+ $ip = $1;
+ } else {
+ die "internal error: bad output from mtunnel\n"
+ if defined($ip);
+ }
+ });
+ die "failed to get ip for node '$node' in network '$network_cidr'\n"
+ if !defined($ip);
+ } else {
+ $ip = remote_node_ip($node);
+ }
+
+ return {
+ ip => $ip,
+ name => $node,
+ network => $network_cidr,
+ };
+}
+
+sub ssh_info_to_command_base {
+ my ($info, @extra_options) = @_;
+ return [
+ '/usr/bin/ssh',
+ '-e', 'none',
+ '-o', 'BatchMode=yes',
+ '-o', 'HostKeyAlias='.$info->{name},
+ @extra_options
+ ];
+}
+
+sub ssh_info_to_command {
+ my ($info, @extra_options) = @_;
+ my $cmd = ssh_info_to_command_base($info, @extra_options);
+ push @$cmd, "root\@$info->{ip}";
+ return $cmd;
+}
+
+sub assert_joinable {
+ my ($ring0_addr, $ring1_addr, $force) = @_;
+
+ my $errors = '';
+ my $error = sub { $errors .= "* $_[0]\n"; };
+
+ if (-f $authfile) {
+ $error->("authentication key '$authfile' already exists");
+ }
+
+ if (-f $clusterconf) {
+ $error->("cluster config '$clusterconf' already exists");
+ }
+
+ my $vmlist = get_vmlist();
+ if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
+ $error->("this host already contains virtual guests");
+ }
+
+ if (run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
+ $error->("corosync is already running, is this node already in a cluster?!");
+ }
+
+ # check if corosync ring IPs are configured on the current nodes interfaces
+ my $check_ip = sub {
+ my $ip = shift // return;
+ if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
+ my $host = $ip;
+ eval { $ip = PVE::Network::get_ip_from_hostname($host); };
+ if ($@) {
+ $error->("cannot use '$host': $@\n") ;
+ return;
+ }
+ }
+
+ my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
+ my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
+
+ $error->("cannot use IP '$ip', it must be configured exactly once on local node!\n")
+ if (scalar(@$configured_ips) != 1);
+ };
+
+ $check_ip->($ring0_addr);
+ $check_ip->($ring1_addr);
+
+ if ($errors) {
+ warn "detected the following error(s):\n$errors";
+ die "Check if node may join a cluster failed!\n" if !$force;
+ }
+}
+
+# NOTE: filesystem must be offline here, no DB changes allowed
+my $backup_cfs_database = sub {
+ my ($dbfile) = @_;
+
+ mkdir $dbbackupdir;
+
+ my $ctime = time();
+ my $backup_fn = "$dbbackupdir/config-$ctime.sql.gz";
+
+ print "backup old database to '$backup_fn'\n";
+
+ my $cmd = [ ['sqlite3', $dbfile, '.dump'], ['gzip', '-', \ ">${backup_fn}"] ];
+ run_command($cmd, 'errmsg' => "cannot backup old database\n");
+
+ my $maxfiles = 10; # purge older backup
+ my $backups = [ sort { $b cmp $a } <$dbbackupdir/config-*.sql.gz> ];
+
+ if ((my $count = scalar(@$backups)) > $maxfiles) {
+ foreach my $f (@$backups[$maxfiles..$count-1]) {
+ next if $f !~ m/^(\S+)$/; # untaint
+ print "delete old backup '$1'\n";
+ unlink $1;
+ }
+ }
+};
+
+sub join {
+ my ($param) = @_;
+
+ my $nodename = PVE::INotify::nodename();
+
+ setup_sshd_config();
+ setup_rootsshconfig();
+ setup_ssh_keys();
+
+ # check if we can join with the given parameters and current node state
+ my ($ring0_addr, $ring1_addr) = $param->@{'ring0_addr', 'ring1_addr'};
+ assert_joinable($ring0_addr, $ring1_addr, $param->{force});
+
+ # make sure known_hosts is on local filesystem
+ ssh_unmerge_known_hosts();
+
+ my $host = $param->{hostname};
+ my $local_ip_address = remote_node_ip($nodename);
+
+ my $conn_args = {
+ username => 'root@pam',
+ password => $param->{password},
+ cookie_name => 'PVEAuthCookie',
+ protocol => 'https',
+ host => $host,
+ port => 8006,
+ };
+
+ if (my $fp = $param->{fingerprint}) {
+ $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
+ } else {
+ # API schema ensures that we can only get here from CLI handler
+ $conn_args->{manual_verification} = 1;
+ }
+
+ print "Establishing API connection with host '$host'\n";
+
+ my $conn = PVE::APIClient::LWP->new(%$conn_args);
+ $conn->login();
+
+ # login raises an exception on failure, so if we get here we're good
+ print "Login succeeded.\n";
+
+ my $args = {};
+ $args->{force} = $param->{force} if defined($param->{force});
+ $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
+ $args->{votes} = $param->{votes} if defined($param->{votes});
+ $args->{ring0_addr} = $ring0_addr // $local_ip_address;
+ $args->{ring1_addr} = $ring1_addr if defined($ring1_addr);
+
+ print "Request addition of this node\n";
+ my $res = $conn->post("/cluster/config/nodes/$nodename", $args);
+
+ print "Join request OK, finishing setup locally\n";
+
+ # added successfuly - now prepare local node
+ finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
+}
+
+sub finish_join {
+ my ($nodename, $corosync_conf, $corosync_authkey) = @_;
+
+ mkdir "$localclusterdir";
+ PVE::Tools::file_set_contents($authfile, $corosync_authkey);
+ PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
+
+ print "stopping pve-cluster service\n";
+ my $cmd = ['systemctl', 'stop', 'pve-cluster'];
+ run_command($cmd, errmsg => "can't stop pve-cluster service");
+
+ $backup_cfs_database->($dbfile);
+ unlink $dbfile;
+
+ $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
+ run_command($cmd, errmsg => "starting pve-cluster failed");
+
+ # wait for quorum
+ my $printqmsg = 1;
+ while (!check_cfs_quorum(1)) {
+ if ($printqmsg) {
+ print "waiting for quorum...";
+ STDOUT->flush();
+ $printqmsg = 0;
+ }
+ sleep(1);
+ }
+ print "OK\n" if !$printqmsg;
+
+ updatecerts_and_ssh(1);
+
+ print "generated new node certificate, restart pveproxy and pvedaemon services\n";
+ run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
+
+ print "successfully added node '$nodename' to cluster.\n";
+}
+
+sub updatecerts_and_ssh {
+ my ($force_new_cert, $silent) = @_;
+
+ my $p = sub { print "$_[0]\n" if !$silent };
+
+ setup_rootsshconfig();
+
+ gen_pve_vzdump_symlink();
+
+ if (!check_cfs_quorum(1)) {
+ return undef if $silent;
+ die "no quorum - unable to update files\n";
+ }
+
+ setup_ssh_keys();
+
+ my $nodename = PVE::INotify::nodename();
+ my $local_ip_address = remote_node_ip($nodename);
+
+ $p->("(re)generate node files");
+ $p->("generate new node certificate") if $force_new_cert;
+ gen_pve_node_files($nodename, $local_ip_address, $force_new_cert);
+
+ $p->("merge authorized SSH keys and known hosts");
+ ssh_merge_keys();
+ ssh_merge_known_hosts($nodename, $local_ip_address, 1);
+ gen_pve_vzdump_files();
+}
+
1;
projects / pve-cluster.git / blobdiff