use strict;
use warnings;
-use POSIX qw(EEXIST);
+use POSIX qw(EEXIST ENOENT);
use File::stat qw();
use Socket;
use Storable qw(dclone);
use IO::File;
use MIME::Base64;
-use XML::Parser;
use Digest::SHA;
use Digest::HMAC_SHA1;
-use PVE::Tools;
+use Net::SSLeay;
+use PVE::Tools qw(run_command);
use PVE::INotify;
use PVE::IPCC;
use PVE::SafeSyslog;
use PVE::JSONSchema;
+use PVE::Network;
use JSON;
use RRDs;
use Encode;
+use UUID;
use base 'Exporter';
our @EXPORT_OK = qw(
my $authdir = "$basedir/priv";
my $lockdir = "/etc/pve/priv/lock";
+# cfs and corosync files
+my $dbfile = "/var/lib/pve-cluster/config.db";
+my $dbbackupdir = "/var/lib/pve-cluster/backup";
+my $localclusterdir = "/etc/corosync";
+my $localclusterconf = "$localclusterdir/corosync.conf";
+my $authfile = "$localclusterdir/authkey";
+my $clusterconf = "$basedir/corosync.conf";
+
my $authprivkeyfn = "$authdir/authkey.key";
my $authpubkeyfn = "$basedir/authkey.pub";
my $pveca_key_fn = "$authdir/pve-root-ca.key";
'vzdump.cron' => 1,
'storage.cfg' => 1,
'datacenter.cfg' => 1,
+ 'replication.cfg' => 1,
'corosync.conf' => 1,
'corosync.conf.new' => 1,
'user.cfg' => 1,
'ha/manager_status' => 1,
'ha/resources.cfg' => 1,
'ha/groups.cfg' => 1,
+ 'ha/fence.cfg' => 1,
'status.cfg' => 1,
};
};
eval {
- PVE::Tools::run_command($cmd, outfunc => $record_output,
+ PVE::Tools::run_command($cmd, outfunc => $record_output,
errfunc => $record_output);
};
my @required_dirs = (
"$basedir/priv",
- "$basedir/nodes",
+ "$basedir/nodes",
"$basedir/nodes/$nodename",
"$basedir/nodes/$nodename/lxc",
"$basedir/nodes/$nodename/qemu-server",
"$basedir/nodes/$nodename/openvz",
"$basedir/nodes/$nodename/priv");
-
+
foreach my $dir (@required_dirs) {
if (! -d $dir) {
mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
mkdir $authdir || $! == EEXIST || die "unable to create dir '$authdir' - $!\n";
- my $cmd = "openssl genrsa -out '$authprivkeyfn' 2048";
- run_silent_cmd($cmd);
+ run_silent_cmd(['openssl', 'genrsa', '-out', $authprivkeyfn, '2048']);
- $cmd = "openssl rsa -in '$authprivkeyfn' -pubout -out '$authpubkeyfn'";
- run_silent_cmd($cmd)
+ run_silent_cmd(['openssl', 'rsa', '-in', $authprivkeyfn, '-pubout', '-out', $authpubkeyfn]);
}
sub gen_pveca_key {
return if -f $pveca_key_fn;
eval {
- run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '2048']);
+ run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']);
};
die "unable to generate pve ca key:\n$@" if $@;
# we try to generate an unique 'subject' to avoid browser problems
# (reused serial numbers, ..)
- my $nid = (split (/\s/, `md5sum '$pveca_key_fn'`))[0] || time();
+ my $uuid;
+ UUID::generate($uuid);
+ my $uuid_str;
+ UUID::unparse($uuid, $uuid_str);
eval {
- run_silent_cmd(['openssl', 'req', '-batch', '-days', '3650', '-new',
- '-x509', '-nodes', '-key',
+ # wrap openssl with faketime to prevent bug #904
+ run_silent_cmd(['faketime', 'yesterday', 'openssl', 'req', '-batch',
+ '-days', '3650', '-new', '-x509', '-nodes', '-key',
$pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
- "/CN=Proxmox Virtual Environment/OU=$nid/O=PVE Cluster Manager CA/"]);
+ "/CN=Proxmox Virtual Environment/OU=$uuid_str/O=PVE Cluster Manager CA/"]);
};
die "generating pve root certificate failed:\n$@" if $@;
my $rc = PVE::INotify::read_file('resolvconf');
$names .= ",IP:$ip";
-
+
my $fqdn = $nodename;
$names .= ",DNS:$nodename";
[ v3_req ]
basicConstraints = CA:FALSE
-nsCertType = server
-keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+extendedKeyUsage = serverAuth
subjectAltName = $names
__EOD
update_serial("0000000000000000") if ! -f $pveca_srl_fn;
eval {
- run_silent_cmd(['openssl', 'x509', '-req', '-in', $reqfn, '-days', '3650',
- '-out', $pvessl_cert_fn, '-CAkey', $pveca_key_fn,
- '-CA', $pveca_cert_fn, '-CAserial', $pveca_srl_fn,
- '-extfile', $cfgfn]);
+ # wrap openssl with faketime to prevent bug #904
+ run_silent_cmd(['faketime', 'yesterday', 'openssl', 'x509', '-req',
+ '-in', $reqfn, '-days', '3650', '-out', $pvessl_cert_fn,
+ '-CAkey', $pveca_key_fn, '-CA', $pveca_cert_fn,
+ '-CAserial', $pveca_srl_fn, '-extfile', $cfgfn]);
};
if (my $err = $@) {
my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
- die "ipcc_send_rec failed: $!\n" if !defined($res) && ($! != 0);
+ die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
return $res;
};
my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
- die "ipcc_send_rec failed: $!\n" if !defined($res) && ($! != 0);
+ die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
return decode_json($res);
};
my $bindata = pack "Z*", $path;
my $res = PVE::IPCC::ipcc_send_rec(6, $bindata);
if (!defined($res)) {
- return undef if ($! != 0);
+ if ($! != 0) {
+ return undef if $! == ENOENT;
+ die "$!\n";
+ }
return '';
}
my $ccache = {};
sub cfs_update {
+ my ($fail) = @_;
eval {
my $res = &$ipcc_send_rec_json(1);
#warn "GOT1: " . Dumper($res);
$vmlist = {};
$clinfo = {};
$ccache = {};
+ die $err if $fail;
warn $err;
}
$err = $@;
if ($err) {
$clinfo = {};
+ die $err if $fail;
warn $err;
}
$err = $@;
if ($err) {
$vmlist = {};
+ die $err if $fail;
warn $err;
}
}
return [ keys %$nodelist ];
}
+# $data must be a chronological descending ordered array of tasks
sub broadcast_tasklist {
my ($data) = @_;
+ # the serialized list may not get bigger than 32kb (CFS_MAX_STATUS_SIZE
+ # from pmxcfs) - drop older items until we satisfy this constraint
+ my $size = length(encode_json($data));
+ while ($size >= (32 * 1024)) {
+ pop @$data;
+ $size = length(encode_json($data));
+ }
+
eval {
&$ipcc_update_status("tasklist", $data);
};
eval {
my $ver = $kvstore->{$node}->{tasklist} if $kvstore->{$node};
my $cd = $tasklistcache->{$node};
- if (!$cd || !$ver || !$cd->{version} ||
+ if (!$cd || !$ver || !$cd->{version} ||
($cd->{version} != $ver)) {
my $raw = &$ipcc_get_status("tasklist", $node) || '[]';
my $data = decode_json($raw);
my ($key, @ela) = split(/:/, $1);
next if !$key;
next if !(scalar(@ela) > 1);
- $res->{$key} = \@ela;
+ $res->{$key} = [ map { $_ eq 'U' ? undef : $_ } @ela ];
}
}
my $err = RRDs::error;
die "RRD error: $err\n" if $err;
-
- die "got wrong time resolution ($step != $reso)\n"
+
+ die "got wrong time resolution ($step != $reso)\n"
if $step != $reso;
my $res = [];
for my $line (@$data) {
my $entry = { 'time' => $start };
$start += $step;
- my $found_undefs;
for (my $i = 0; $i < $fields; $i++) {
my $name = $names->[$i];
if (defined(my $val = $line->[$i])) {
$entry->{$name} = $val;
} else {
- # we only add entryies with all data defined
- # extjs chart has problems with undefined values
- $found_undefs = 1;
+ # leave empty fields undefined
+ # maybe make this configurable?
}
}
- push @$res, $entry if !$found_undefs;
+ push @$res, $entry;
}
return $res;
# Using RRD graph is clumsy - maybe it
# is better to simply fetch the data, and do all display
# related things with javascript (new extjs html5 graph library).
-
+
my $rrddir = "/var/lib/rrdcached/db";
my $rrd = "$rrddir/$rrdname";
"--width" => 800,
"--start" => - $reso*$count,
"--end" => 'now' ,
+ "--lower-limit" => 0,
);
my $socket = "/var/run/rrdcached.sock";
push @args, '--full-size-mode';
# we do not really store data into the file
- my $res = RRDs::graphv('', @args);
+ my $res = RRDs::graphv('-', @args);
my $err = RRDs::error;
die "RRD error: $err\n" if $err;
sub cfs_read_file {
my ($filename) = @_;
- my ($version, $info) = cfs_file_version($filename);
+ my ($version, $info) = cfs_file_version($filename);
my $parser = $info->{parser};
return &$ccache_read($filename, $parser, $version);
sub cfs_write_file {
my ($filename, $data) = @_;
- my ($version, $info) = cfs_file_version($filename);
+ my ($version, $info) = cfs_file_version($filename);
my $writer = $info->{writer} || die "no writer defined";
my $cfs_lock = sub {
my ($lockid, $timeout, $code, @param) = @_;
+ my $prev_alarm = alarm(0); # suspend outer alarm early
+
my $res;
+ my $got_lock = 0;
# this timeout is for aquire the lock
$timeout = 10 if !$timeout;
my $filename = "$lockdir/$lockid";
- my $msg = "can't aquire cfs lock '$lockid'";
-
eval {
mkdir $lockdir;
if (! -d $lockdir) {
- die "$msg: pve cluster filesystem not online.\n";
+ die "pve cluster filesystem not online.\n";
}
- local $SIG{ALRM} = sub { die "got lock request timeout\n"; };
+ my $timeout_err = sub { die "got lock request timeout\n"; };
+ local $SIG{ALRM} = $timeout_err;
- alarm ($timeout);
+ while (1) {
+ alarm ($timeout);
+ $got_lock = mkdir($filename);
+ $timeout = alarm(0) - 1; # we'll sleep for 1s, see down below
- if (!(mkdir $filename)) {
- print STDERR "trying to aquire cfs lock '$lockid' ...";
- while (1) {
- if (!(mkdir $filename)) {
- (utime 0, 0, $filename); # cfs unlock request
- } else {
- print STDERR " OK\n";
- last;
- }
- sleep(1);
- }
+ last if $got_lock;
+
+ $timeout_err->() if $timeout <= 0;
+
+ print STDERR "trying to aquire cfs lock '$lockid' ...\n";
+ utime (0, 0, $filename); # cfs unlock request
+ sleep(1);
}
# fixed command timeout: cfs locks have a timeout of 120
# using 60 gives us another 60 seconds to abort the task
- alarm(60);
local $SIG{ALRM} = sub { die "got lock timeout - aborting command\n"; };
+ alarm(60);
cfs_update(); # make sure we read latest versions inside code()
my $err = $@;
- alarm(0);
+ $err = "no quorum!\n" if !$got_lock && !check_cfs_quorum(1);
- if ($err && ($err eq "got lock request timeout\n") &&
- !check_cfs_quorum()){
- $err = "$msg: no quorum!\n";
- }
+ rmdir $filename if $got_lock; # if we held the lock always unlock again
- if (!$err || $err !~ /^got lock timeout -/) {
- rmdir $filename; # cfs unlock
- }
+ alarm($prev_alarm);
if ($err) {
- $@ = $err;
+ $@ = "error with cfs lock '$lockid': $err";
return undef;
}
&$cfs_lock($lockid, $timeout, $code, @param);
}
+sub cfs_lock_domain {
+ my ($domainname, $timeout, $code, @param) = @_;
+
+ my $lockid = "domain-$domainname";
+
+ &$cfs_lock($lockid, $timeout, $code, @param);
+}
+
my $log_levels = {
"emerg" => 0,
"alert" => 1,
$msg = "empty message" if !$msg;
$ident = "" if !$ident;
- $ident = encode("ascii", decode_utf8($ident),
+ $ident = encode("ascii", $ident,
sub { sprintf "\\u%04x", shift });
- my $utf8 = decode_utf8($msg);
-
- my $ascii = encode("ascii", $utf8, sub { sprintf "\\u%04x", shift });
+ my $ascii = encode("ascii", $msg, sub { sprintf "\\u%04x", shift });
if ($ident) {
syslog($priority, "<%s> %s", $ident, $ascii);
sub check_vmid_unused {
my ($vmid, $noerr) = @_;
-
+
my $vmlist = get_vmlist();
- my $node = PVE::INotify::nodename();
my $d = $vmlist->{ids}->{$vmid};
return 1 if !defined($d);
-
- return undef if $noerr;
- my $msg = "$vmid already exists";
- my $msg_cluster = "$msg on cluster node '$d->{node}'";
+ return undef if $noerr;
- if ($d->{node} eq $node) {
- die "VM $msg\n" if $d->{type} eq 'qemu';
- die "CT $msg\n";
- } else {
- die "VM $msg_cluster\n" if $d->{type} eq 'qemu';
- die "CT $msg_cluster\n";
- }
+ my $vmtypestr = $d->{type} eq 'qemu' ? 'VM' : 'CT';
+ die "$vmtypestr $vmid already exists on node '$d->{node}'\n";
}
sub check_node_exists {
$family =
PVE::Tools::get_host_address_family($ip);
}
- return ($ip, $family);
+ return wantarray ? ($ip, $family) : $ip;
}
}
# fallback: try to get IP by other means
- my ($family, $packed_ip);
+ return PVE::Network::get_ip_from_hostname($nodename, $noerr);
+}
- eval {
- my @res = PVE::Tools::getaddrinfo_all($nodename);
- $family = $res[0]->{family};
- $packed_ip = (PVE::Tools::unpack_sockaddr_in46($res[0]->{addr}))[2];
- };
+sub get_local_migration_ip {
+ my ($migration_network, $noerr) = @_;
+
+ my $cidr = $migration_network;
- if ($@) {
- die "hostname lookup failed:\n$@" if !$noerr;
- return undef;
+ if (!defined($cidr)) {
+ my $dc_conf = cfs_read_file('datacenter.cfg');
+ $cidr = $dc_conf->{migration}->{network}
+ if defined($dc_conf->{migration}->{network});
}
- my $ip = Socket::inet_ntop($family, $packed_ip);
- if ($ip =~ m/^127\.|^::1$/) {
- die "hostname lookup failed - got local IP address ($nodename = $ip)\n" if !$noerr;
- return undef;
+ if (defined($cidr)) {
+ my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
+
+ die "could not get migration ip: no IP address configured on local " .
+ "node for network '$cidr'\n" if !$noerr && (scalar(@$ips) == 0);
+
+ die "could not get migration ip: multiple IP address configured for " .
+ "network '$cidr'\n" if !$noerr && (scalar(@$ips) > 1);
+
+ return @$ips[0];
}
- return wantarray ? ($ip, $family) : $ip;
-}
+ return undef;
+};
# ssh related utility functions
}
sub setup_sshd_config {
+ my () = @_;
my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
-
+
return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
chomp $conf;
$conf .= "\nPermitRootLogin yes\n";
- }
+ }
PVE::Tools::file_set_contents($sshd_config_fn, $conf);
if (! -f $rootsshconfig) {
mkdir '/root/.ssh';
if (my $fh = IO::File->new($rootsshconfig, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
- # this is the default ciphers list from debian openssl0.9.8 except blowfish is added as prefered
- print $fh "Ciphers blowfish-cbc,aes128-ctr,aes192-ctr,aes256-ctr,arcfour256,arcfour128,aes128-cbc,3des-cbc\n";
+ # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k 26 Jan 2017)
+ # changed order to put AES before Chacha20 (most hardware has AESNI)
+ print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
close($fh);
}
}
}
}
- warn "can't create shared ssh key database '$sshauthkeys'\n"
+ warn "can't create shared ssh key database '$sshauthkeys'\n"
if ! -f $sshauthkeys;
if (-f $rootsshauthkeys && ! -l $rootsshauthkeys) {
die "no node name specified" if !$nodename;
die "no ip address specified" if !$ip_address;
-
+
+ # ssh lowercases hostnames (aliases) before comparision, so we need too
+ $nodename = lc($nodename);
+ $ip_address = lc($ip_address);
+
mkdir $authdir;
if (! -f $sshknownhosts) {
}
}
- my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
-
+ my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
+
my $new = '';
-
+
if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
$new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
}
my $merge_line = sub {
my ($line, $all) = @_;
+ return if $line =~ m/^\s*$/; # skip empty lines
+ return if $line =~ m/^#/; # skip comments
+
if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
my $key = $1;
my $rsakey = $2;
}
return;
}
+ } else {
+ $key = lc($key); # avoid duplicate entries, ssh compares lowercased
+ if ($key eq $ip_address) {
+ $found_local_ip = 1 if $rsakey eq $hostkey;
+ } elsif ($key eq $nodename) {
+ $found_nodename = 1 if $rsakey eq $hostkey;
+ }
}
$data .= $line;
}
while ($old && $old =~ s/^((.*?)(\n|$))//) {
my $line = "$2\n";
- next if $line =~ m/^\s*$/; # skip empty lines
- next if $line =~ m/^#/; # skip comments
&$merge_line($line, 1);
}
while ($new && $new =~ s/^((.*?)(\n|$))//) {
my $line = "$2\n";
- next if $line =~ m/^\s*$/; # skip empty lines
- next if $line =~ m/^#/; # skip comments
&$merge_line($line);
}
- my $addIndex = $$;
- my $add_known_hosts_entry = sub {
- my ($name, $hostkey) = @_;
- $addIndex++;
- my $hmac = Digest::HMAC_SHA1->new("$addIndex" . time());
- my $b64salt = $hmac->b64digest . '=';
- $hmac = Digest::HMAC_SHA1->new(decode_base64($b64salt));
- $hmac->add($name);
- my $digest = $hmac->b64digest . '=';
- $data .= "|1|$b64salt|$digest $hostkey\n";
- };
-
- if (!$found_nodename || !$found_local_ip) {
- &$add_known_hosts_entry($nodename, $hostkey) if !$found_nodename;
- &$add_known_hosts_entry($ip_address, $hostkey) if !$found_local_ip;
- }
+ # add our own key if not already there
+ $data .= "$nodename $hostkey\n" if !$found_nodename;
+ $data .= "$ip_address $hostkey\n" if !$found_local_ip;
PVE::Tools::file_set_contents($sshknownhosts, $data);
unlink $sshglobalknownhosts;
symlink $sshknownhosts, $sshglobalknownhosts;
-
- warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
+
+ warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
if ! -l $sshglobalknownhosts;
}
+my $migration_format = {
+ type => {
+ default_key => 1,
+ type => 'string',
+ enum => ['secure', 'insecure'],
+ description => "Migration traffic is encrypted using an SSH tunnel by " .
+ "default. On secure, completely private networks this can be " .
+ "disabled to increase performance.",
+ default => 'secure',
+ },
+ network => {
+ optional => 1,
+ type => 'string', format => 'CIDR',
+ format_description => 'CIDR',
+ description => "CIDR of the (sub) network that is used for migration."
+ },
+};
+
my $datacenter_schema = {
type => "object",
additionalProperties => 0,
migration_unsecure => {
optional => 1,
type => 'boolean',
- description => "Migration is secure using SSH tunnel by default. For secure private networks you can disable it to speed up migration.",
+ description => "Migration is secure using SSH tunnel by default. " .
+ "For secure private networks you can disable it to speed up " .
+ "migration. Deprecated, use the 'migration' property instead!",
+ },
+ migration => {
+ optional => 1,
+ type => 'string', format => $migration_format,
+ description => "For cluster wide migration settings.",
},
console => {
optional => 1,
type => 'string',
- description => "Select the default Console viewer. You can either use the builtin java applet (VNC), an external virt-viewer comtatible application (SPICE), or an HTML5 based viewer (noVNC).",
+ description => "Select the default Console viewer. You can either use the builtin java applet (VNC; deprecated and maps to html5), an external virt-viewer comtatible application (SPICE), or an HTML5 based viewer (noVNC).",
enum => ['applet', 'vv', 'html5'],
},
email_from => {
format => 'email-opt',
description => "Specify email address to send notification from (default is root@\$hostname)",
},
+ max_workers => {
+ optional => 1,
+ type => 'integer',
+ minimum => 1,
+ description => "Defines how many workers (per node) are maximal started ".
+ " on actions like 'stopall VMs' or task from the ha-manager.",
+ },
+ fencing => {
+ optional => 1,
+ type => 'string',
+ default => 'watchdog',
+ enum => [ 'watchdog', 'hardware', 'both' ],
+ description => "Set the fencing mode of the HA cluster. Hardware mode " .
+ "needs a valid configuration of fence devices in /etc/pve/ha/fence.cfg." .
+ " With both all two modes are used." .
+ "\n\nWARNING: 'hardware' and 'both' are EXPERIMENTAL & WIP",
+ },
+ mac_prefix => {
+ optional => 1,
+ type => 'string',
+ pattern => qr/[a-f0-9]{2}(?::[a-f0-9]{2}){0,2}:?/i,
+ description => 'Prefix for autogenerated MAC addresses.',
+ },
+ bwlimit => PVE::JSONSchema::get_standard_option('bwlimit'),
},
};
sub parse_datacenter_config {
my ($filename, $raw) = @_;
- return PVE::JSONSchema::parse_config($datacenter_schema, $filename, $raw // '');
+ my $res = PVE::JSONSchema::parse_config($datacenter_schema, $filename, $raw // '');
+
+ if (my $migration = $res->{migration}) {
+ $res->{migration} = PVE::JSONSchema::parse_property_string($migration_format, $migration);
+ }
+
+ # for backwards compatibility only, new migration property has precedence
+ if (defined($res->{migration_unsecure})) {
+ if (defined($res->{migration}->{type})) {
+ warn "deprecated setting 'migration_unsecure' and new 'migration: type' " .
+ "set at same time! Ignore 'migration_unsecure'\n";
+ } else {
+ $res->{migration}->{type} = ($res->{migration_unsecure}) ? 'insecure' : 'secure';
+ }
+ }
+
+ # for backwards compatibility only, applet maps to html5
+ if (defined($res->{console}) && $res->{console} eq 'applet') {
+ $res->{console} = 'html5';
+ }
+
+ return $res;
}
sub write_datacenter_config {
my ($filename, $cfg) = @_;
-
+
+ # map deprecated setting to new one
+ if (defined($cfg->{migration_unsecure}) && !defined($cfg->{migration})) {
+ my $migration_unsecure = delete $cfg->{migration_unsecure};
+ $cfg->{migration}->{type} = ($migration_unsecure) ? 'insecure' : 'secure';
+ }
+
+ # map deprecated applet setting to html5
+ if (defined($cfg->{console}) && $cfg->{console} eq 'applet') {
+ $cfg->{console} = 'html5';
+ }
+
+ if (my $migration = $cfg->{migration}) {
+ $cfg->{migration} = PVE::JSONSchema::print_property_string($migration, $migration_format);
+ }
+
return PVE::JSONSchema::dump_config($datacenter_schema, $filename, $cfg);
}
-cfs_register_file('datacenter.cfg',
- \&parse_datacenter_config,
+cfs_register_file('datacenter.cfg',
+ \&parse_datacenter_config,
\&write_datacenter_config);
-# a very simply parser ...
-sub parse_corosync_conf {
- my ($filename, $raw) = @_;
+# X509 Certificate cache helper
- return {} if !$raw;
-
- my $digest = Digest::SHA::sha1_hex(defined($raw) ? $raw : '');
-
- $raw =~ s/#.*$//mg;
- $raw =~ s/\r?\n/ /g;
- $raw =~ s/\s+/ /g;
- $raw =~ s/^\s+//;
- $raw =~ s/\s*$//;
-
- my @tokens = split(/\s/, $raw);
-
- my $conf = { section => 'main', children => [] };
-
- my $stack = [];
- my $section = $conf;
-
- while (defined(my $token = shift @tokens)) {
- my $nexttok = $tokens[0];
-
- if ($nexttok && ($nexttok eq '{')) {
- shift @tokens; # skip '{'
- my $new_section = {
- section => $token,
- children => [],
- };
- push @{$section->{children}}, $new_section;
- push @$stack, $section;
- $section = $new_section;
- next;
- }
+my $cert_cache_nodes = {};
+my $cert_cache_timestamp = time();
+my $cert_cache_fingerprints = {};
+
+sub update_cert_cache {
+ my ($update_node, $clear) = @_;
+
+ syslog('info', "Clearing outdated entries from certificate cache")
+ if $clear;
- if ($token eq '}') {
- $section = pop @$stack;
- die "parse error - uncexpected '}'\n" if !$section;
+ $cert_cache_timestamp = time() if !defined($update_node);
+
+ my $node_list = defined($update_node) ?
+ [ $update_node ] : [ keys %$cert_cache_nodes ];
+
+ foreach my $node (@$node_list) {
+ my $clear_old = sub {
+ if (my $old_fp = $cert_cache_nodes->{$node}) {
+ # distrust old fingerprint
+ delete $cert_cache_fingerprints->{$old_fp};
+ # ensure reload on next proxied request
+ delete $cert_cache_nodes->{$node};
+ }
+ };
+
+ my $fp = eval { get_node_fingerprint($node) };
+ if (my $err = $@) {
+ warn "$err\n";
+ &$clear_old() if $clear;
next;
}
- my $key = $token;
- die "missing ':' after key '$key'\n" if ! ($key =~ s/:$//);
-
- die "parse error - no value for '$key'\n" if !defined($nexttok);
- my $value = shift @tokens;
+ my $old_fp = $cert_cache_nodes->{$node};
+ $cert_cache_fingerprints->{$fp} = 1;
+ $cert_cache_nodes->{$node} = $fp;
- push @{$section->{children}}, { key => $key, value => $value };
+ if (defined($old_fp) && $fp ne $old_fp) {
+ delete $cert_cache_fingerprints->{$old_fp};
+ }
}
+}
- $conf->{digest} = $digest;
+# load and cache cert fingerprint once
+sub initialize_cert_cache {
+ my ($node) = @_;
- return $conf;
+ update_cert_cache($node)
+ if defined($node) && !defined($cert_cache_nodes->{$node});
}
-my $dump_corosync_section;
-$dump_corosync_section = sub {
- my ($section, $prefix) = @_;
+sub read_ssl_cert_fingerprint {
+ my ($cert_path) = @_;
- my $raw = $prefix . $section->{section} . " {\n";
-
- my @list = grep { defined($_->{key}) } @{$section->{children}};
- foreach my $child (sort {$a->{key} cmp $b->{key}} @list) {
- $raw .= $prefix . " $child->{key}: $child->{value}\n";
- }
-
- @list = grep { defined($_->{section}) } @{$section->{children}};
- foreach my $child (sort {$a->{section} cmp $b->{section}} @list) {
- $raw .= &$dump_corosync_section($child, "$prefix ");
+ my $bio = Net::SSLeay::BIO_new_file($cert_path, 'r')
+ or die "unable to read '$cert_path' - $!\n";
+
+ my $cert = Net::SSLeay::PEM_read_bio_X509($bio);
+ if (!$cert) {
+ Net::SSLeay::BIO_free($bio);
+ die "unable to read certificate from '$cert_path'\n";
}
- $raw .= $prefix . "}\n\n";
-
- return $raw;
-
-};
+ my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
+ Net::SSLeay::X509_free($cert);
-sub write_corosync_conf {
- my ($filename, $conf) = @_;
+ die "unable to get fingerprint for '$cert_path' - got empty value\n"
+ if !defined($fp) || $fp eq '';
- my $raw = '';
+ return $fp;
+}
- my $prefix = '';
-
- die "no main section" if $conf->{section} ne 'main';
+sub get_node_fingerprint {
+ my ($node) = @_;
- my @list = grep { defined($_->{key}) } @{$conf->{children}};
- foreach my $child (sort {$a->{key} cmp $b->{key}} @list) {
- $raw .= "$child->{key}: $child->{value}\n";
- }
+ my $cert_path = "/etc/pve/nodes/$node/pve-ssl.pem";
+ my $custom_cert_path = "/etc/pve/nodes/$node/pveproxy-ssl.pem";
- @list = grep { defined($_->{section}) } @{$conf->{children}};
- foreach my $child (sort {$a->{section} cmp $b->{section}} @list) {
- $raw .= &$dump_corosync_section($child, $prefix);
- }
+ $cert_path = $custom_cert_path if -f $custom_cert_path;
- return $raw;
+ return read_ssl_cert_fingerprint($cert_path);
}
-sub corosync_conf_version {
- my ($conf, $noerr, $new_value) = @_;
-
- foreach my $child (@{$conf->{children}}) {
- next if !defined($child->{section});
- if ($child->{section} eq 'totem') {
- foreach my $e (@{$child->{children}}) {
- next if !defined($e->{key});
- if ($e->{key} eq 'config_version') {
- if ($new_value) {
- $e->{value} = $new_value;
- return $new_value;
- } elsif (my $version = int($e->{value})) {
- return $version;
- }
- last;
- }
- }
+
+sub check_cert_fingerprint {
+ my ($cert) = @_;
+
+ # clear cache every 30 minutes at least
+ update_cert_cache(undef, 1) if time() - $cert_cache_timestamp >= 60*30;
+
+ # get fingerprint of server certificate
+ my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
+ return 0 if !defined($fp) || $fp eq ''; # error
+
+ my $check = sub {
+ for my $expected (keys %$cert_cache_fingerprints) {
+ return 1 if $fp eq $expected;
}
+ return 0;
+ };
+
+ return 1 if &$check();
+
+ # clear cache and retry at most once every minute
+ if (time() - $cert_cache_timestamp >= 60) {
+ syslog ('info', "Could not verify remote node certificate '$fp' with list of pinned certificates, refreshing cache");
+ update_cert_cache();
+ return &$check();
}
-
- return undef if $noerr;
- die "invalid corosync config - unable to read version\n";
+ return 0;
}
-# read only - use "rename corosync.conf.new corosync.conf" to write
-PVE::Cluster::cfs_register_file('corosync.conf', \&parse_corosync_conf);
-# this is read/write
-PVE::Cluster::cfs_register_file('corosync.conf.new', \&parse_corosync_conf,
- \&write_corosync_conf);
-
# bash completion helpers
sub complete_next_vmid {
return $res;
}
+sub get_ssh_info {
+ my ($node, $network_cidr) = @_;
+
+ my $ip;
+ if (defined($network_cidr)) {
+ # Use mtunnel via to get the remote node's ip inside $network_cidr.
+ # This goes over the regular network (iow. uses get_ssh_info() with
+ # $network_cidr undefined.
+ # FIXME: Use the REST API client for this after creating an API entry
+ # for get_migration_ip.
+ my $default_remote = get_ssh_info($node, undef);
+ my $default_ssh = ssh_info_to_command($default_remote);
+ my $cmd =[@$default_ssh, 'pvecm', 'mtunnel',
+ '-migration_network', $network_cidr,
+ '-get_migration_ip'
+ ];
+ PVE::Tools::run_command($cmd, outfunc => sub {
+ my ($line) = @_;
+ chomp $line;
+ die "internal error: unexpected output from mtunnel\n"
+ if defined($ip);
+ if ($line =~ /^ip: '(.*)'$/) {
+ $ip = $1;
+ } else {
+ die "internal error: bad output from mtunnel\n"
+ if defined($ip);
+ }
+ });
+ die "failed to get ip for node '$node' in network '$network_cidr'\n"
+ if !defined($ip);
+ } else {
+ $ip = remote_node_ip($node);
+ }
+
+ return {
+ ip => $ip,
+ name => $node,
+ network => $network_cidr,
+ };
+}
+
+sub ssh_info_to_command_base {
+ my ($info, @extra_options) = @_;
+ return [
+ '/usr/bin/ssh',
+ '-e', 'none',
+ '-o', 'BatchMode=yes',
+ '-o', 'HostKeyAlias='.$info->{name},
+ @extra_options
+ ];
+}
+
+sub ssh_info_to_command {
+ my ($info, @extra_options) = @_;
+ my $cmd = ssh_info_to_command_base($info, @extra_options);
+ push @$cmd, "root\@$info->{ip}";
+ return $cmd;
+}
+
+sub assert_joinable {
+ my ($ring0_addr, $ring1_addr, $force) = @_;
+
+ my $errors = '';
+ my $error = sub { $errors .= "* $_[0]\n"; };
+
+ if (-f $authfile) {
+ $error->("authentication key '$authfile' already exists");
+ }
+
+ if (-f $clusterconf) {
+ $error->("cluster config '$clusterconf' already exists");
+ }
+
+ my $vmlist = get_vmlist();
+ if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
+ $error->("this host already contains virtual guests");
+ }
+
+ if (run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
+ $error->("corosync is already running, is this node already in a cluster?!");
+ }
+
+ # check if corosync ring IPs are configured on the current nodes interfaces
+ my $check_ip = sub {
+ my $ip = shift // return;
+ if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
+ my $host = $ip;
+ eval { $ip = PVE::Network::get_ip_from_hostname($host); };
+ if ($@) {
+ $error->("cannot use '$host': $@\n") ;
+ return;
+ }
+ }
+
+ my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
+ my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
+
+ $error->("cannot use IP '$ip', it must be configured exactly once on local node!\n")
+ if (scalar(@$configured_ips) != 1);
+ };
+
+ $check_ip->($ring0_addr);
+ $check_ip->($ring1_addr);
+
+ if ($errors) {
+ warn "detected the following error(s):\n$errors";
+ die "Check if node may join a cluster failed!\n" if !$force;
+ }
+}
+
+my $backup_cfs_database = sub {
+ my ($dbfile) = @_;
+
+ mkdir $dbbackupdir;
+
+ print "backup old database\n";
+ my $ctime = time();
+ my $cmd = [
+ ['echo', '.dump'],
+ ['sqlite3', $dbfile],
+ ['gzip', '-', \ ">${dbbackupdir}/config-${ctime}.sql.gz"],
+ ];
+
+ PVE::Tools::run_command($cmd, 'errmsg' => "cannot backup old database\n");
+
+ # purge older backup
+ my $maxfiles = 10;
+ my @bklist = ();
+ foreach my $fn (<$dbbackupdir/config-*.sql.gz>) {
+ if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
+ push @bklist, [$fn, $1];
+ }
+ }
+
+ @bklist = sort { $b->[1] <=> $a->[1] } @bklist;
+ while (scalar (@bklist) >= $maxfiles) {
+ my $d = pop @bklist;
+ print "delete old backup '$d->[0]'\n";
+ unlink $d->[0];
+ }
+};
+
+sub join {
+ my ($param) = @_;
+
+ my $nodename = PVE::INotify::nodename();
+
+ setup_sshd_config();
+ setup_rootsshconfig();
+ setup_ssh_keys();
+
+ # check if we can join with the given parameters and current node state
+ my ($ring0_addr, $ring1_addr) = $param->@{'ring0_addr', 'ring1_addr'};
+ assert_joinable($ring0_addr, $ring1_addr, $param->{force});
+
+ # make sure known_hosts is on local filesystem
+ ssh_unmerge_known_hosts();
+
+ my $host = $param->{hostname};
+
+ my $conn_args = {
+ username => 'root@pam',
+ password => $param->{password},
+ cookie_name => 'PVEAuthCookie',
+ protocol => 'https',
+ host => $host,
+ port => 8006,
+ };
+
+ if (my $fp = $param->{fingerprint}) {
+ $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
+ } else {
+ # API schema ensures that we can only get here from CLI handler
+ $conn_args->{manual_verification} = 1;
+ }
+
+ print "Etablishing API connection with host '$host'\n";
+
+ my $conn = PVE::APIClient::LWP->new(%$conn_args);
+ $conn->login();
+
+ # login raises an exception on failure, so if we get here we're good
+ print "Login succeeded.\n";
+
+ my $args = {};
+ $args->{force} = $param->{force} if defined($param->{force});
+ $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
+ $args->{votes} = $param->{votes} if defined($param->{votes});
+ $args->{ring0_addr} = $ring0_addr if defined($ring0_addr);
+ $args->{ring1_addr} = $ring1_addr if defined($ring1_addr);
+
+ print "Request addition of this node\n";
+ my $res = $conn->post("/cluster/config/nodes/$nodename", $args);
+
+ print "Join request OK, finishing setup locally\n";
+
+ # added successfuly - now prepare local node
+ finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
+}
+
+sub finish_join {
+ my ($nodename, $corosync_conf, $corosync_authkey) = @_;
+
+ mkdir "$localclusterdir";
+ PVE::Tools::file_set_contents($authfile, $corosync_authkey);
+ PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
+
+ print "stopping pve-cluster service\n";
+ my $cmd = ['systemctl', 'stop', 'pve-cluster'];
+ run_command($cmd, errmsg => "can't stop pve-cluster service");
+
+ $backup_cfs_database->($dbfile);
+ unlink $dbfile;
+
+ $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
+ run_command($cmd, errmsg => "starting pve-cluster failed");
+
+ # wait for quorum
+ my $printqmsg = 1;
+ while (!check_cfs_quorum(1)) {
+ if ($printqmsg) {
+ print "waiting for quorum...";
+ STDOUT->flush();
+ $printqmsg = 0;
+ }
+ sleep(1);
+ }
+ print "OK\n" if !$printqmsg;
+
+ my $local_ip_address = remote_node_ip($nodename);
+
+ print "generating node certificates\n";
+ gen_pve_node_files($nodename, $local_ip_address);
+
+ print "merge known_hosts file\n";
+ ssh_merge_known_hosts($nodename, $local_ip_address, 1);
+
+ print "node certificate changed, restart pveproxy and pvedaemon services\n";
+ run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
+
+ print "successfully added node '$nodename' to cluster.\n";
+}
+
+
1;
projects / pve-cluster.git / blobdiff