use strict;
use warnings;
-use POSIX qw(EEXIST);
+use POSIX qw(EEXIST ENOENT);
use File::stat qw();
use Socket;
use Storable qw(dclone);
use Digest::SHA;
use Digest::HMAC_SHA1;
use Net::SSLeay;
-use PVE::Tools;
+use PVE::Tools qw(run_command);
use PVE::INotify;
use PVE::IPCC;
use PVE::SafeSyslog;
my $authdir = "$basedir/priv";
my $lockdir = "/etc/pve/priv/lock";
+# cfs and corosync files
+my $dbfile = "/var/lib/pve-cluster/config.db";
+my $dbbackupdir = "/var/lib/pve-cluster/backup";
+my $localclusterdir = "/etc/corosync";
+my $localclusterconf = "$localclusterdir/corosync.conf";
+my $authfile = "$localclusterdir/authkey";
+my $clusterconf = "$basedir/corosync.conf";
+
my $authprivkeyfn = "$authdir/authkey.key";
my $authpubkeyfn = "$basedir/authkey.pub";
my $pveca_key_fn = "$authdir/pve-root-ca.key";
'vzdump.cron' => 1,
'storage.cfg' => 1,
'datacenter.cfg' => 1,
+ 'replication.cfg' => 1,
'corosync.conf' => 1,
'corosync.conf.new' => 1,
'user.cfg' => 1,
my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
- die "ipcc_send_rec failed: $!\n" if !defined($res) && ($! != 0);
+ die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
return $res;
};
my $res = PVE::IPCC::ipcc_send_rec($msgid, $data);
- die "ipcc_send_rec failed: $!\n" if !defined($res) && ($! != 0);
+ die "ipcc_send_rec[$msgid] failed: $!\n" if !defined($res) && ($! != 0);
return decode_json($res);
};
my $bindata = pack "Z*", $path;
my $res = PVE::IPCC::ipcc_send_rec(6, $bindata);
if (!defined($res)) {
- return undef if ($! != 0);
+ if ($! != 0) {
+ return undef if $! == ENOENT;
+ die "$!\n";
+ }
return '';
}
my $ccache = {};
sub cfs_update {
+ my ($fail) = @_;
eval {
my $res = &$ipcc_send_rec_json(1);
#warn "GOT1: " . Dumper($res);
$vmlist = {};
$clinfo = {};
$ccache = {};
+ die $err if $fail;
warn $err;
}
$err = $@;
if ($err) {
$clinfo = {};
+ die $err if $fail;
warn $err;
}
$err = $@;
if ($err) {
$vmlist = {};
+ die $err if $fail;
warn $err;
}
}
return [ keys %$nodelist ];
}
+# $data must be a chronological descending ordered array of tasks
sub broadcast_tasklist {
my ($data) = @_;
+ # the serialized list may not get bigger than 32kb (CFS_MAX_STATUS_SIZE
+ # from pmxcfs) - drop older items until we satisfy this constraint
+ my $size = length(encode_json($data));
+ while ($size >= (32 * 1024)) {
+ pop @$data;
+ $size = length(encode_json($data));
+ }
+
eval {
&$ipcc_update_status("tasklist", $data);
};
push @args, '--full-size-mode';
# we do not really store data into the file
- my $res = RRDs::graphv('', @args);
+ my $res = RRDs::graphv('-', @args);
my $err = RRDs::error;
die "RRD error: $err\n" if $err;
my $cfs_lock = sub {
my ($lockid, $timeout, $code, @param) = @_;
+ my $prev_alarm = alarm(0); # suspend outer alarm early
+
my $res;
+ my $got_lock = 0;
# this timeout is for aquire the lock
$timeout = 10 if !$timeout;
my $filename = "$lockdir/$lockid";
- my $msg = "can't aquire cfs lock '$lockid'";
-
eval {
mkdir $lockdir;
if (! -d $lockdir) {
- die "$msg: pve cluster filesystem not online.\n";
+ die "pve cluster filesystem not online.\n";
}
- local $SIG{ALRM} = sub { die "got lock request timeout\n"; };
+ my $timeout_err = sub { die "got lock request timeout\n"; };
+ local $SIG{ALRM} = $timeout_err;
- alarm ($timeout);
+ while (1) {
+ alarm ($timeout);
+ $got_lock = mkdir($filename);
+ $timeout = alarm(0) - 1; # we'll sleep for 1s, see down below
- if (!(mkdir $filename)) {
- print STDERR "trying to aquire cfs lock '$lockid' ...";
- while (1) {
- if (!(mkdir $filename)) {
- (utime 0, 0, $filename); # cfs unlock request
- } else {
- print STDERR " OK\n";
- last;
- }
- sleep(1);
- }
+ last if $got_lock;
+
+ $timeout_err->() if $timeout <= 0;
+
+ print STDERR "trying to aquire cfs lock '$lockid' ...\n";
+ utime (0, 0, $filename); # cfs unlock request
+ sleep(1);
}
# fixed command timeout: cfs locks have a timeout of 120
# using 60 gives us another 60 seconds to abort the task
- alarm(60);
local $SIG{ALRM} = sub { die "got lock timeout - aborting command\n"; };
+ alarm(60);
cfs_update(); # make sure we read latest versions inside code()
my $err = $@;
- alarm(0);
+ $err = "no quorum!\n" if !$got_lock && !check_cfs_quorum(1);
- if ($err && ($err eq "got lock request timeout\n") &&
- !check_cfs_quorum()){
- $err = "$msg: no quorum!\n";
- }
+ rmdir $filename if $got_lock; # if we held the lock always unlock again
- if (!$err || $err !~ /^got lock timeout -/) {
- rmdir $filename; # cfs unlock
- }
+ alarm($prev_alarm);
if ($err) {
- $@ = $err;
+ $@ = "error with cfs lock '$lockid': $err";
return undef;
}
$family =
PVE::Tools::get_host_address_family($ip);
}
- return ($ip, $family);
+ return wantarray ? ($ip, $family) : $ip;
}
}
# fallback: try to get IP by other means
- my ($family, $packed_ip);
-
- eval {
- my @res = PVE::Tools::getaddrinfo_all($nodename);
- $family = $res[0]->{family};
- $packed_ip = (PVE::Tools::unpack_sockaddr_in46($res[0]->{addr}))[2];
- };
-
- if ($@) {
- die "hostname lookup failed:\n$@" if !$noerr;
- return undef;
- }
-
- my $ip = Socket::inet_ntop($family, $packed_ip);
- if ($ip =~ m/^127\.|^::1$/) {
- die "hostname lookup failed - got local IP address ($nodename = $ip)\n" if !$noerr;
- return undef;
- }
-
- return wantarray ? ($ip, $family) : $ip;
+ return PVE::Network::get_ip_from_hostname($nodename, $noerr);
}
sub get_local_migration_ip {
}
sub setup_sshd_config {
- my ($start_sshd) = @_;
+ my () = @_;
my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
PVE::Tools::file_set_contents($sshd_config_fn, $conf);
- my $cmd = $start_sshd ? 'reload-or-restart' : 'reload-or-try-restart';
- PVE::Tools::run_command(['systemctl', $cmd, 'sshd']);
+ PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
}
sub setup_rootsshconfig {
if (! -f $rootsshconfig) {
mkdir '/root/.ssh';
if (my $fh = IO::File->new($rootsshconfig, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
- # this is the default ciphers list from debian openssl0.9.8 except blowfish is added as prefered
- print $fh "Ciphers blowfish-cbc,aes128-ctr,aes192-ctr,aes256-ctr,arcfour256,arcfour128,aes128-cbc,3des-cbc\n";
+ # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k 26 Jan 2017)
+ # changed order to put AES before Chacha20 (most hardware has AESNI)
+ print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
close($fh);
}
}
die "no node name specified" if !$nodename;
die "no ip address specified" if !$ip_address;
+ # ssh lowercases hostnames (aliases) before comparision, so we need too
+ $nodename = lc($nodename);
+ $ip_address = lc($ip_address);
+
mkdir $authdir;
if (! -f $sshknownhosts) {
my $merge_line = sub {
my ($line, $all) = @_;
+ return if $line =~ m/^\s*$/; # skip empty lines
+ return if $line =~ m/^#/; # skip comments
+
if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
my $key = $1;
my $rsakey = $2;
}
return;
}
+ } else {
+ $key = lc($key); # avoid duplicate entries, ssh compares lowercased
+ if ($key eq $ip_address) {
+ $found_local_ip = 1 if $rsakey eq $hostkey;
+ } elsif ($key eq $nodename) {
+ $found_nodename = 1 if $rsakey eq $hostkey;
+ }
}
$data .= $line;
}
while ($old && $old =~ s/^((.*?)(\n|$))//) {
my $line = "$2\n";
- next if $line =~ m/^\s*$/; # skip empty lines
- next if $line =~ m/^#/; # skip comments
&$merge_line($line, 1);
}
while ($new && $new =~ s/^((.*?)(\n|$))//) {
my $line = "$2\n";
- next if $line =~ m/^\s*$/; # skip empty lines
- next if $line =~ m/^#/; # skip comments
&$merge_line($line);
}
- my $addIndex = $$;
- my $add_known_hosts_entry = sub {
- my ($name, $hostkey) = @_;
- $addIndex++;
- my $hmac = Digest::HMAC_SHA1->new("$addIndex" . time());
- my $b64salt = $hmac->b64digest . '=';
- $hmac = Digest::HMAC_SHA1->new(decode_base64($b64salt));
- $hmac->add($name);
- my $digest = $hmac->b64digest . '=';
- $data .= "|1|$b64salt|$digest $hostkey\n";
- };
-
- if (!$found_nodename || !$found_local_ip) {
- &$add_known_hosts_entry($nodename, $hostkey) if !$found_nodename;
- &$add_known_hosts_entry($ip_address, $hostkey) if !$found_local_ip;
- }
+ # add our own key if not already there
+ $data .= "$nodename $hostkey\n" if !$found_nodename;
+ $data .= "$ip_address $hostkey\n" if !$found_local_ip;
PVE::Tools::file_set_contents($sshknownhosts, $data);
console => {
optional => 1,
type => 'string',
- description => "Select the default Console viewer. You can either use the builtin java applet (VNC), an external virt-viewer comtatible application (SPICE), or an HTML5 based viewer (noVNC).",
+ description => "Select the default Console viewer. You can either use the builtin java applet (VNC; deprecated and maps to html5), an external virt-viewer comtatible application (SPICE), or an HTML5 based viewer (noVNC).",
enum => ['applet', 'vv', 'html5'],
},
email_from => {
pattern => qr/[a-f0-9]{2}(?::[a-f0-9]{2}){0,2}:?/i,
description => 'Prefix for autogenerated MAC addresses.',
},
+ bwlimit => PVE::JSONSchema::get_standard_option('bwlimit'),
},
};
}
}
+ # for backwards compatibility only, applet maps to html5
+ if (defined($res->{console}) && $res->{console} eq 'applet') {
+ $res->{console} = 'html5';
+ }
+
return $res;
}
$cfg->{migration}->{type} = ($migration_unsecure) ? 'insecure' : 'secure';
}
- return PVE::JSONSchema::dump_config($datacenter_schema, $filename, $cfg);
-}
-
-cfs_register_file('datacenter.cfg',
- \&parse_datacenter_config,
- \&write_datacenter_config);
-
-# a very simply parser ...
-sub parse_corosync_conf {
- my ($filename, $raw) = @_;
-
- return {} if !$raw;
-
- my $digest = Digest::SHA::sha1_hex(defined($raw) ? $raw : '');
-
- $raw =~ s/#.*$//mg;
- $raw =~ s/\r?\n/ /g;
- $raw =~ s/\s+/ /g;
- $raw =~ s/^\s+//;
- $raw =~ s/\s*$//;
-
- my @tokens = split(/\s/, $raw);
-
- my $conf = { section => 'main', children => [] };
-
- my $stack = [];
- my $section = $conf;
-
- while (defined(my $token = shift @tokens)) {
- my $nexttok = $tokens[0];
-
- if ($nexttok && ($nexttok eq '{')) {
- shift @tokens; # skip '{'
- my $new_section = {
- section => $token,
- children => [],
- };
- push @{$section->{children}}, $new_section;
- push @$stack, $section;
- $section = $new_section;
- next;
- }
-
- if ($token eq '}') {
- $section = pop @$stack;
- die "parse error - uncexpected '}'\n" if !$section;
- next;
- }
-
- my $key = $token;
- die "missing ':' after key '$key'\n" if ! ($key =~ s/:$//);
-
- die "parse error - no value for '$key'\n" if !defined($nexttok);
- my $value = shift @tokens;
-
- push @{$section->{children}}, { key => $key, value => $value };
+ # map deprecated applet setting to html5
+ if (defined($cfg->{console}) && $cfg->{console} eq 'applet') {
+ $cfg->{console} = 'html5';
}
- $conf->{digest} = $digest;
-
- return $conf;
-}
-
-my $dump_corosync_section;
-$dump_corosync_section = sub {
- my ($section, $prefix) = @_;
-
- my $raw = $prefix . $section->{section} . " {\n";
-
- my @list = grep { defined($_->{key}) } @{$section->{children}};
- foreach my $child (sort {$a->{key} cmp $b->{key}} @list) {
- $raw .= $prefix . " $child->{key}: $child->{value}\n";
- }
-
- @list = grep { defined($_->{section}) } @{$section->{children}};
- foreach my $child (sort {$a->{section} cmp $b->{section}} @list) {
- $raw .= &$dump_corosync_section($child, "$prefix ");
- }
-
- $raw .= $prefix . "}\n\n";
-
- return $raw;
-
-};
-
-sub write_corosync_conf {
- my ($filename, $conf) = @_;
-
- my $raw = '';
-
- my $prefix = '';
-
- die "no main section" if $conf->{section} ne 'main';
-
- my @list = grep { defined($_->{key}) } @{$conf->{children}};
- foreach my $child (sort {$a->{key} cmp $b->{key}} @list) {
- $raw .= "$child->{key}: $child->{value}\n";
- }
-
- @list = grep { defined($_->{section}) } @{$conf->{children}};
- foreach my $child (sort {$a->{section} cmp $b->{section}} @list) {
- $raw .= &$dump_corosync_section($child, $prefix);
+ if (my $migration = $cfg->{migration}) {
+ $cfg->{migration} = PVE::JSONSchema::print_property_string($migration, $migration_format);
}
- return $raw;
-}
-
-sub corosync_conf_version {
- my ($conf, $noerr, $new_value) = @_;
-
- foreach my $child (@{$conf->{children}}) {
- next if !defined($child->{section});
- if ($child->{section} eq 'totem') {
- foreach my $e (@{$child->{children}}) {
- next if !defined($e->{key});
- if ($e->{key} eq 'config_version') {
- if ($new_value) {
- $e->{value} = $new_value;
- return $new_value;
- } elsif (my $version = int($e->{value})) {
- return $version;
- }
- last;
- }
- }
- }
- }
-
- return undef if $noerr;
-
- die "invalid corosync config - unable to read version\n";
-}
-
-# read only - use "rename corosync.conf.new corosync.conf" to write
-PVE::Cluster::cfs_register_file('corosync.conf', \&parse_corosync_conf);
-# this is read/write
-PVE::Cluster::cfs_register_file('corosync.conf.new', \&parse_corosync_conf,
- \&write_corosync_conf);
-
-sub check_corosync_conf_exists {
- my ($silent) = @_;
-
- $silent = $silent // 0;
-
- my $exists = -f "$basedir/corosync.conf";
-
- warn "Corosync config '$basedir/corosync.conf' does not exist - is this node part of a cluster?\n"
- if !$silent && !$exists;
-
- return $exists;
-}
-
-sub corosync_update_nodelist {
- my ($conf, $nodelist) = @_;
-
- delete $conf->{digest};
-
- my $version = corosync_conf_version($conf);
- corosync_conf_version($conf, undef, $version + 1);
-
- my $children = [];
- foreach my $v (values %$nodelist) {
- next if !($v->{ring0_addr} || $v->{name});
- my $kv = [];
- foreach my $k (keys %$v) {
- push @$kv, { key => $k, value => $v->{$k} };
- }
- my $ns = { section => 'node', children => $kv };
- push @$children, $ns;
- }
-
- foreach my $main (@{$conf->{children}}) {
- next if !defined($main->{section});
- if ($main->{section} eq 'nodelist') {
- $main->{children} = $children;
- last;
- }
- }
-
-
- cfs_write_file("corosync.conf.new", $conf);
-
- rename("/etc/pve/corosync.conf.new", "/etc/pve/corosync.conf")
- || die "activate corosync.conf.new failed - $!\n";
-}
-
-sub corosync_nodelist {
- my ($conf) = @_;
-
- my $nodelist = {};
-
- foreach my $main (@{$conf->{children}}) {
- next if !defined($main->{section});
- if ($main->{section} eq 'nodelist') {
- foreach my $ne (@{$main->{children}}) {
- next if !defined($ne->{section}) || ($ne->{section} ne 'node');
- my $node = { quorum_votes => 1 };
- my $name;
- foreach my $child (@{$ne->{children}}) {
- next if !defined($child->{key});
- $node->{$child->{key}} = $child->{value};
- # use 'name' over 'ring0_addr' if set
- if ($child->{key} eq 'name') {
- delete $nodelist->{$name} if $name;
- $name = $child->{value};
- $nodelist->{$name} = $node;
- } elsif(!$name && $child->{key} eq 'ring0_addr') {
- $name = $child->{value};
- $nodelist->{$name} = $node;
- }
- }
- }
- }
- }
-
- return $nodelist;
+ return PVE::JSONSchema::dump_config($datacenter_schema, $filename, $cfg);
}
-# get a hash representation of the corosync config totem section
-sub corosync_totem_config {
- my ($conf) = @_;
-
- my $res = {};
-
- foreach my $main (@{$conf->{children}}) {
- next if !defined($main->{section}) ||
- $main->{section} ne 'totem';
-
- foreach my $e (@{$main->{children}}) {
-
- if ($e->{section} && $e->{section} eq 'interface') {
- my $entry = {};
-
- $res->{interface} = {};
-
- foreach my $child (@{$e->{children}}) {
- next if !defined($child->{key});
- $entry->{$child->{key}} = $child->{value};
- if($child->{key} eq 'ringnumber') {
- $res->{interface}->{$child->{value}} = $entry;
- }
- }
-
- } elsif ($e->{key}) {
- $res->{$e->{key}} = $e->{value};
- }
- }
- }
-
- return $res;
-}
+cfs_register_file('datacenter.cfg',
+ \&parse_datacenter_config,
+ \&write_datacenter_config);
# X509 Certificate cache helper
}
};
- my $cert_path = "/etc/pve/nodes/$node/pve-ssl.pem";
- my $custom_cert_path = "/etc/pve/nodes/$node/pveproxy-ssl.pem";
-
- $cert_path = $custom_cert_path if -f $custom_cert_path;
-
- my $cert;
- eval {
- my $bio = Net::SSLeay::BIO_new_file($cert_path, 'r');
- $cert = Net::SSLeay::PEM_read_bio_X509($bio);
- Net::SSLeay::BIO_free($bio);
- };
- my $err = $@;
- if ($err || !defined($cert)) {
- &$clear_old() if $clear;
- next;
- }
-
- my $fp;
- eval {
- $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
- };
- $err = $@;
- if ($err || !defined($fp) || $fp eq '') {
+ my $fp = eval { get_node_fingerprint($node) };
+ if (my $err = $@) {
+ warn "$err\n";
&$clear_old() if $clear;
next;
}
if defined($node) && !defined($cert_cache_nodes->{$node});
}
+sub read_ssl_cert_fingerprint {
+ my ($cert_path) = @_;
+
+ my $bio = Net::SSLeay::BIO_new_file($cert_path, 'r')
+ or die "unable to read '$cert_path' - $!\n";
+
+ my $cert = Net::SSLeay::PEM_read_bio_X509($bio);
+ if (!$cert) {
+ Net::SSLeay::BIO_free($bio);
+ die "unable to read certificate from '$cert_path'\n";
+ }
+
+ my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
+ Net::SSLeay::X509_free($cert);
+
+ die "unable to get fingerprint for '$cert_path' - got empty value\n"
+ if !defined($fp) || $fp eq '';
+
+ return $fp;
+}
+
+sub get_node_fingerprint {
+ my ($node) = @_;
+
+ my $cert_path = "/etc/pve/nodes/$node/pve-ssl.pem";
+ my $custom_cert_path = "/etc/pve/nodes/$node/pveproxy-ssl.pem";
+
+ $cert_path = $custom_cert_path if -f $custom_cert_path;
+
+ return read_ssl_cert_fingerprint($cert_path);
+}
+
+
sub check_cert_fingerprint {
my ($cert) = @_;
update_cert_cache(undef, 1) if time() - $cert_cache_timestamp >= 60*30;
# get fingerprint of server certificate
- my $fp;
- eval {
- $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
- };
- return 0 if $@ || !defined($fp) || $fp eq ''; # error
+ my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
+ return 0 if !defined($fp) || $fp eq ''; # error
my $check = sub {
for my $expected (keys %$cert_cache_fingerprints) {
return $res;
}
+sub get_ssh_info {
+ my ($node, $network_cidr) = @_;
+
+ my $ip;
+ if (defined($network_cidr)) {
+ # Use mtunnel via to get the remote node's ip inside $network_cidr.
+ # This goes over the regular network (iow. uses get_ssh_info() with
+ # $network_cidr undefined.
+ # FIXME: Use the REST API client for this after creating an API entry
+ # for get_migration_ip.
+ my $default_remote = get_ssh_info($node, undef);
+ my $default_ssh = ssh_info_to_command($default_remote);
+ my $cmd =[@$default_ssh, 'pvecm', 'mtunnel',
+ '-migration_network', $network_cidr,
+ '-get_migration_ip'
+ ];
+ PVE::Tools::run_command($cmd, outfunc => sub {
+ my ($line) = @_;
+ chomp $line;
+ die "internal error: unexpected output from mtunnel\n"
+ if defined($ip);
+ if ($line =~ /^ip: '(.*)'$/) {
+ $ip = $1;
+ } else {
+ die "internal error: bad output from mtunnel\n"
+ if defined($ip);
+ }
+ });
+ die "failed to get ip for node '$node' in network '$network_cidr'\n"
+ if !defined($ip);
+ } else {
+ $ip = remote_node_ip($node);
+ }
+
+ return {
+ ip => $ip,
+ name => $node,
+ network => $network_cidr,
+ };
+}
+
+sub ssh_info_to_command_base {
+ my ($info, @extra_options) = @_;
+ return [
+ '/usr/bin/ssh',
+ '-e', 'none',
+ '-o', 'BatchMode=yes',
+ '-o', 'HostKeyAlias='.$info->{name},
+ @extra_options
+ ];
+}
+
+sub ssh_info_to_command {
+ my ($info, @extra_options) = @_;
+ my $cmd = ssh_info_to_command_base($info, @extra_options);
+ push @$cmd, "root\@$info->{ip}";
+ return $cmd;
+}
+
+sub assert_joinable {
+ my ($ring0_addr, $ring1_addr, $force) = @_;
+
+ my $errors = '';
+ my $error = sub { $errors .= "* $_[0]\n"; };
+
+ if (-f $authfile) {
+ $error->("authentication key '$authfile' already exists");
+ }
+
+ if (-f $clusterconf) {
+ $error->("cluster config '$clusterconf' already exists");
+ }
+
+ my $vmlist = get_vmlist();
+ if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
+ $error->("this host already contains virtual guests");
+ }
+
+ if (run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
+ $error->("corosync is already running, is this node already in a cluster?!");
+ }
+
+ # check if corosync ring IPs are configured on the current nodes interfaces
+ my $check_ip = sub {
+ my $ip = shift // return;
+ if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
+ my $host = $ip;
+ eval { $ip = PVE::Network::get_ip_from_hostname($host); };
+ if ($@) {
+ $error->("cannot use '$host': $@\n") ;
+ return;
+ }
+ }
+
+ my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
+ my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
+
+ $error->("cannot use IP '$ip', it must be configured exactly once on local node!\n")
+ if (scalar(@$configured_ips) != 1);
+ };
+
+ $check_ip->($ring0_addr);
+ $check_ip->($ring1_addr);
+
+ if ($errors) {
+ warn "detected the following error(s):\n$errors";
+ die "Check if node may join a cluster failed!\n" if !$force;
+ }
+}
+
+my $backup_cfs_database = sub {
+ my ($dbfile) = @_;
+
+ mkdir $dbbackupdir;
+
+ print "backup old database\n";
+ my $ctime = time();
+ my $cmd = [
+ ['echo', '.dump'],
+ ['sqlite3', $dbfile],
+ ['gzip', '-', \ ">${dbbackupdir}/config-${ctime}.sql.gz"],
+ ];
+
+ PVE::Tools::run_command($cmd, 'errmsg' => "cannot backup old database\n");
+
+ # purge older backup
+ my $maxfiles = 10;
+ my @bklist = ();
+ foreach my $fn (<$dbbackupdir/config-*.sql.gz>) {
+ if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
+ push @bklist, [$fn, $1];
+ }
+ }
+
+ @bklist = sort { $b->[1] <=> $a->[1] } @bklist;
+ while (scalar (@bklist) >= $maxfiles) {
+ my $d = pop @bklist;
+ print "delete old backup '$d->[0]'\n";
+ unlink $d->[0];
+ }
+};
+
+sub join {
+ my ($param) = @_;
+
+ my $nodename = PVE::INotify::nodename();
+
+ setup_sshd_config();
+ setup_rootsshconfig();
+ setup_ssh_keys();
+
+ # check if we can join with the given parameters and current node state
+ my ($ring0_addr, $ring1_addr) = $param->@{'ring0_addr', 'ring1_addr'};
+ assert_joinable($ring0_addr, $ring1_addr, $param->{force});
+
+ # make sure known_hosts is on local filesystem
+ ssh_unmerge_known_hosts();
+
+ my $host = $param->{hostname};
+
+ my $conn_args = {
+ username => 'root@pam',
+ password => $param->{password},
+ cookie_name => 'PVEAuthCookie',
+ protocol => 'https',
+ host => $host,
+ port => 8006,
+ };
+
+ if (my $fp = $param->{fingerprint}) {
+ $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
+ } else {
+ # API schema ensures that we can only get here from CLI handler
+ $conn_args->{manual_verification} = 1;
+ }
+
+ print "Etablishing API connection with host '$host'\n";
+
+ my $conn = PVE::APIClient::LWP->new(%$conn_args);
+ $conn->login();
+
+ # login raises an exception on failure, so if we get here we're good
+ print "Login succeeded.\n";
+
+ my $args = {};
+ $args->{force} = $param->{force} if defined($param->{force});
+ $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
+ $args->{votes} = $param->{votes} if defined($param->{votes});
+ $args->{ring0_addr} = $ring0_addr if defined($ring0_addr);
+ $args->{ring1_addr} = $ring1_addr if defined($ring1_addr);
+
+ print "Request addition of this node\n";
+ my $res = $conn->post("/cluster/config/nodes/$nodename", $args);
+
+ print "Join request OK, finishing setup locally\n";
+
+ # added successfuly - now prepare local node
+ finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
+}
+
+sub finish_join {
+ my ($nodename, $corosync_conf, $corosync_authkey) = @_;
+
+ mkdir "$localclusterdir";
+ PVE::Tools::file_set_contents($authfile, $corosync_authkey);
+ PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
+
+ print "stopping pve-cluster service\n";
+ my $cmd = ['systemctl', 'stop', 'pve-cluster'];
+ run_command($cmd, errmsg => "can't stop pve-cluster service");
+
+ $backup_cfs_database->($dbfile);
+ unlink $dbfile;
+
+ $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
+ run_command($cmd, errmsg => "starting pve-cluster failed");
+
+ # wait for quorum
+ my $printqmsg = 1;
+ while (!check_cfs_quorum(1)) {
+ if ($printqmsg) {
+ print "waiting for quorum...";
+ STDOUT->flush();
+ $printqmsg = 0;
+ }
+ sleep(1);
+ }
+ print "OK\n" if !$printqmsg;
+
+ my $local_ip_address = remote_node_ip($nodename);
+
+ print "generating node certificates\n";
+ gen_pve_node_files($nodename, $local_ip_address);
+
+ print "merge known_hosts file\n";
+ ssh_merge_known_hosts($nodename, $local_ip_address, 1);
+
+ print "node certificate changed, restart pveproxy and pvedaemon services\n";
+ run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
+
+ print "successfully added node '$nodename' to cluster.\n";
+}
+
+
1;
projects / pve-cluster.git / blobdiff