use Digest::SHA;
use Digest::HMAC_SHA1;
use Net::SSLeay;
-use PVE::Tools;
+use PVE::Tools qw(run_command);
use PVE::INotify;
use PVE::IPCC;
use PVE::SafeSyslog;
my $authdir = "$basedir/priv";
my $lockdir = "/etc/pve/priv/lock";
+# cfs and corosync files
+my $dbfile = "/var/lib/pve-cluster/config.db";
+my $dbbackupdir = "/var/lib/pve-cluster/backup";
+my $localclusterdir = "/etc/corosync";
+my $localclusterconf = "$localclusterdir/corosync.conf";
+my $authfile = "$localclusterdir/authkey";
+my $clusterconf = "$basedir/corosync.conf";
+
my $authprivkeyfn = "$authdir/authkey.key";
my $authpubkeyfn = "$basedir/authkey.pub";
my $pveca_key_fn = "$authdir/pve-root-ca.key";
my $cfs_lock = sub {
my ($lockid, $timeout, $code, @param) = @_;
+ my $prev_alarm = alarm(0); # suspend outer alarm early
+
my $res;
+ my $got_lock = 0;
# this timeout is for aquire the lock
$timeout = 10 if !$timeout;
my $filename = "$lockdir/$lockid";
- my $msg = "can't aquire cfs lock '$lockid'";
-
eval {
mkdir $lockdir;
if (! -d $lockdir) {
- die "$msg: pve cluster filesystem not online.\n";
+ die "pve cluster filesystem not online.\n";
}
- local $SIG{ALRM} = sub { die "got lock request timeout\n"; };
+ my $timeout_err = sub { die "got lock request timeout\n"; };
+ local $SIG{ALRM} = $timeout_err;
- alarm ($timeout);
+ while (1) {
+ alarm ($timeout);
+ $got_lock = mkdir($filename);
+ $timeout = alarm(0) - 1; # we'll sleep for 1s, see down below
- if (!(mkdir $filename)) {
- print STDERR "trying to aquire cfs lock '$lockid' ...";
- while (1) {
- if (!(mkdir $filename)) {
- (utime 0, 0, $filename); # cfs unlock request
- } else {
- print STDERR " OK\n";
- last;
- }
- sleep(1);
- }
+ last if $got_lock;
+
+ $timeout_err->() if $timeout <= 0;
+
+ print STDERR "trying to aquire cfs lock '$lockid' ...\n";
+ utime (0, 0, $filename); # cfs unlock request
+ sleep(1);
}
# fixed command timeout: cfs locks have a timeout of 120
# using 60 gives us another 60 seconds to abort the task
- alarm(60);
local $SIG{ALRM} = sub { die "got lock timeout - aborting command\n"; };
+ alarm(60);
cfs_update(); # make sure we read latest versions inside code()
my $err = $@;
- alarm(0);
+ $err = "no quorum!\n" if !$got_lock && !check_cfs_quorum(1);
- if ($err && ($err eq "got lock request timeout\n") &&
- !check_cfs_quorum()){
- $err = "$msg: no quorum!\n";
- }
+ rmdir $filename if $got_lock; # if we held the lock always unlock again
- if (!$err || $err !~ /^got lock timeout -/) {
- rmdir $filename; # cfs unlock
- }
+ alarm($prev_alarm);
if ($err) {
- $@ = $err;
+ $@ = "error with cfs lock '$lockid': $err";
return undef;
}
}
sub setup_sshd_config {
- my ($start_sshd) = @_;
+ my () = @_;
my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
PVE::Tools::file_set_contents($sshd_config_fn, $conf);
- my $cmd = $start_sshd ? 'reload-or-restart' : 'reload-or-try-restart';
- PVE::Tools::run_command(['systemctl', $cmd, 'sshd']);
+ PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
}
sub setup_rootsshconfig {
console => {
optional => 1,
type => 'string',
- description => "Select the default Console viewer. You can either use the builtin java applet (VNC), an external virt-viewer comtatible application (SPICE), or an HTML5 based viewer (noVNC).",
+ description => "Select the default Console viewer. You can either use the builtin java applet (VNC; deprecated and maps to html5), an external virt-viewer comtatible application (SPICE), or an HTML5 based viewer (noVNC).",
enum => ['applet', 'vv', 'html5'],
},
email_from => {
pattern => qr/[a-f0-9]{2}(?::[a-f0-9]{2}){0,2}:?/i,
description => 'Prefix for autogenerated MAC addresses.',
},
+ bwlimit => PVE::JSONSchema::get_standard_option('bwlimit'),
},
};
}
}
+ # for backwards compatibility only, applet maps to html5
+ if (defined($res->{console}) && $res->{console} eq 'applet') {
+ $res->{console} = 'html5';
+ }
+
return $res;
}
$cfg->{migration}->{type} = ($migration_unsecure) ? 'insecure' : 'secure';
}
+ # map deprecated applet setting to html5
+ if (defined($cfg->{console}) && $cfg->{console} eq 'applet') {
+ $cfg->{console} = 'html5';
+ }
+
+ if (my $migration = $cfg->{migration}) {
+ $cfg->{migration} = PVE::JSONSchema::print_property_string($migration, $migration_format);
+ }
+
return PVE::JSONSchema::dump_config($datacenter_schema, $filename, $cfg);
}
}
};
- my $cert_path = "/etc/pve/nodes/$node/pve-ssl.pem";
- my $custom_cert_path = "/etc/pve/nodes/$node/pveproxy-ssl.pem";
-
- $cert_path = $custom_cert_path if -f $custom_cert_path;
-
- my $cert;
- eval {
- my $bio = Net::SSLeay::BIO_new_file($cert_path, 'r');
- $cert = Net::SSLeay::PEM_read_bio_X509($bio);
- Net::SSLeay::BIO_free($bio);
- };
- my $err = $@;
- if ($err || !defined($cert)) {
- &$clear_old() if $clear;
- next;
- }
-
- my $fp;
- eval {
- $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
- };
- $err = $@;
- if ($err || !defined($fp) || $fp eq '') {
+ my $fp = eval { get_node_fingerprint($node) };
+ if (my $err = $@) {
+ warn "$err\n";
&$clear_old() if $clear;
next;
}
if defined($node) && !defined($cert_cache_nodes->{$node});
}
+sub read_ssl_cert_fingerprint {
+ my ($cert_path) = @_;
+
+ my $bio = Net::SSLeay::BIO_new_file($cert_path, 'r')
+ or die "unable to read '$cert_path' - $!\n";
+
+ my $cert = Net::SSLeay::PEM_read_bio_X509($bio);
+ if (!$cert) {
+ Net::SSLeay::BIO_free($bio);
+ die "unable to read certificate from '$cert_path'\n";
+ }
+
+ my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
+ Net::SSLeay::X509_free($cert);
+
+ die "unable to get fingerprint for '$cert_path' - got empty value\n"
+ if !defined($fp) || $fp eq '';
+
+ return $fp;
+}
+
+sub get_node_fingerprint {
+ my ($node) = @_;
+
+ my $cert_path = "/etc/pve/nodes/$node/pve-ssl.pem";
+ my $custom_cert_path = "/etc/pve/nodes/$node/pveproxy-ssl.pem";
+
+ $cert_path = $custom_cert_path if -f $custom_cert_path;
+
+ return read_ssl_cert_fingerprint($cert_path);
+}
+
+
sub check_cert_fingerprint {
my ($cert) = @_;
update_cert_cache(undef, 1) if time() - $cert_cache_timestamp >= 60*30;
# get fingerprint of server certificate
- my $fp;
- eval {
- $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
- };
- return 0 if $@ || !defined($fp) || $fp eq ''; # error
+ my $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
+ return 0 if !defined($fp) || $fp eq ''; # error
my $check = sub {
for my $expected (keys %$cert_cache_fingerprints) {
my ($info, @extra_options) = @_;
return [
'/usr/bin/ssh',
+ '-e', 'none',
'-o', 'BatchMode=yes',
'-o', 'HostKeyAlias='.$info->{name},
@extra_options
return $cmd;
}
+sub assert_joinable {
+ my ($ring0_addr, $ring1_addr, $force) = @_;
+
+ my $errors = '';
+ my $error = sub { $errors .= "* $_[0]\n"; };
+
+ if (-f $authfile) {
+ $error->("authentication key '$authfile' already exists");
+ }
+
+ if (-f $clusterconf) {
+ $error->("cluster config '$clusterconf' already exists");
+ }
+
+ my $vmlist = get_vmlist();
+ if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
+ $error->("this host already contains virtual guests");
+ }
+
+ if (run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
+ $error->("corosync is already running, is this node already in a cluster?!");
+ }
+
+ # check if corosync ring IPs are configured on the current nodes interfaces
+ my $check_ip = sub {
+ my $ip = shift // return;
+ if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
+ my $host = $ip;
+ eval { $ip = PVE::Network::get_ip_from_hostname($host); };
+ if ($@) {
+ $error->("cannot use '$host': $@\n") ;
+ return;
+ }
+ }
+
+ my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
+ my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
+
+ $error->("cannot use IP '$ip', it must be configured exactly once on local node!\n")
+ if (scalar(@$configured_ips) != 1);
+ };
+
+ $check_ip->($ring0_addr);
+ $check_ip->($ring1_addr);
+
+ if ($errors) {
+ warn "detected the following error(s):\n$errors";
+ die "Check if node may join a cluster failed!\n" if !$force;
+ }
+}
+
+my $backup_cfs_database = sub {
+ my ($dbfile) = @_;
+
+ mkdir $dbbackupdir;
+
+ print "backup old database\n";
+ my $ctime = time();
+ my $cmd = [
+ ['echo', '.dump'],
+ ['sqlite3', $dbfile],
+ ['gzip', '-', \ ">${dbbackupdir}/config-${ctime}.sql.gz"],
+ ];
+
+ PVE::Tools::run_command($cmd, 'errmsg' => "cannot backup old database\n");
+
+ # purge older backup
+ my $maxfiles = 10;
+ my @bklist = ();
+ foreach my $fn (<$dbbackupdir/config-*.sql.gz>) {
+ if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
+ push @bklist, [$fn, $1];
+ }
+ }
+
+ @bklist = sort { $b->[1] <=> $a->[1] } @bklist;
+ while (scalar (@bklist) >= $maxfiles) {
+ my $d = pop @bklist;
+ print "delete old backup '$d->[0]'\n";
+ unlink $d->[0];
+ }
+};
+
+sub join {
+ my ($param) = @_;
+
+ my $nodename = PVE::INotify::nodename();
+
+ setup_sshd_config();
+ setup_rootsshconfig();
+ setup_ssh_keys();
+
+ # check if we can join with the given parameters and current node state
+ my ($ring0_addr, $ring1_addr) = $param->@{'ring0_addr', 'ring1_addr'};
+ assert_joinable($ring0_addr, $ring1_addr, $param->{force});
+
+ # make sure known_hosts is on local filesystem
+ ssh_unmerge_known_hosts();
+
+ my $host = $param->{hostname};
+
+ my $conn_args = {
+ username => 'root@pam',
+ password => $param->{password},
+ cookie_name => 'PVEAuthCookie',
+ protocol => 'https',
+ host => $host,
+ port => 8006,
+ };
+
+ if (my $fp = $param->{fingerprint}) {
+ $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
+ } else {
+ # API schema ensures that we can only get here from CLI handler
+ $conn_args->{manual_verification} = 1;
+ }
+
+ print "Etablishing API connection with host '$host'\n";
+
+ my $conn = PVE::APIClient::LWP->new(%$conn_args);
+ $conn->login();
+
+ # login raises an exception on failure, so if we get here we're good
+ print "Login succeeded.\n";
+
+ my $args = {};
+ $args->{force} = $param->{force} if defined($param->{force});
+ $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
+ $args->{votes} = $param->{votes} if defined($param->{votes});
+ $args->{ring0_addr} = $ring0_addr if defined($ring0_addr);
+ $args->{ring1_addr} = $ring1_addr if defined($ring1_addr);
+
+ print "Request addition of this node\n";
+ my $res = $conn->post("/cluster/config/nodes/$nodename", $args);
+
+ print "Join request OK, finishing setup locally\n";
+
+ # added successfuly - now prepare local node
+ finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
+}
+
+sub finish_join {
+ my ($nodename, $corosync_conf, $corosync_authkey) = @_;
+
+ mkdir "$localclusterdir";
+ PVE::Tools::file_set_contents($authfile, $corosync_authkey);
+ PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
+
+ print "stopping pve-cluster service\n";
+ my $cmd = ['systemctl', 'stop', 'pve-cluster'];
+ run_command($cmd, errmsg => "can't stop pve-cluster service");
+
+ $backup_cfs_database->($dbfile);
+ unlink $dbfile;
+
+ $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
+ run_command($cmd, errmsg => "starting pve-cluster failed");
+
+ # wait for quorum
+ my $printqmsg = 1;
+ while (!check_cfs_quorum(1)) {
+ if ($printqmsg) {
+ print "waiting for quorum...";
+ STDOUT->flush();
+ $printqmsg = 0;
+ }
+ sleep(1);
+ }
+ print "OK\n" if !$printqmsg;
+
+ my $local_ip_address = remote_node_ip($nodename);
+
+ print "generating node certificates\n";
+ gen_pve_node_files($nodename, $local_ip_address);
+
+ print "merge known_hosts file\n";
+ ssh_merge_known_hosts($nodename, $local_ip_address, 1);
+
+ print "node certificate changed, restart pveproxy and pvedaemon services\n";
+ run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
+
+ print "successfully added node '$nodename' to cluster.\n";
+}
+
+
1;
projects / pve-cluster.git / blobdiff