use strict;
use warnings;
-use Getopt::Long;
-use Socket;
-use IO::File;
-use Net::IP;
+
use File::Path;
use File::Basename;
-use Data::Dumper; # fixme: remove
-use PVE::Tools;
+use PVE::Tools qw(run_command);
use PVE::Cluster;
use PVE::INotify;
-use PVE::JSONSchema;
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::RPCEnvironment;
use PVE::CLIHandler;
+use PVE::PTY;
+use PVE::API2::ClusterConfig;
+use PVE::Corosync;
+use PVE::Cluster::Setup;
use base qw(PVE::CLIHandler);
my $basedir = "/etc/pve";
my $clusterconf = "$basedir/corosync.conf";
my $libdir = "/var/lib/pve-cluster";
-my $backupdir = "/var/lib/pve-cluster/backup";
-my $dbfile = "$libdir/config.db";
my $authfile = "/etc/corosync/authkey";
-sub backup_database {
-
- print "backup old database\n";
-
- mkdir $backupdir;
-
- my $ctime = time();
- my $cmd = [
- ['echo', '.dump'],
- ['sqlite3', $dbfile],
- ['gzip', '-', \">${backupdir}/config-${ctime}.sql.gz"],
- ];
-
- PVE::Tools::run_command($cmd, 'errmsg' => "can't backup old database\n");
-
- # purge older backup
- my $maxfiles = 10;
-
- my @bklist = ();
- foreach my $fn (<$backupdir/config-*.sql.gz>) {
- if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
- push @bklist, [$fn, $1];
- }
- }
-
- @bklist = sort { $b->[1] <=> $a->[1] } @bklist;
- while (scalar (@bklist) >= $maxfiles) {
- my $d = pop @bklist;
- print "delete old backup '$d->[0]'\n";
- unlink $d->[0];
- }
+sub setup_environment {
+ PVE::RPCEnvironment->setup_default_cli_env();
}
__PACKAGE__->register_method ({
- name => 'keygen',
+ name => 'keygen',
path => 'keygen',
method => 'PUT',
description => "Generate new cryptographic key for corosync.",
},
},
returns => { type => 'null' },
-
+
code => sub {
my ($param) = @_;
# test EUID
$> == 0 || die "Error: Authorization key must be generated as root user.\n";
my $dirname = dirname($filename);
- my $basename = basename($filename);
die "key file '$filename' already exists\n" if -e $filename;
File::Path::make_path($dirname) if $dirname;
- my $cmd = ['corosync-keygen', '-l', '-k', $filename];
- PVE::Tools::run_command($cmd);
+ run_command(['corosync-keygen', '-l', '-k', $filename]);
return undef;
}});
+my $foreach_member = sub {
+ my ($code, $noerr) = @_;
+
+ my $members = PVE::Cluster::get_members();
+ foreach my $node (sort keys %$members) {
+ if (my $ip = $members->{$node}->{ip}) {
+ $code->($node, $ip);
+ } else {
+ die "cannot get the cluster IP for node '$node'.\n" if !$noerr;
+ warn "cannot get the cluster IP for node '$node'.\n";
+ return undef;
+ }
+ }
+};
+
__PACKAGE__->register_method ({
- name => 'create',
- path => 'create',
+ name => 'setup_qdevice',
+ path => 'setup_qdevice',
method => 'PUT',
- description => "Generate new cluster configuration.",
+ description => "Setup the use of a QDevice",
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
- clustername => {
- description => "The name of the cluster.",
- type => 'string', format => 'pve-node',
- maxLength => 15,
- },
- nodeid => {
- type => 'integer',
- description => "Node id for this node.",
- minimum => 1,
- optional => 1,
- },
- votes => {
- type => 'integer',
- description => "Number of votes for this node.",
- minimum => 1,
- optional => 1,
- },
- bindnet0_addr => {
+ address => {
type => 'string', format => 'ip',
- description => "This specifies the network address the corosync ring 0".
- " executive should bind to and defaults to the local IP address of the node.",
- optional => 1,
- },
- ring0_addr => {
- type => 'string', format => 'address',
- description => "Hostname (or IP) of the corosync ring0 address of this node.".
- " Defaults to the hostname of the node.",
- optional => 1,
+ description => "Specifies the network address of an external corosync QDevice" ,
},
- rrp_mode => {
+ network => {
type => 'string',
- enum => ['none', 'active', 'passive'],
- description => "This specifies the mode of redundant ring, which" .
- " may be none, active or passive. Using multiple interfaces".
- " only allows 'active' or 'passive'.",
- default => 'none',
- optional => 1,
- },
- bindnet1_addr => {
- type => 'string', format => 'ip',
- description => "This specifies the network address the corosync ring 1".
- " executive should bind to and is optional.",
+ format => 'CIDR',
+ description => 'The network which should be used to connect to the external qdevice',
optional => 1,
},
- ring1_addr => {
- type => 'string', format => 'address',
- description => "Hostname (or IP) of the corosync ring1 address, this".
- " needs an valid bindnet1_addr.",
+ force => {
+ type => 'boolean',
+ description => "Do not throw error on possible dangerous operations.",
optional => 1,
},
},
},
returns => { type => 'null' },
-
+
code => sub {
my ($param) = @_;
- -f $clusterconf && die "cluster config '$clusterconf' already exists\n";
-
- PVE::Cluster::setup_sshd_config();
- PVE::Cluster::setup_rootsshconfig();
- PVE::Cluster::setup_ssh_keys();
-
- -f $authfile || __PACKAGE__->keygen({filename => $authfile});
-
- -f $authfile || die "no authentication key available\n";
-
- my $clustername = $param->{clustername};
-
- $param->{nodeid} = 1 if !$param->{nodeid};
-
- $param->{votes} = 1 if !defined($param->{votes});
-
- my $nodename = PVE::INotify::nodename();
-
- my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
+ PVE::Corosync::check_conf_exists();
- $param->{bindnet0_addr} = $local_ip_address
- if !defined($param->{bindnet0_addr});
-
- $param->{ring0_addr} = $nodename if !defined($param->{ring0_addr});
-
- die "Param bindnet1_addr and ring1_addr are dependend, use both or none!\n"
- if (defined($param->{bindnet1_addr}) != defined($param->{ring1_addr}));
-
- my $bind_is_ipv6 = Net::IP::ip_is_ipv6($param->{bindnet0_addr});
-
- # use string as here-doc format distracts more
- my $interfaces = "interface {\n ringnumber: 0\n" .
- " bindnetaddr: $param->{bindnet0_addr}\n }";
-
- my $ring_addresses = "ring0_addr: $param->{ring0_addr}" ;
-
- # allow use of multiple rings (rrp) at cluster creation time
- if ($param->{bindnet1_addr}) {
- die "IPv6 and IPv4 cannot be mixed, use one or the other!\n"
- if Net::IP::ip_is_ipv6($param->{bindnet1_addr}) != $bind_is_ipv6;
-
- die "rrp_mode 'none' is not allowed when using multiple interfaces,".
- " use 'active' or 'passive'!\n"
- if !$param->{rrp_mode} || $param->{rrp_mode} eq 'none';
-
- $interfaces .= "\n interface {\n ringnumber: 1\n" .
- " bindnetaddr: $param->{bindnet1_addr}\n }\n";
+ my $members = PVE::Cluster::get_members();
+ foreach my $node (sort keys %$members) {
+ die "All nodes must be online! Node $node is offline, aborting.\n"
+ if !$members->{$node}->{online};
+ }
- $ring_addresses .= "\n ring1_addr: $param->{ring1_addr}";
+ my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
- } elsif($param->{rrp_mode} && $param->{rrp_mode} ne 'none') {
+ die "QDevice already configured!\n"
+ if defined($conf->{main}->{quorum}->{device}) && !$param->{force};
- warn "rrp_mode '$param->{rrp_mode}' useless when using only one".
- " ring, using 'none' instead";
- # corosync defaults to none if only one interface is configured
- $param->{rrp_mode} = undef;
+ my $network = $param->{network};
+ my $model = "net";
+ my $algorithm = 'ffsplit';
+ if (scalar(%{$members}) & 1) {
+ if ($param->{force}) {
+ $algorithm = 'lms';
+ } else {
+ die "Clusters with an odd node count are not officially supported!\n";
+ }
}
- $interfaces = "rrp_mode: $param->{rrp_mode}\n " . $interfaces
- if $param->{rrp_mode};
-
- # No, corosync cannot deduce this on its own
- my $ipversion = $bind_is_ipv6 ? 'ipv6' : 'ipv4';
-
- my $config = <<_EOD;
-totem {
- version: 2
- secauth: on
- cluster_name: $clustername
- config_version: 1
- ip_version: $ipversion
- $interfaces
-}
-
-nodelist {
- node {
- $ring_addresses
- name: $nodename
- nodeid: $param->{nodeid}
- quorum_votes: $param->{votes}
- }
-}
+ my $qnetd_addr = $param->{address};
+ my $base_dir = "/etc/corosync/qdevice/net";
+ my $db_dir_qnetd = "/etc/corosync/qnetd/nssdb";
+ my $db_dir_node = "$base_dir/nssdb";
+ my $ca_export_base = "qnetd-cacert.crt";
+ my $ca_export_file = "$db_dir_qnetd/$ca_export_base";
+ my $crq_file_base = "qdevice-net-node.crq";
+ my $p12_file_base = "qdevice-net-node.p12";
+ my $qdevice_certutil = "corosync-qdevice-net-certutil";
+ my $qnetd_certutil= "corosync-qnetd-certutil";
+ my $clustername = $conf->{main}->{totem}->{cluster_name};
+
+ run_command(['ssh-copy-id', '-i', '/root/.ssh/id_rsa', "root\@$qnetd_addr"]);
+
+ if (-d $db_dir_node) {
+ # FIXME: check on all nodes?!
+ if ($param->{force}) {
+ rmtree $db_dir_node;
+ } else {
+ die "QDevice certificate store already initialised, set force to delete!\n";
+ }
+ }
-quorum {
- provider: corosync_votequorum
-}
+ my $ssh_cmd = ['ssh', '-o', 'BatchMode=yes', '-lroot'];
+ my $scp_cmd = ['scp', '-o', 'BatchMode=yes'];
+
+ print "\nINFO: initializing qnetd server\n";
+ run_command(
+ [@$ssh_cmd, $qnetd_addr, $qnetd_certutil, "-i"],
+ noerr => 1
+ );
+
+ print "\nINFO: copying CA cert and initializing on all nodes\n";
+ run_command([@$scp_cmd, "root\@\[$qnetd_addr\]:$ca_export_file", "/etc/pve/$ca_export_base"]);
+ $foreach_member->(sub {
+ my ($node, $ip) = @_;
+ my $outsub = sub { print "\nnode '$node': " . shift };
+ run_command(
+ [@$ssh_cmd, $ip, $qdevice_certutil, "-i", "-c", "/etc/pve/$ca_export_base"],
+ noerr => 1, outfunc => \&$outsub
+ );
+ });
+ unlink "/etc/pve/$ca_export_base";
+
+ print "\nINFO: generating cert request\n";
+ run_command([$qdevice_certutil, "-r", "-n", $clustername]);
+
+ print "\nINFO: copying exported cert request to qnetd server\n";
+ run_command([@$scp_cmd, "$db_dir_node/$crq_file_base", "root\@\[$qnetd_addr\]:/tmp"]);
+
+ print "\nINFO: sign and export cluster cert\n";
+ run_command([
+ @$ssh_cmd, $qnetd_addr, $qnetd_certutil, "-s", "-c",
+ "/tmp/$crq_file_base", "-n", "$clustername"
+ ]);
+
+ print "\nINFO: copy exported CRT\n";
+ run_command([
+ @$scp_cmd, "root\@\[$qnetd_addr\]:$db_dir_qnetd/cluster-$clustername.crt",
+ "$db_dir_node"
+ ]);
+
+ print "\nINFO: import certificate\n";
+ run_command(["$qdevice_certutil", "-M", "-c", "$db_dir_node/cluster-$clustername.crt"]);
+
+ print "\nINFO: copy and import pk12 cert to all nodes\n";
+ run_command([@$scp_cmd, "$db_dir_node/$p12_file_base", "/etc/pve/"]);
+ $foreach_member->(sub {
+ my ($node, $ip) = @_;
+ my $outsub = sub { print "\nnode '$node': " . shift };
+ run_command([
+ @$ssh_cmd, $ip, "$qdevice_certutil", "-m", "-c",
+ "/etc/pve/$p12_file_base"], outfunc => \&$outsub
+ );
+ });
+ unlink "/etc/pve/$p12_file_base";
+
+
+ my $code = sub {
+ my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
+ my $quorum_section = $conf->{main}->{quorum};
+
+ die "Qdevice already configured, must be removed before setting up new one!\n"
+ if defined($quorum_section->{device}); # must not be forced!
+
+ my $qdev_section = {
+ model => $model,
+ "$model" => {
+ tls => 'on',
+ host => $qnetd_addr,
+ algorithm => $algorithm,
+ }
+ };
+ $qdev_section->{votes} = 1 if $algorithm eq 'ffsplit';
-logging {
- to_syslog: yes
- debug: off
-}
-_EOD
-;
- PVE::Tools::file_set_contents($clusterconf, $config);
+ $quorum_section->{device} = $qdev_section;
- PVE::Cluster::ssh_merge_keys();
+ PVE::Corosync::atomic_write_conf($conf);
+ };
- PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
+ print "\nINFO: add QDevice to cluster configuration\n";
+ PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code);
+ die $@ if $@;
- PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
+ $foreach_member->(sub {
+ my ($node, $ip) = @_;
+ my $outsub = sub { print "\nnode '$node': " . shift };
+ print "\nINFO: start and enable corosync qdevice daemon on node '$node'...\n";
+ run_command([@$ssh_cmd, $ip, 'systemctl', 'start', 'corosync-qdevice'], outfunc => \&$outsub);
+ run_command([@$ssh_cmd, $ip, 'systemctl', 'enable', 'corosync-qdevice'], outfunc => \&$outsub);
+ });
- PVE::Tools::run_command('systemctl restart pve-cluster'); # restart
+ run_command(['corosync-cfgtool', '-R']); # do cluster wide config reload
- PVE::Tools::run_command('systemctl restart corosync'); # restart
-
return undef;
}});
__PACKAGE__->register_method ({
- name => 'addnode',
- path => 'addnode',
- method => 'PUT',
- description => "Adds a node to the cluster configuration.",
+ name => 'remove_qdevice',
+ path => 'remove_qdevice',
+ method => 'DELETE',
+ description => "Remove a configured QDevice",
parameters => {
- additionalProperties => 0,
- properties => {
- node => PVE::JSONSchema::get_standard_option('pve-node'),
- nodeid => {
- type => 'integer',
- description => "Node id for this node.",
- minimum => 1,
- optional => 1,
- },
- votes => {
- type => 'integer',
- description => "Number of votes for this node",
- minimum => 0,
- optional => 1,
- },
- force => {
- type => 'boolean',
- description => "Do not throw error if node already exists.",
- optional => 1,
- },
- ring0_addr => {
- type => 'string', format => 'address',
- description => "Hostname (or IP) of the corosync ring0 address of this node.".
- " Defaults to nodes hostname.",
- optional => 1,
- },
- ring1_addr => {
- type => 'string', format => 'address',
- description => "Hostname (or IP) of the corosync ring1 address, this".
- " needs an valid bindnet1_addr.",
- optional => 1,
- },
- },
+ additionalProperties => 0,
+ properties => {},
},
returns => { type => 'null' },
-
+
code => sub {
my ($param) = @_;
- PVE::Cluster::check_cfs_quorum();
-
- my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
+ PVE::Corosync::check_conf_exists();
- my $nodelist = corosync_nodelist($conf);
-
- my $totem_cfg = corosync_totem_config($conf);
-
- my $name = $param->{node};
-
- $param->{ring0_addr} = $name if !$param->{ring0_addr};
-
- die " ring1_addr needs a configured ring 1 interface!\n"
- if $param->{ring1_addr} && !defined($totem_cfg->{interface}->{1});
-
- if (defined(my $res = $nodelist->{$name})) {
- $param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
- $param->{votes} = $res->{quorum_votes} if !defined($param->{votes});
-
- if ($res->{quorum_votes} == $param->{votes} &&
- $res->{nodeid} == $param->{nodeid}) {
- print "node $name already defined\n";
- if ($param->{force}) {
- exit (0);
- } else {
- exit (-1);
- }
- } else {
- die "can't add existing node\n";
- }
- } elsif (!$param->{nodeid}) {
- my $nodeid = 1;
-
- while(1) {
- my $found = 0;
- foreach my $v (values %$nodelist) {
- if ($v->{nodeid} eq $nodeid) {
- $found = 1;
- $nodeid++;
- last;
- }
- }
- last if !$found;
- };
-
- $param->{nodeid} = $nodeid;
+ my $members = PVE::Cluster::get_members();
+ foreach my $node (sort keys %$members) {
+ die "All nodes must be online! Node $node is offline, aborting.\n"
+ if !$members->{$node}->{online};
}
- $param->{votes} = 1 if !defined($param->{votes});
-
- PVE::Cluster::gen_local_dirs($name);
+ my $ssh_cmd = ['ssh', '-o', 'BatchMode=yes', '-lroot'];
- eval { PVE::Cluster::ssh_merge_keys(); };
- warn $@ if $@;
+ my $code = sub {
+ my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
+ my $quorum_section = $conf->{main}->{quorum};
- $nodelist->{$name} = {
- ring0_addr => $param->{ring0_addr},
- nodeid => $param->{nodeid},
- name => $name,
- };
- $nodelist->{$name}->{ring1_addr} = $param->{ring1_addr} if $param->{ring1_addr};
- $nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes};
-
- corosync_update_nodelist($conf, $nodelist);
-
- exit (0);
- }});
-
-
-__PACKAGE__->register_method ({
- name => 'delnode',
- path => 'delnode',
- method => 'PUT',
- description => "Removes a node to the cluster configuration.",
- parameters => {
- additionalProperties => 0,
- properties => {
- node => {
- type => 'string',
- description => "Hostname or IP of the corosync ring0 address of this node.",
- },
- },
- },
- returns => { type => 'null' },
-
- code => sub {
- my ($param) = @_;
+ die "No QDevice configured!\n" if !defined($quorum_section->{device});
- PVE::Cluster::check_cfs_quorum();
+ delete $quorum_section->{device};
- my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
+ PVE::Corosync::atomic_write_conf($conf);
- my $nodelist = corosync_nodelist($conf);
-
- my $node;
- my $nodeid;
-
- foreach my $tmp_node (keys %$nodelist) {
- my $d = $nodelist->{$tmp_node};
- my $ring0_addr = $d->{ring0_addr};
- my $ring1_addr = $d->{ring1_addr};
- if (($tmp_node eq $param->{node}) ||
- (defined($ring0_addr) && ($ring0_addr eq $param->{node})) ||
- (defined($ring1_addr) && ($ring1_addr eq $param->{node}))) {
- $node = $tmp_node;
- $nodeid = $d->{nodeid};
- last;
- }
- }
+ # cleanup qdev state (cert storage)
+ my $qdev_state_dir = "/etc/corosync/qdevice";
+ $foreach_member->(sub {
+ my (undef, $ip) = @_;
+ run_command([@$ssh_cmd, $ip, '--', 'rm', '-rf', $qdev_state_dir]);
+ });
+ };
- die "Node/IP: $param->{node} is not a known host of the cluster.\n"
- if !defined($node);
+ PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code);
+ die $@ if $@;
- delete $nodelist->{$node};
+ $foreach_member->(sub {
+ my (undef, $ip) = @_;
+ run_command([@$ssh_cmd, $ip, 'systemctl', 'stop', 'corosync-qdevice']);
+ run_command([@$ssh_cmd, $ip, 'systemctl', 'disable', 'corosync-qdevice']);
+ });
- corosync_update_nodelist($conf, $nodelist);
+ run_command(['corosync-cfgtool', '-R']);
- PVE::Tools::run_command(['corosync-cfgtool','-k', $nodeid])
- if defined($nodeid);
+ print "\nRemoved Qdevice.\n";
return undef;
- }});
+}});
__PACKAGE__->register_method ({
- name => 'add',
+ name => 'add',
path => 'add',
method => 'PUT',
description => "Adds the current node to an existing cluster.",
parameters => {
additionalProperties => 0,
- properties => {
+ properties => PVE::Corosync::add_corosync_link_properties({
hostname => {
type => 'string',
description => "Hostname (or IP) of an existing cluster member."
},
- nodeid => {
- type => 'integer',
- description => "Node id for this node.",
- minimum => 1,
- optional => 1,
- },
+ nodeid => get_standard_option('corosync-nodeid'),
votes => {
type => 'integer',
description => "Number of votes for this node",
description => "Do not throw error if node already exists.",
optional => 1,
},
- ring0_addr => {
- type => 'string', format => 'address',
- description => "Hostname (or IP) of the corosync ring0 address of this node.".
- " Defaults to nodes hostname.",
+ fingerprint => get_standard_option('fingerprint-sha256', {
optional => 1,
- },
- ring1_addr => {
- type => 'string', format => 'address',
- description => "Hostname (or IP) of the corosync ring1 address, this".
- " needs an valid configured ring 1 interface in the cluster.",
+ }),
+ 'use_ssh' => {
+ type => 'boolean',
+ description => "Always use SSH to join, even if peer may do it over API.",
optional => 1,
},
- },
+ }),
},
returns => { type => 'null' },
-
+
code => sub {
my ($param) = @_;
my $nodename = PVE::INotify::nodename();
-
- PVE::Cluster::setup_sshd_config();
- PVE::Cluster::setup_rootsshconfig();
- PVE::Cluster::setup_ssh_keys();
-
my $host = $param->{hostname};
- if (!$param->{force}) {
-
- if (-f $authfile) {
- die "authentication key already exists\n";
- }
+ my $worker = sub {
- if (-f $clusterconf) {
- die "cluster config '$clusterconf' already exists\n";
- }
+ if (!$param->{use_ssh}) {
+ my $password = PVE::PTY::read_password("Please enter superuser (root) password for '$host': ");
- my $vmlist = PVE::Cluster::get_vmlist();
- if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
- die "this host already contains virtual machines - please remove them first\n";
- }
+ delete $param->{use_ssh};
+ $param->{password} = $password;
- if (system("corosync-quorumtool >/dev/null 2>&1") == 0) {
- die "corosync is already running\n";
- }
- }
-
- # make sure known_hosts is on local filesystem
- PVE::Cluster::ssh_unmerge_known_hosts();
-
- my $cmd = ['ssh-copy-id', '-i', '/root/.ssh/id_rsa', "root\@$host"];
- PVE::Tools::run_command($cmd, 'outfunc' => sub {}, 'errfunc' => sub {},
- 'errmsg' => "unable to copy ssh ID");
-
- $cmd = ['ssh', $host, '-o', 'BatchMode=yes',
- 'pvecm', 'addnode', $nodename, '--force', 1];
+ my $local_cluster_lock = "/var/lock/pvecm.lock";
+ PVE::Tools::lock_file($local_cluster_lock, 10, \&PVE::Cluster::Setup::join, $param);
- push @$cmd, '--nodeid', $param->{nodeid} if $param->{nodeid};
-
- push @$cmd, '--votes', $param->{votes} if defined($param->{votes});
-
- push @$cmd, '--ring0_addr', $param->{ring0_addr} if defined($param->{ring0_addr});
-
- push @$cmd, '--ring1_addr', $param->{ring1_addr} if defined($param->{ring1_addr});
+ if (my $err = $@) {
+ if (ref($err) eq 'PVE::APIClient::Exception' && defined($err->{code}) && $err->{code} == 501) {
+ $err = "Remote side is not able to use API for Cluster join!\n" .
+ "Pass the 'use_ssh' switch or update the remote side.\n";
+ }
+ die $err;
+ }
+ return; # all OK, the API join endpoint successfully set us up
+ }
- if (system (@$cmd) != 0) {
- my $cmdtxt = join (' ', @$cmd);
- die "unable to add node: command failed ($cmdtxt)\n";
- }
+ # allow fallback to old ssh only join if wished or needed
- my $tmpdir = "$libdir/.pvecm_add.tmp.$$";
- mkdir $tmpdir;
+ my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
+ my $links = PVE::Corosync::extract_corosync_link_args($param);
- eval {
- print "copy corosync auth key\n";
- $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
- "[$host]:$authfile $clusterconf", $tmpdir];
+ PVE::Cluster::Setup::assert_joinable($local_ip_address, $links, $param->{force});
- system(@$cmd) == 0 || die "can't rsync data from host '$host'\n";
+ PVE::Cluster::Setup::setup_sshd_config();
+ PVE::Cluster::Setup::setup_rootsshconfig();
+ PVE::Cluster::Setup::setup_ssh_keys();
- mkdir "/etc/corosync";
- my $confbase = basename($clusterconf);
+ # make sure known_hosts is on local filesystem
+ PVE::Cluster::Setup::ssh_unmerge_known_hosts();
- $cmd = "cp '$tmpdir/$confbase' '/etc/corosync/$confbase'";
- system($cmd) == 0 || die "can't copy cluster configuration\n";
+ my $cmd = ['ssh-copy-id', '-i', '/root/.ssh/id_rsa', "root\@$host"];
+ run_command($cmd, 'outfunc' => sub {}, 'errfunc' => sub {},
+ 'errmsg' => "unable to copy ssh ID");
- my $keybase = basename($authfile);
- system ("cp '$tmpdir/$keybase' '$authfile'") == 0 ||
- die "can't copy '$tmpdir/$keybase' to '$authfile'\n";
+ $cmd = ['ssh', $host, '-o', 'BatchMode=yes', 'pvecm', 'apiver'];
+ my $remote_apiver = 0;
+ run_command($cmd, 'outfunc' => sub {
+ $remote_apiver = shift;
+ chomp $remote_apiver;
+ }, 'noerr' => 1);
- print "stopping pve-cluster service\n";
+ if ($remote_apiver < (PVE::Cluster::Setup::JOIN_API_VERSION -
+ PVE::Cluster::Setup::JOIN_API_AGE_AS_JOINEE)) {
+ die "error: incompatible join API version on cluster ($remote_apiver,"
+ . " local has " . PVE::Cluster::Setup::JOIN_API_VERSION . "). Make"
+ . " sure all nodes are up-to-date.\n";
+ }
- system("umount $basedir -f >/dev/null 2>&1");
- system("systemctl stop pve-cluster") == 0 ||
- die "can't stop pve-cluster service\n";
+ $cmd = ['ssh', $host, '-o', 'BatchMode=yes',
+ 'pvecm', 'addnode', $nodename, '--force', 1];
- backup_database();
+ push @$cmd, '--nodeid', $param->{nodeid} if $param->{nodeid};
+ push @$cmd, '--votes', $param->{votes} if defined($param->{votes});
- unlink $dbfile;
+ foreach my $link (keys %$links) {
+ push @$cmd, "--link$link", PVE::JSONSchema::print_property_string(
+ $links->{$link}, get_standard_option('corosync-link'));
+ }
- system("systemctl start pve-cluster") == 0 ||
- die "starting pve-cluster failed\n";
+ # this will be used as fallback if no links are specified
+ if (!%$links) {
+ push @$cmd, '--link0', $local_ip_address if $remote_apiver == 0;
+ push @$cmd, '--new_node_ip', $local_ip_address if $remote_apiver >= 1;
- system("systemctl start corosync");
+ print "No cluster network links passed explicitly, fallback to local node"
+ . " IP '$local_ip_address'\n";
+ }
- # wait for quorum
- my $printqmsg = 1;
- while (!PVE::Cluster::check_cfs_quorum(1)) {
- if ($printqmsg) {
- print "waiting for quorum...";
- STDOUT->flush();
- $printqmsg = 0;
- }
- sleep(1);
+ if (system (@$cmd) != 0) {
+ my $cmdtxt = join (' ', @$cmd);
+ die "unable to add node: command failed ($cmdtxt)\n";
}
- print "OK\n" if !$printqmsg;
- # system("systemctl start clvm");
+ my $tmpdir = "$libdir/.pvecm_add.tmp.$$";
+ mkdir $tmpdir;
- my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
+ eval {
+ print "copy corosync auth key\n";
+ $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
+ "[$host]:$authfile $clusterconf", $tmpdir];
- print "generating node certificates\n";
- PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
+ system(@$cmd) == 0 || die "can't rsync data from host '$host'\n";
- print "merge known_hosts file\n";
- PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
+ my $corosync_conf = PVE::Tools::file_get_contents("$tmpdir/corosync.conf");
+ my $corosync_authkey = PVE::Tools::file_get_contents("$tmpdir/authkey");
+
+ PVE::Cluster::Setup::finish_join($host, $corosync_conf, $corosync_authkey);
+ };
+ my $err = $@;
- print "restart services\n";
- # restart pvedaemon (changed certs)
- system("systemctl restart pvedaemon");
- # restart pveproxy (changed certs)
- system("systemctl restart pveproxy");
+ rmtree $tmpdir;
- print "successfully added node '$nodename' to cluster.\n";
+ die $err if $err;
};
- my $err = $@;
- rmtree $tmpdir;
+ # use a synced worker so we get a nice task log when joining through CLI
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $authuser = $rpcenv->get_user();
- die $err if $err;
+ $rpcenv->fork_worker('clusterjoin', '', $authuser, $worker);
return undef;
}});
__PACKAGE__->register_method ({
- name => 'status',
+ name => 'status',
path => 'status',
method => 'GET',
description => "Displays the local view of the cluster status.",
properties => {},
},
returns => { type => 'null' },
-
+
code => sub {
my ($param) = @_;
- PVE::Cluster::check_corosync_conf_exists();
+ PVE::Corosync::check_conf_exists();
+ my $conf = eval { PVE::Cluster::cfs_read_file("corosync.conf") } // {};
+ warn "$@" if $@;
+ my $totem = PVE::Corosync::totem_config($conf);
- my $cmd = ['corosync-quorumtool', '-siH'];
+ if (scalar(%$totem)) {
+ my $print_info = sub {
+ my ($label, $key, $default) = @_;
+ my $val = $totem->{$key} // $default;
+ printf "%-17s %s\n", "$label:", "$val";
+ };
- exec (@$cmd);
+ printf "Cluster information\n";
+ printf "-------------------\n";
+ $print_info->('Name', 'cluster_name', 'UNKOWN?');
+ $print_info->('Config Version', 'config_version', -1);
+ $print_info->('Transport', 'transport', 'knet');
+ $print_info->('Secure auth', 'secauth', 'off');
+ printf "\n";
+ }
+ exec ('corosync-quorumtool', '-siH');
exit (-1); # should not be reached
}});
__PACKAGE__->register_method ({
- name => 'nodes',
+ name => 'nodes',
path => 'nodes',
method => 'GET',
description => "Displays the local view of the cluster nodes.",
properties => {},
},
returns => { type => 'null' },
-
+
code => sub {
my ($param) = @_;
- PVE::Cluster::check_corosync_conf_exists();
-
- my $cmd = ['corosync-quorumtool', '-l'];
-
- exec (@$cmd);
+ PVE::Corosync::check_conf_exists();
+ exec ('corosync-quorumtool', '-l');
exit (-1); # should not be reached
}});
__PACKAGE__->register_method ({
- name => 'expected',
+ name => 'expected',
path => 'expected',
method => 'PUT',
description => "Tells corosync a new value of expected votes.",
},
},
returns => { type => 'null' },
-
+
code => sub {
my ($param) = @_;
- PVE::Cluster::check_corosync_conf_exists();
-
- my $cmd = ['corosync-quorumtool', '-e', $param->{expected}];
-
- exec (@$cmd);
+ PVE::Corosync::check_conf_exists();
+ exec ('corosync-quorumtool', '-e', $param->{expected});
exit (-1); # should not be reached
-
}});
-sub corosync_update_nodelist {
- my ($conf, $nodelist) = @_;
-
- delete $conf->{digest};
-
- my $version = PVE::Cluster::corosync_conf_version($conf);
- PVE::Cluster::corosync_conf_version($conf, undef, $version + 1);
-
- my $children = [];
- foreach my $v (values %$nodelist) {
- next if !($v->{ring0_addr} || $v->{name});
- my $kv = [];
- foreach my $k (keys %$v) {
- push @$kv, { key => $k, value => $v->{$k} };
- }
- my $ns = { section => 'node', children => $kv };
- push @$children, $ns;
- }
-
- foreach my $main (@{$conf->{children}}) {
- next if !defined($main->{section});
- if ($main->{section} eq 'nodelist') {
- $main->{children} = $children;
- last;
- }
- }
-
-
- PVE::Cluster::cfs_write_file("corosync.conf.new", $conf);
-
- rename("/etc/pve/corosync.conf.new", "/etc/pve/corosync.conf")
- || die "activate corosync.conf.new failed - $!\n";
-}
-
-sub corosync_nodelist {
- my ($conf) = @_;
-
- my $nodelist = {};
-
- foreach my $main (@{$conf->{children}}) {
- next if !defined($main->{section});
- if ($main->{section} eq 'nodelist') {
- foreach my $ne (@{$main->{children}}) {
- next if !defined($ne->{section}) || ($ne->{section} ne 'node');
- my $node = { quorum_votes => 1 };
- my $name;
- foreach my $child (@{$ne->{children}}) {
- next if !defined($child->{key});
- $node->{$child->{key}} = $child->{value};
- # use 'name' over 'ring0_addr' if set
- if ($child->{key} eq 'name') {
- delete $nodelist->{$name} if $name;
- $name = $child->{value};
- $nodelist->{$name} = $node;
- } elsif(!$name && $child->{key} eq 'ring0_addr') {
- $name = $child->{value};
- $nodelist->{$name} = $node;
- }
- }
- }
- }
- }
-
- return $nodelist;
-}
-
-# get a hash representation of the corosync config totem section
-sub corosync_totem_config {
- my ($conf) = @_;
-
- my $res = {};
-
- foreach my $main (@{$conf->{children}}) {
- next if !defined($main->{section}) ||
- $main->{section} ne 'totem';
-
- foreach my $e (@{$main->{children}}) {
-
- if ($e->{section} && $e->{section} eq 'interface') {
- my $entry = {};
-
- $res->{interface} = {};
-
- foreach my $child (@{$e->{children}}) {
- next if !defined($child->{key});
- $entry->{$child->{key}} = $child->{value};
- if($child->{key} eq 'ringnumber') {
- $res->{interface}->{$child->{value}} = $entry;
- }
- }
-
- } elsif ($e->{key}) {
- $res->{$e->{key}} = $e->{value};
- }
- }
- }
-
- return $res;
-}
-
__PACKAGE__->register_method ({
- name => 'updatecerts',
+ name => 'updatecerts',
path => 'updatecerts',
method => 'PUT',
description => "Update node certificates (and generate all needed files/directories).",
code => sub {
my ($param) = @_;
- PVE::Cluster::setup_rootsshconfig();
-
- PVE::Cluster::gen_pve_vzdump_symlink();
-
- if (!PVE::Cluster::check_cfs_quorum(1)) {
- return undef if $param->{silent};
- die "no quorum - unable to update files\n";
- }
-
- PVE::Cluster::setup_ssh_keys();
-
- my $nodename = PVE::INotify::nodename();
-
- my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
-
- PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address, $param->{force});
- PVE::Cluster::ssh_merge_keys();
- PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address);
- PVE::Cluster::gen_pve_vzdump_files();
+ # we get called by the pve-cluster.service ExecStartPost and as we do
+ # IO (on /etc/pve) which can hang (uninterruptedly D state). That'd be
+ # no-good for ExecStartPost as it fails the whole service in this case
+ PVE::Tools::run_fork_with_timeout(30, sub {
+ PVE::Cluster::Setup::updatecerts_and_ssh($param->@{qw(force silent)});
+ });
return undef;
}});
description => 'the migration network used to detect the local migration IP',
optional => 1,
},
+ 'run-command' => {
+ type => 'boolean',
+ description => 'Run a command with a tcp socket as standard input.'
+ .' The IP address and port are printed via this'
+ ." command's stdandard output first, each on a separate line.",
+ optional => 1,
+ },
+ 'extra-args' => PVE::JSONSchema::get_standard_option('extra-args'),
},
},
returns => { type => 'null'},
return undef;
}
+ my $get_local_migration_ip = sub {
+ my ($cidr) = @_;
+
+ if (!defined($cidr)) {
+ my $dc_conf = cfs_read_file('datacenter.cfg');
+ $cidr = $dc_conf->{migration}->{network}
+ if defined($dc_conf->{migration}->{network});
+ }
+
+ if (defined($cidr)) {
+ my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
+
+ die "could not get migration ip: no IP address configured on local " .
+ "node for network '$cidr'\n" if scalar(@$ips) == 0;
+
+ die "could not get migration ip: multiple, different, IP address configured for " .
+ "network '$cidr'\n" if scalar(@$ips) > 1 && grep { @$ips[0] ne $_ } @$ips;
+
+ return @$ips[0];
+ }
+
+ return undef;
+ };
+
+ my $network = $param->{migration_network};
if ($param->{get_migration_ip}) {
- my $network = $param->{migration_network};
- if (my $ip = PVE::Cluster::get_local_migration_ip($network)) {
+ die "cannot use --run-command with --get_migration_ip\n"
+ if $param->{'run-command'};
+
+ if (my $ip = $get_local_migration_ip->($network)) {
print "ip: '$ip'\n";
} else {
print "no ip\n";
return undef;
}
+ if ($param->{'run-command'}) {
+ my $cmd = $param->{'extra-args'};
+ die "missing command\n"
+ if !$cmd || !scalar(@$cmd);
+
+ # Get an ip address to listen on, and find a free migration port
+ my ($ip, $family);
+ if (defined($network)) {
+ $ip = $get_local_migration_ip->($network)
+ or die "failed to get migration IP address to listen on\n";
+ $family = PVE::Tools::get_host_address_family($ip);
+ } else {
+ my $nodename = PVE::INotify::nodename();
+ ($ip, $family) = PVE::Network::get_ip_from_hostname($nodename, 0);
+ }
+ my $port = PVE::Tools::next_migrate_port($family, $ip);
+
+ PVE::Tools::pipe_socket_to_command($cmd, $ip, $port);
+ return undef;
+ }
+
print "tunnel online\n";
*STDOUT->flush();
- while (my $line = <>) {
+ while (my $line = <STDIN>) {
chomp $line;
last if $line =~ m/^quit$/;
}
our $cmddef = {
+ apiver => [ 'PVE::API2::ClusterConfig', 'join_api_version', [], {}, sub {
+ my $apiver = shift;
+ print "$apiver\n";
+ }],
keygen => [ __PACKAGE__, 'keygen', ['filename']],
- create => [ __PACKAGE__, 'create', ['clustername']],
+ create => [ 'PVE::API2::ClusterConfig', 'create', ['clustername']],
add => [ __PACKAGE__, 'add', ['hostname']],
- addnode => [ __PACKAGE__, 'addnode', ['node']],
- delnode => [ __PACKAGE__, 'delnode', ['node']],
+ addnode => [ 'PVE::API2::ClusterConfig', 'addnode', ['node']],
+ delnode => [ 'PVE::API2::ClusterConfig', 'delnode', ['node']],
status => [ __PACKAGE__, 'status' ],
nodes => [ __PACKAGE__, 'nodes' ],
expected => [ __PACKAGE__, 'expected', ['expected']],
updatecerts => [ __PACKAGE__, 'updatecerts', []],
- mtunnel => [ __PACKAGE__, 'mtunnel', []],
+ mtunnel => [ __PACKAGE__, 'mtunnel', ['extra-args']],
+ qdevice => {
+ setup => [ __PACKAGE__, 'setup_qdevice', ['address']],
+ remove => [ __PACKAGE__, 'remove_qdevice', []],
+ }
};
1;
projects / pve-cluster.git / blobdiff