use Getopt::Long;
use Socket;
use IO::File;
+use IO::Socket::IP;
+use POSIX;
use Net::IP;
use File::Path;
use File::Basename;
-use Data::Dumper; # fixme: remove
+use Data::Dumper; # fixme: remove
use PVE::Tools;
use PVE::Cluster;
use PVE::INotify;
use PVE::JSONSchema;
use PVE::CLIHandler;
+use PVE::Corosync;
use base qw(PVE::CLIHandler);
print "backup old database\n";
mkdir $backupdir;
-
+
my $ctime = time();
my $cmd = [
['echo', '.dump'],
push @bklist, [$fn, $1];
}
}
-
+
@bklist = sort { $b->[1] <=> $a->[1] } @bklist;
while (scalar (@bklist) >= $maxfiles) {
}
__PACKAGE__->register_method ({
- name => 'keygen',
+ name => 'keygen',
path => 'keygen',
method => 'PUT',
description => "Generate new cryptographic key for corosync.",
},
},
returns => { type => 'null' },
-
+
code => sub {
my ($param) = @_;
File::Path::make_path($dirname) if $dirname;
my $cmd = ['corosync-keygen', '-l', '-k', $filename];
- PVE::Tools::run_command($cmd);
+ PVE::Tools::run_command($cmd);
return undef;
}});
__PACKAGE__->register_method ({
- name => 'create',
+ name => 'create',
path => 'create',
method => 'PUT',
description => "Generate new cluster configuration.",
" Defaults to the hostname of the node.",
optional => 1,
},
- rrp_mode => {
- type => 'string',
- enum => ['none', 'active', 'passive'],
- description => "This specifies the mode of redundant ring, which" .
- " may be none, active or passive. Using multiple interfaces".
- " only allows 'active' or 'passive'.",
- default => 'none',
- optional => 1,
- },
bindnet1_addr => {
type => 'string', format => 'ip',
description => "This specifies the network address the corosync ring 1".
},
},
returns => { type => 'null' },
-
+
code => sub {
my ($param) = @_;
$param->{votes} = 1 if !defined($param->{votes});
my $nodename = PVE::INotify::nodename();
-
+
my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
$param->{bindnet0_addr} = $local_ip_address
die "IPv6 and IPv4 cannot be mixed, use one or the other!\n"
if Net::IP::ip_is_ipv6($param->{bindnet1_addr}) != $bind_is_ipv6;
- die "rrp_mode 'none' is not allowed when using multiple interfaces,".
- " use 'active' or 'passive'!\n"
- if !$param->{rrp_mode} || $param->{rrp_mode} eq 'none';
-
$interfaces .= "\n interface {\n ringnumber: 1\n" .
" bindnetaddr: $param->{bindnet1_addr}\n }\n";
+ $interfaces .= "rrp_mode: passive\n"; # only passive is stable and tested
+
$ring_addresses .= "\n ring1_addr: $param->{ring1_addr}";
} elsif($param->{rrp_mode} && $param->{rrp_mode} ne 'none') {
}
- $interfaces = "rrp_mode: $param->{rrp_mode}\n " . $interfaces
- if $param->{rrp_mode};
-
# No, corosync cannot deduce this on its own
my $ipversion = $bind_is_ipv6 ? 'ipv6' : 'ipv4';
PVE::Tools::run_command('systemctl restart pve-cluster'); # restart
PVE::Tools::run_command('systemctl restart corosync'); # restart
-
+
return undef;
}});
__PACKAGE__->register_method ({
- name => 'addnode',
+ name => 'addnode',
path => 'addnode',
method => 'PUT',
description => "Adds a node to the cluster configuration.",
},
},
returns => { type => 'null' },
-
+
code => sub {
my ($param) = @_;
+ if (!$param->{force} && (-t STDIN || -t STDOUT)) {
+ die "error: `addnode` should not get called interactively!\nUse ".
+ "`pvecm add <cluster-node>` to add a node to a cluster!\n";
+ }
+
PVE::Cluster::check_cfs_quorum();
- my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
+ my $code = sub {
+ my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
+ my $nodelist = PVE::Corosync::nodelist($conf);
+ my $totem_cfg = PVE::Corosync::totem_config($conf);
+
+ my $name = $param->{node};
- my $nodelist = corosync_nodelist($conf);
+ # ensure we do not reuse an address, that can crash the whole cluster!
+ my $check_duplicate_addr = sub {
+ my $addr = shift;
+ return if !defined($addr);
+
+ while (my ($k, $v) = each %$nodelist) {
+ next if $k eq $name; # allows re-adding a node if force is set
+ if ($v->{ring0_addr} eq $addr || ($v->{ring1_addr} && $v->{ring1_addr} eq $addr)) {
+ die "corosync: address '$addr' already defined by node '$k'\n";
+ }
+ }
+ };
- my $totem_cfg = corosync_totem_config($conf);
+ &$check_duplicate_addr($param->{ring0_addr});
+ &$check_duplicate_addr($param->{ring1_addr});
- my $name = $param->{node};
+ $param->{ring0_addr} = $name if !$param->{ring0_addr};
- $param->{ring0_addr} = $name if !$param->{ring0_addr};
+ die "corosync: using 'ring1_addr' parameter needs a configured ring 1 interface!\n"
+ if $param->{ring1_addr} && !defined($totem_cfg->{interface}->{1});
- die " ring1_addr needs a configured ring 1 interface!\n"
- if $param->{ring1_addr} && !defined($totem_cfg->{interface}->{1});
+ die "corosync: ring 1 interface configured but 'ring1_addr' parameter not defined!\n"
+ if defined($totem_cfg->{interface}->{1}) && !defined($param->{ring1_addr});
- if (defined(my $res = $nodelist->{$name})) {
- $param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
- $param->{votes} = $res->{quorum_votes} if !defined($param->{votes});
+ if (defined(my $res = $nodelist->{$name})) {
+ $param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
+ $param->{votes} = $res->{quorum_votes} if !defined($param->{votes});
- if ($res->{quorum_votes} == $param->{votes} &&
- $res->{nodeid} == $param->{nodeid}) {
- print "node $name already defined\n";
- if ($param->{force}) {
- exit (0);
+ if ($res->{quorum_votes} == $param->{votes} &&
+ $res->{nodeid} == $param->{nodeid}) {
+ print "node $name already defined\n";
+ if ($param->{force}) {
+ exit (0);
+ } else {
+ exit (-1);
+ }
} else {
- exit (-1);
+ die "can't add existing node\n";
}
- } else {
- die "can't add existing node\n";
- }
- } elsif (!$param->{nodeid}) {
- my $nodeid = 1;
-
- while(1) {
- my $found = 0;
- foreach my $v (values %$nodelist) {
- if ($v->{nodeid} eq $nodeid) {
- $found = 1;
- $nodeid++;
- last;
+ } elsif (!$param->{nodeid}) {
+ my $nodeid = 1;
+
+ while(1) {
+ my $found = 0;
+ foreach my $v (values %$nodelist) {
+ if ($v->{nodeid} eq $nodeid) {
+ $found = 1;
+ $nodeid++;
+ last;
+ }
}
- }
- last if !$found;
- };
+ last if !$found;
+ };
- $param->{nodeid} = $nodeid;
- }
+ $param->{nodeid} = $nodeid;
+ }
- $param->{votes} = 1 if !defined($param->{votes});
+ $param->{votes} = 1 if !defined($param->{votes});
- PVE::Cluster::gen_local_dirs($name);
+ PVE::Cluster::gen_local_dirs($name);
- eval { PVE::Cluster::ssh_merge_keys(); };
- warn $@ if $@;
+ eval { PVE::Cluster::ssh_merge_keys(); };
+ warn $@ if $@;
+
+ $nodelist->{$name} = {
+ ring0_addr => $param->{ring0_addr},
+ nodeid => $param->{nodeid},
+ name => $name,
+ };
+ $nodelist->{$name}->{ring1_addr} = $param->{ring1_addr} if $param->{ring1_addr};
+ $nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes};
- $nodelist->{$name} = {
- ring0_addr => $param->{ring0_addr},
- nodeid => $param->{nodeid},
- name => $name,
+ PVE::Corosync::update_nodelist($conf, $nodelist);
};
- $nodelist->{$name}->{ring1_addr} = $param->{ring1_addr} if $param->{ring1_addr};
- $nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes};
-
- corosync_update_nodelist($conf, $nodelist);
-
+
+ PVE::Cluster::cfs_lock_file('corosync.conf', 10, &$code);
+ die $@ if $@;
+
exit (0);
}});
__PACKAGE__->register_method ({
- name => 'delnode',
+ name => 'delnode',
path => 'delnode',
method => 'PUT',
description => "Removes a node to the cluster configuration.",
},
},
returns => { type => 'null' },
-
+
code => sub {
my ($param) = @_;
PVE::Cluster::check_cfs_quorum();
- my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
-
- my $nodelist = corosync_nodelist($conf);
-
- my $node;
- my $nodeid;
-
- foreach my $tmp_node (keys %$nodelist) {
- my $d = $nodelist->{$tmp_node};
- my $ring0_addr = $d->{ring0_addr};
- my $ring1_addr = $d->{ring1_addr};
- if (($tmp_node eq $param->{node}) ||
- (defined($ring0_addr) && ($ring0_addr eq $param->{node})) ||
- (defined($ring1_addr) && ($ring1_addr eq $param->{node}))) {
- $node = $tmp_node;
- $nodeid = $d->{nodeid};
- last;
+ my $code = sub {
+ my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
+ my $nodelist = PVE::Corosync::nodelist($conf);
+
+ my $node;
+ my $nodeid;
+
+ foreach my $tmp_node (keys %$nodelist) {
+ my $d = $nodelist->{$tmp_node};
+ my $ring0_addr = $d->{ring0_addr};
+ my $ring1_addr = $d->{ring1_addr};
+ if (($tmp_node eq $param->{node}) ||
+ (defined($ring0_addr) && ($ring0_addr eq $param->{node})) ||
+ (defined($ring1_addr) && ($ring1_addr eq $param->{node}))) {
+ $node = $tmp_node;
+ $nodeid = $d->{nodeid};
+ last;
+ }
}
- }
- die "Node/IP: $param->{node} is not a known host of the cluster.\n"
+ die "Node/IP: $param->{node} is not a known host of the cluster.\n"
if !defined($node);
- delete $nodelist->{$node};
+ my $our_nodename = PVE::INotify::nodename();
+ die "Cannot delete myself from cluster!\n" if $node eq $our_nodename;
- corosync_update_nodelist($conf, $nodelist);
+ delete $nodelist->{$node};
- PVE::Tools::run_command(['corosync-cfgtool','-k', $nodeid])
- if defined($nodeid);
+ PVE::Corosync::update_nodelist($conf, $nodelist);
+
+ PVE::Tools::run_command(['corosync-cfgtool','-k', $nodeid])
+ if defined($nodeid);
+ };
+
+ PVE::Cluster::cfs_lock_file('corosync.conf', 10, &$code);
+ die $@ if $@;
return undef;
}});
__PACKAGE__->register_method ({
- name => 'add',
+ name => 'add',
path => 'add',
method => 'PUT',
description => "Adds the current node to an existing cluster.",
},
},
returns => { type => 'null' },
-
+
code => sub {
my ($param) = @_;
my $host = $param->{hostname};
+ my ($errors, $warnings) = ('', '');
+
+ my $error = sub {
+ my ($msg, $suppress) = @_;
+
+ if ($suppress) {
+ $warnings .= "* $msg\n";
+ } else {
+ $errors .= "* $msg\n";
+ }
+ };
+
if (!$param->{force}) {
-
+
if (-f $authfile) {
- die "authentication key already exists\n";
+ &$error("authentication key '$authfile' already exists", $param->{force});
}
if (-f $clusterconf) {
- die "cluster config '$clusterconf' already exists\n";
+ &$error("cluster config '$clusterconf' already exists", $param->{force});
}
my $vmlist = PVE::Cluster::get_vmlist();
if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
- die "this host already contains virtual machines - please remove them first\n";
+ &$error("this host already contains virtual guests", $param->{force});
}
- if (system("corosync-quorumtool >/dev/null 2>&1") == 0) {
- die "corosync is already running\n";
+ if (system("corosync-quorumtool -l >/dev/null 2>&1") == 0) {
+ &$error("corosync is already running, is this node already in a cluster?!", $param->{force});
}
}
+ # check if corosync ring IPs are configured on the current nodes interfaces
+ my $check_ip = sub {
+ my $ip = shift;
+ if (defined($ip)) {
+ if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
+ my $host = $ip;
+ eval { $ip = PVE::Network::get_ip_from_hostname($host); };
+ if ($@) {
+ &$error("cannot use '$host': $@\n") ;
+ return;
+ }
+ }
+
+ my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
+ my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
+
+ &$error("cannot use IP '$ip', it must be configured exactly once on local node!\n")
+ if (scalar(@$configured_ips) != 1);
+ }
+ };
+
+ &$check_ip($param->{ring0_addr});
+ &$check_ip($param->{ring1_addr});
+
+ warn "warning, ignore the following errors:\n$warnings" if $warnings;
+ die "detected the following error(s):\n$errors" if $errors;
+
# make sure known_hosts is on local filesystem
PVE::Cluster::ssh_unmerge_known_hosts();
eval {
print "copy corosync auth key\n";
- $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
+ $cmd = ['rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq',
"[$host]:$authfile $clusterconf", $tmpdir];
system(@$cmd) == 0 || die "can't rsync data from host '$host'\n";
}
print "OK\n" if !$printqmsg;
- # system("systemctl start clvm");
-
my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
print "generating node certificates\n";
- PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
+ PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
print "merge known_hosts file\n";
PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
}});
__PACKAGE__->register_method ({
- name => 'status',
+ name => 'status',
path => 'status',
method => 'GET',
description => "Displays the local view of the cluster status.",
properties => {},
},
returns => { type => 'null' },
-
+
code => sub {
my ($param) = @_;
- PVE::Cluster::check_corosync_conf_exists();
+ PVE::Corosync::check_conf_exists();
my $cmd = ['corosync-quorumtool', '-siH'];
}});
__PACKAGE__->register_method ({
- name => 'nodes',
+ name => 'nodes',
path => 'nodes',
method => 'GET',
description => "Displays the local view of the cluster nodes.",
properties => {},
},
returns => { type => 'null' },
-
+
code => sub {
my ($param) = @_;
- PVE::Cluster::check_corosync_conf_exists();
+ PVE::Corosync::check_conf_exists();
my $cmd = ['corosync-quorumtool', '-l'];
}});
__PACKAGE__->register_method ({
- name => 'expected',
+ name => 'expected',
path => 'expected',
method => 'PUT',
description => "Tells corosync a new value of expected votes.",
},
},
returns => { type => 'null' },
-
+
code => sub {
my ($param) = @_;
- PVE::Cluster::check_corosync_conf_exists();
+ PVE::Corosync::check_conf_exists();
my $cmd = ['corosync-quorumtool', '-e', $param->{expected}];
}});
-sub corosync_update_nodelist {
- my ($conf, $nodelist) = @_;
-
- delete $conf->{digest};
-
- my $version = PVE::Cluster::corosync_conf_version($conf);
- PVE::Cluster::corosync_conf_version($conf, undef, $version + 1);
-
- my $children = [];
- foreach my $v (values %$nodelist) {
- next if !($v->{ring0_addr} || $v->{name});
- my $kv = [];
- foreach my $k (keys %$v) {
- push @$kv, { key => $k, value => $v->{$k} };
- }
- my $ns = { section => 'node', children => $kv };
- push @$children, $ns;
- }
-
- foreach my $main (@{$conf->{children}}) {
- next if !defined($main->{section});
- if ($main->{section} eq 'nodelist') {
- $main->{children} = $children;
- last;
- }
- }
-
-
- PVE::Cluster::cfs_write_file("corosync.conf.new", $conf);
-
- rename("/etc/pve/corosync.conf.new", "/etc/pve/corosync.conf")
- || die "activate corosync.conf.new failed - $!\n";
-}
-
-sub corosync_nodelist {
- my ($conf) = @_;
-
- my $nodelist = {};
-
- foreach my $main (@{$conf->{children}}) {
- next if !defined($main->{section});
- if ($main->{section} eq 'nodelist') {
- foreach my $ne (@{$main->{children}}) {
- next if !defined($ne->{section}) || ($ne->{section} ne 'node');
- my $node = { quorum_votes => 1 };
- my $name;
- foreach my $child (@{$ne->{children}}) {
- next if !defined($child->{key});
- $node->{$child->{key}} = $child->{value};
- # use 'name' over 'ring0_addr' if set
- if ($child->{key} eq 'name') {
- delete $nodelist->{$name} if $name;
- $name = $child->{value};
- $nodelist->{$name} = $node;
- } elsif(!$name && $child->{key} eq 'ring0_addr') {
- $name = $child->{value};
- $nodelist->{$name} = $node;
- }
- }
- }
- }
- }
-
- return $nodelist;
-}
-
-# get a hash representation of the corosync config totem section
-sub corosync_totem_config {
- my ($conf) = @_;
-
- my $res = {};
-
- foreach my $main (@{$conf->{children}}) {
- next if !defined($main->{section}) ||
- $main->{section} ne 'totem';
-
- foreach my $e (@{$main->{children}}) {
-
- if ($e->{section} && $e->{section} eq 'interface') {
- my $entry = {};
-
- $res->{interface} = {};
-
- foreach my $child (@{$e->{children}}) {
- next if !defined($child->{key});
- $entry->{$child->{key}} = $child->{value};
- if($child->{key} eq 'ringnumber') {
- $res->{interface}->{$child->{value}} = $entry;
- }
- }
-
- } elsif ($e->{key}) {
- $res->{$e->{key}} = $e->{value};
- }
- }
- }
-
- return $res;
-}
-
__PACKAGE__->register_method ({
- name => 'updatecerts',
+ name => 'updatecerts',
path => 'updatecerts',
method => 'PUT',
description => "Update node certificates (and generate all needed files/directories).",
description => 'the migration network used to detect the local migration IP',
optional => 1,
},
+ 'run-command' => {
+ type => 'boolean',
+ description => 'Run a command with a tcp socket as standard input.'
+ .' The IP address and port are printed via this'
+ ." command's stdandard output first, each on a separate line.",
+ optional => 1,
+ },
+ 'extra-args' => PVE::JSONSchema::get_standard_option('extra-args'),
},
},
returns => { type => 'null'},
return undef;
}
+ my $network = $param->{migration_network};
if ($param->{get_migration_ip}) {
- my $network = $param->{migration_network};
+ die "cannot use --run-command with --get_migration_ip\n"
+ if $param->{'run-command'};
if (my $ip = PVE::Cluster::get_local_migration_ip($network)) {
print "ip: '$ip'\n";
} else {
return undef;
}
+ if ($param->{'run-command'}) {
+ my $cmd = $param->{'extra-args'};
+ die "missing command\n"
+ if !$cmd || !scalar(@$cmd);
+
+ # Get an ip address to listen on, and find a free migration port
+ my ($ip, $family);
+ if (defined($network)) {
+ $ip = PVE::Cluster::get_local_migration_ip($network)
+ or die "failed to get migration IP address to listen on\n";
+ $family = Net::IP::ip_is_ipv6($ip) ? AF_INET6 : AF_INET;
+ } else {
+ my $nodename = PVE::INotify::nodename();
+ ($ip, $family) = PVE::Network::get_ip_from_hostname($nodename, 0);
+ }
+ my $port = PVE::Tools::next_migrate_port($family, $ip);
+
+ # Wait for a client
+ my $socket = IO::Socket::IP->new(
+ Listen => 1,
+ ReuseAddr => 1,
+ Family => $family,
+ Proto => &Socket::IPPROTO_TCP,
+ GetAddrInfoFlags => 0,
+ LocalAddr => $ip,
+ LocalPort => $port,
+ ) or die "failed to open socket: $!\n";
+ print "$ip\n$port\n";
+ *STDOUT->flush();
+ alarm 0;
+ local $SIG{ALRM} = sub { die "timed out waiting for client\n" };
+ alarm 30;
+ my $client = $socket->accept;
+ alarm 0;
+ close($socket);
+
+ # We want that the command talks over the TCP socket and takes
+ # ownership of it, so that when it closes it the connection is
+ # terminated, so we need to be able to close the socket. So we
+ # can't really use PVE::Tools::run_command().
+ my $pid = fork();
+ die "fork failed: $!\n" if !defined($pid);
+ if (!$pid) {
+ POSIX::dup2(fileno($client), 0);
+ POSIX::dup2(fileno($client), 1);
+ close($client);
+ exec {$cmd->[0]} @$cmd or do {
+ warn "exec failed: $!\n";
+ POSIX::_exit(1);
+ };
+ }
+ close($client);
+ if (waitpid($pid, 0) != $pid) {
+ kill(9 => $pid);
+ 1 while waitpid($pid, 0) != $pid;
+ }
+ if (my $sig = ($? & 127)) {
+ die "got signal $sig\n";
+ } elsif (my $exitcode = ($? >> 8)) {
+ die "exit code $exitcode\n";
+ }
+ return undef;
+ }
+
print "tunnel online\n";
*STDOUT->flush();
nodes => [ __PACKAGE__, 'nodes' ],
expected => [ __PACKAGE__, 'expected', ['expected']],
updatecerts => [ __PACKAGE__, 'updatecerts', []],
- mtunnel => [ __PACKAGE__, 'mtunnel', []],
+ mtunnel => [ __PACKAGE__, 'mtunnel', ['extra-args']],
};
1;
projects / pve-cluster.git / blobdiff