use strict;
use warnings;
-use Digest::SHA;
use Clone 'clone';
+use Digest::SHA;
use Net::IP qw(ip_is_ipv6);
+use Scalar::Util qw(weaken);
+use Socket qw(AF_INET AF_INET6 inet_ntop);
use PVE::Cluster;
+use PVE::JSONSchema;
+use PVE::Tools;
+use PVE::Tools qw($IPV4RE $IPV6RE);
my $basedir = "/etc/pve";
+our $link_addr_re = qw/^(ring|link)(\d+)_addr$/;
my $conf_array_sections = {
node => 1,
interface => 1,
};
+my $corosync_link_format = {
+ address => {
+ default_key => 1,
+ type => 'string', format => 'address',
+ format_description => 'IP',
+ description => "Hostname (or IP) of this corosync link address.",
+ },
+ priority => {
+ optional => 1,
+ type => 'integer',
+ minimum => 0,
+ maximum => 255,
+ default => 0,
+ description => "The priority for the link when knet is used in 'passive' mode. Lower value means higher priority.",
+ },
+};
+my $corosync_link_desc = {
+ type => 'string', format => $corosync_link_format,
+ description => "Address and priority information of a single corosync link.",
+ optional => 1,
+};
+PVE::JSONSchema::register_standard_option("corosync-link", $corosync_link_desc);
+
+sub parse_corosync_link {
+ my ($value) = @_;
+
+ return undef if !defined($value);
+
+ return PVE::JSONSchema::parse_property_string($corosync_link_format, $value);
+}
+
# a very simply parser ...
sub parse_conf {
my ($filename, $raw) = @_;
# make working with the config way easier
my ($totem, $nodelist) = $conf->{main}->@{"totem", "nodelist"};
- $nodelist->{node} = { map { $_->{name} // $_->{ring0_addr} => $_ } @{$nodelist->{node}} };
- $totem->{interface} = { map { $_->{ringnumber} => $_ } @{$totem->{interface}} };
+
+ $nodelist->{node} = {
+ map {
+ $_->{name} // $_->{ring0_addr} => $_
+ } @{$nodelist->{node}}
+ };
+ $totem->{interface} = {
+ map {
+ $_->{linknumber} // $_->{ringnumber} => $_
+ } @{$totem->{interface}}
+ };
$conf->{digest} = $digest;
return $conf;
}
-my $dump_section;
-$dump_section = sub {
- my ($section, $prefix) = @_;
-
- my $raw = '';
-
- foreach my $k (sort keys %$section) {
- my $v = $section->{$k};
- if (ref($v) eq 'HASH') {
- $raw .= $prefix . "$k {\n";
- $raw .= &$dump_section($v, "$prefix ");
- $raw .= $prefix . "}\n";
- $raw .= "\n" if !$prefix; # add extra newline at 1st level only
- } elsif (ref($v) eq 'ARRAY') {
- foreach my $child (@$v) {
- $raw .= $prefix . "$k {\n";
- $raw .= &$dump_section($child, "$prefix ");
- $raw .= $prefix . "}\n";
- }
- } elsif (!ref($v)) {
- $raw .= $prefix . "$k: $v\n";
- } else {
- die "unexpected reference in config hash: $k => ". ref($v) ."\n";
- }
- }
-
- return $raw;
-};
-
sub write_conf {
my ($filename, $conf) = @_;
$c->{nodelist}->{node} = &$hash_to_array($c->{nodelist}->{node});
$c->{totem}->{interface} = &$hash_to_array($c->{totem}->{interface});
- my $raw = &$dump_section($c, '');
+ my $dump_section_weak;
+ $dump_section_weak = sub {
+ my ($section, $prefix) = @_;
+
+ my $raw = '';
+
+ foreach my $k (sort keys %$section) {
+ my $v = $section->{$k};
+ if (ref($v) eq 'HASH') {
+ $raw .= $prefix . "$k {\n";
+ $raw .= $dump_section_weak->($v, "$prefix ");
+ $raw .= $prefix . "}\n";
+ $raw .= "\n" if !$prefix; # add extra newline at 1st level only
+ } elsif (ref($v) eq 'ARRAY') {
+ foreach my $child (@$v) {
+ $raw .= $prefix . "$k {\n";
+ $raw .= $dump_section_weak->($child, "$prefix ");
+ $raw .= $prefix . "}\n";
+ }
+ } elsif (!ref($v)) {
+ die "got undefined value for key '$k'!\n" if !defined($v);
+ $raw .= $prefix . "$k: $v\n";
+ } else {
+ die "unexpected reference in config hash: $k => ". ref($v) ."\n";
+ }
+ }
+
+ return $raw;
+ };
+ my $dump_section = $dump_section_weak;
+ weaken($dump_section_weak);
+
+ my $raw = $dump_section->($c, '');
return $raw;
}
\&write_conf);
sub check_conf_exists {
- my ($silent) = @_;
-
- $silent = $silent // 0;
+ my ($noerr) = @_;
my $exists = -f "$basedir/corosync.conf";
- warn "Corosync config '$basedir/corosync.conf' does not exist - is this node part of a cluster?\n"
- if !$silent && !$exists;
+ die "Error: Corosync config '$basedir/corosync.conf' does not exist - is this node part of a cluster?\n"
+ if !$noerr && !$exists;
return $exists;
}
my $votes = $param{votes} || 1;
my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
- my $ring0_addr = $param{ring0_addr} // $local_ip_address;
- my $bindnet0_addr = $param{bindnet0_addr} // $ring0_addr;
- my $use_ipv6 = ip_is_ipv6($ring0_addr);
- die "ring 0 addresses must be from same IP family!\n"
- if $use_ipv6 != ip_is_ipv6($bindnet0_addr);
+ my $link0 = parse_corosync_link($param{link0});
+ $link0->{address} //= $local_ip_address;
my $conf = {
totem => {
secauth => 'on',
cluster_name => $clustername,
config_version => 0,
- ip_version => $use_ipv6 ? 'ipv6' : 'ipv4',
+ ip_version => 'ipv4-6',
interface => {
0 => {
- bindnetaddr => $bindnet0_addr,
- ringnumber => 0,
+ linknumber => 0,
},
},
},
name => $nodename,
nodeid => $nodeid,
quorum_votes => $votes,
- ring0_addr => $ring0_addr,
+ ring0_addr => $link0->{address},
},
},
},
debug => 'off',
},
};
+ my $totem = $conf->{totem};
- die "Param bindnet1_addr set but ring1_addr not specified!\n"
- if (defined($param{bindnet1_addr}) && !defined($param{ring1_addr}));
-
- my $ring1_addr = $param{ring1_addr};
- my $bindnet1_addr = $param{bindnet1_addr} // $param{ring1_addr};
-
- if ($bindnet1_addr) {
- die "ring 1 addresses must be from same IP family as ring 0!\n"
- if $use_ipv6 != ip_is_ipv6($bindnet1_addr) ||
- $use_ipv6 != ip_is_ipv6($ring1_addr);
+ $totem->{interface}->{0}->{knet_link_priority} = $link0->{priority}
+ if defined($link0->{priority});
+ my $link1 = parse_corosync_link($param{link1});
+ if ($link1->{address}) {
$conf->{totem}->{interface}->{1} = {
- bindnetaddr => $bindnet1_addr,
- ringnumber => 1,
+ linknumber => 1,
};
- $conf->{totem}->{rrp_mode} = 'passive';
- $conf->{nodelist}->{node}->{$nodename}->{ring1_addr} = $ring1_addr;
+ $totem->{link_mode} = 'passive';
+ $totem->{interface}->{1}->{knet_link_priority} = $link1->{priority}
+ if defined($link1->{priority});
+ $conf->{nodelist}->{node}->{$nodename}->{ring1_addr} = $link1->{address};
}
return { main => $conf };
}
+# returns (\@errors, \@warnings) to the caller, does *not* 'die' or 'warn'
+# verification was successful if \@errors is empty
+sub verify_conf {
+ my ($conf) = @_;
+
+ my @errors = ();
+ my @warnings = ();
+
+ my $nodelist = nodelist($conf);
+ if (!$nodelist) {
+ push @errors, "no nodes found";
+ return (\@errors, \@warnings);
+ }
+
+ my $totem = $conf->{main}->{totem};
+ if (!$totem) {
+ push @errors, "no totem found";
+ return (\@errors, \@warnings);
+ }
+
+ if ((!defined($totem->{secauth}) || $totem->{secauth} ne 'on') &&
+ (!defined($totem->{crypto_cipher}) || $totem->{crypto_cipher} eq 'none')) {
+ push @warnings, "warning: authentication/encryption is not explicitly enabled"
+ . " (secauth / crypto_cipher / crypto_hash)";
+ }
+
+ my $interfaces = $totem->{interface};
+
+ my $verify_link_ip = sub {
+ my ($key, $link, $node) = @_;
+ my ($resolved_ip, undef) = resolve_hostname_like_corosync($link, $conf);
+ if (!defined($resolved_ip)) {
+ push @warnings, "warning: unable to resolve $key '$link' for node '$node'"
+ . " to an IP address according to Corosync's resolve strategy -"
+ . " cluster could fail on restart!";
+ } elsif ($resolved_ip ne $link) {
+ push @warnings, "warning: $key '$link' for node '$node' resolves to"
+ . " '$resolved_ip' - consider replacing it with the currently"
+ . " resolved IP address for stability";
+ }
+ };
+
+ # sort for output order stability
+ my @node_names = sort keys %$nodelist;
+
+ my $node_links = {};
+ foreach my $node (@node_names) {
+ my $options = $nodelist->{$node};
+ foreach my $opt (keys %$options) {
+ my ($linktype, $linkid) = parse_link_entry($opt);
+ next if !defined($linktype);
+ $node_links->{$node}->{$linkid} = {
+ name => "${linktype}${linkid}_addr",
+ addr => $options->{$opt},
+ };
+ }
+ }
+
+ if (%$interfaces) {
+ # if interfaces are defined, *all* links must have a matching interface
+ # definition, and vice versa
+ for my $link (0..1) {
+ my $have_interface = defined($interfaces->{$link});
+ foreach my $node (@node_names) {
+ my $linkdef = $node_links->{$node}->{$link};
+ if (defined($linkdef)) {
+ $verify_link_ip->($linkdef->{name}, $linkdef->{addr}, $node);
+ if (!$have_interface) {
+ push @errors, "node '$node' has '$linkdef->{name}', but"
+ . " there is no interface number $link configured";
+ }
+ } else {
+ if ($have_interface) {
+ push @errors, "node '$node' is missing address for"
+ . "interface number $link";
+ }
+ }
+ }
+ }
+ } else {
+ # without interfaces, only check that links are consistent among nodes
+ for my $link (0..1) {
+ my $nodes_with_link = {};
+ foreach my $node (@node_names) {
+ my $linkdef = $node_links->{$node}->{$link};
+ if (defined($linkdef)) {
+ $verify_link_ip->($linkdef->{name}, $linkdef->{addr}, $node);
+ $nodes_with_link->{$node} = 1;
+ }
+ }
+
+ if (%$nodes_with_link) {
+ foreach my $node (@node_names) {
+ if (!defined($nodes_with_link->{$node})) {
+ push @errors, "node '$node' is missing link $link,"
+ . " which is configured on other nodes";
+ }
+ }
+ }
+ }
+ }
+
+ return (\@errors, \@warnings);
+}
+
+# returns ($linktype, $linkid) with $linktype being 'ring' for now, and possibly
+# 'link' with upcoming corosync versions
+sub parse_link_entry {
+ my ($opt) = @_;
+ return (undef, undef) if $opt !~ $link_addr_re;
+ return ($1, $2);
+}
+
+sub for_all_corosync_addresses {
+ my ($corosync_conf, $ip_version, $func) = @_;
+
+ my $nodelist = nodelist($corosync_conf);
+ return if !defined($nodelist);
+
+ # iterate sorted to make rules deterministic (for change detection)
+ foreach my $node_name (sort keys %$nodelist) {
+ my $node_config = $nodelist->{$node_name};
+ foreach my $node_key (sort keys %$node_config) {
+ if ($node_key =~ $link_addr_re) {
+ my $node_address = $node_config->{$node_key};
+
+ my($ip, $version) = resolve_hostname_like_corosync($node_address, $corosync_conf);
+ next if !defined($ip);
+ next if defined($version) && defined($ip_version) && $version != $ip_version;
+
+ $func->($node_name, $ip, $version, $node_key);
+ }
+ }
+ }
+}
+
+# NOTE: Corosync actually only resolves on startup or config change, but we
+# currently do not have an easy way to synchronize our behaviour to that.
+sub resolve_hostname_like_corosync {
+ my ($hostname, $corosync_conf) = @_;
+
+ my $corosync_strategy = $corosync_conf->{main}->{totem}->{ip_version};
+ $corosync_strategy = lc ($corosync_strategy // "ipv6-4");
+
+ my $match_ip_and_version = sub {
+ my ($addr) = @_;
+
+ return undef if !defined($addr);
+
+ if ($addr =~ m/^$IPV4RE$/) {
+ return ($addr, 4);
+ } elsif ($addr =~ m/^$IPV6RE$/) {
+ return ($addr, 6);
+ }
+
+ return undef;
+ };
+
+ my ($resolved_ip, $ip_version) = $match_ip_and_version->($hostname);
+
+ return ($resolved_ip, $ip_version) if defined($resolved_ip);
+
+ my $resolved_ip4;
+ my $resolved_ip6;
+
+ my @resolved_raw;
+ eval { @resolved_raw = PVE::Tools::getaddrinfo_all($hostname); };
+
+ return undef if ($@ || !@resolved_raw);
+
+ foreach my $socket_info (@resolved_raw) {
+ next if !$socket_info->{addr};
+
+ my ($family, undef, $host) = PVE::Tools::unpack_sockaddr_in46($socket_info->{addr});
+
+ if ($family == AF_INET && !defined($resolved_ip4)) {
+ $resolved_ip4 = inet_ntop(AF_INET, $host);
+ } elsif ($family == AF_INET6 && !defined($resolved_ip6)) {
+ $resolved_ip6 = inet_ntop(AF_INET6, $host);
+ }
+
+ last if defined($resolved_ip4) && defined($resolved_ip6);
+ }
+
+ # corosync_strategy specifies the order in which IP addresses are resolved
+ # by corosync. We need to match that order, to ensure we create firewall
+ # rules for the correct address family.
+ if ($corosync_strategy eq "ipv4") {
+ $resolved_ip = $resolved_ip4;
+ } elsif ($corosync_strategy eq "ipv6") {
+ $resolved_ip = $resolved_ip6;
+ } elsif ($corosync_strategy eq "ipv6-4") {
+ $resolved_ip = $resolved_ip6 // $resolved_ip4;
+ } elsif ($corosync_strategy eq "ipv4-6") {
+ $resolved_ip = $resolved_ip4 // $resolved_ip6;
+ }
+
+ return $match_ip_and_version->($resolved_ip);
+}
+
1;
projects / pve-cluster.git / blobdiff