[pve-cluster.git] / data / PVE / Corosync.pm

package PVE::Corosync;

use strict;
use warnings;

use Clone 'clone';
use Digest::SHA;
use Net::IP qw(ip_is_ipv6);
use Scalar::Util qw(weaken);
use Socket qw(AF_INET AF_INET6 inet_ntop);

use PVE::Cluster;
use PVE::JSONSchema;
use PVE::Tools;
use PVE::Tools qw($IPV4RE $IPV6RE);

my $basedir = "/etc/pve";
our $link_addr_re = qw/^(ring|link)(\d+)_addr$/;

my $conf_array_sections = {
    node => 1,
    interface => 1,
};

my $corosync_link_format = {
    address => {
	default_key => 1,
	type => 'string', format => 'address',
	format_description => 'IP',
	description => "Hostname (or IP) of this corosync link address.",
    },
    priority => {
	optional => 1,
	type => 'integer',
	minimum => 0,
	maximum => 255,
	default => 0,
	description => "The priority for the link when knet is used in 'passive'"
		     . " mode (default). Lower value means higher priority. Only"
		     . " valid for cluster create, ignored on node add.",
    },
};
my $corosync_link_desc = {
    type => 'string', format => $corosync_link_format,
    description => "Address and priority information of a single corosync link."
		 . " (up to 8 links supported; link0..link7)",
    optional => 1,
};
PVE::JSONSchema::register_standard_option("corosync-link", $corosync_link_desc);

sub parse_corosync_link {
    my ($value) = @_;

    return undef if !defined($value);

    return PVE::JSONSchema::parse_property_string($corosync_link_format, $value);
}

sub print_corosync_link {
    my ($link) = @_;

    return undef if !defined($link);

    return PVE::JSONSchema::print_property_string($link, $corosync_link_format);
}

use constant MAX_LINK_INDEX => 7;

sub add_corosync_link_properties {
    my ($prop) = @_;

    for my $lnum (0..MAX_LINK_INDEX) {
	$prop->{"link$lnum"} = PVE::JSONSchema::get_standard_option("corosync-link");
    }

    return $prop;
}

sub extract_corosync_link_args {
    my ($args) = @_;

    my $links = {};
    for my $lnum (0..MAX_LINK_INDEX) {
	$links->{$lnum} = parse_corosync_link($args->{"link$lnum"})
	    if $args->{"link$lnum"};
    }

    return $links;
}

# a very simply parser ...
sub parse_conf {
    my ($filename, $raw) = @_;

    return {} if !$raw;

    my $digest = Digest::SHA::sha1_hex(defined($raw) ? $raw : '');

    $raw =~ s/#.*$//mg;
    $raw =~ s/\r?\n/ /g;
    $raw =~ s/\s+/ /g;
    $raw =~ s/^\s+//;
    $raw =~ s/\s*$//;

    my @tokens = split(/\s/, $raw);

    my $conf = { 'main' => {} };

    my $stack = [];
    my $section = $conf->{main};

    while (defined(my $token = shift @tokens)) {
	my $nexttok = $tokens[0];

	if ($nexttok && ($nexttok eq '{')) {
	    shift @tokens; # skip '{'
	    my $new_section = {};
	    if ($conf_array_sections->{$token}) {
		$section->{$token} = [] if !defined($section->{$token});
		push @{$section->{$token}}, $new_section;
	    } elsif (!defined($section->{$token})) {
		$section->{$token} = $new_section;
	    } else {
		die "section '$token' already exists and not marked as array!\n";
	    }
	    push @$stack, $section;
	    $section = $new_section;
	    next;
	}

	if ($token eq '}') {
	    $section = pop @$stack;
	    die "parse error - uncexpected '}'\n" if !$section;
	    next;
	}

	my $key = $token;
	die "missing ':' after key '$key'\n" if ! ($key =~ s/:$//);

	die "parse error - no value for '$key'\n" if !defined($nexttok);
	my $value = shift @tokens;

	$section->{$key} = $value;
    }

    # make working with the config way easier
    my ($totem, $nodelist) = $conf->{main}->@{"totem", "nodelist"};

    $nodelist->{node} = {
	map {
	    $_->{name} // $_->{ring0_addr} => $_
	} @{$nodelist->{node}}
    };
    $totem->{interface} = {
	map {
	    $_->{linknumber} // $_->{ringnumber} => $_
	} @{$totem->{interface}}
    };

    $conf->{digest} = $digest;

    return $conf;
}

sub write_conf {
    my ($filename, $conf) = @_;

    my $c = clone($conf->{main}) // die "no main section";

    # retransform back for easier dumping
    my $hash_to_array = sub {
	my ($hash) = @_;
	return [ $hash->@{sort keys %$hash} ];
    };

    $c->{nodelist}->{node} = &$hash_to_array($c->{nodelist}->{node});
    $c->{totem}->{interface} = &$hash_to_array($c->{totem}->{interface});

    my $dump_section_weak;
    $dump_section_weak = sub {
	my ($section, $prefix) = @_;

	my $raw = '';

	foreach my $k (sort keys %$section) {
	    my $v = $section->{$k};
	    if (ref($v) eq 'HASH') {
		$raw .= $prefix . "$k {\n";
		$raw .= $dump_section_weak->($v, "$prefix  ");
		$raw .=  $prefix . "}\n";
		$raw .= "\n" if !$prefix; # add extra newline at 1st level only
	    } elsif (ref($v) eq 'ARRAY') {
		foreach my $child (@$v) {
		    $raw .= $prefix . "$k {\n";
		    $raw .= $dump_section_weak->($child, "$prefix  ");
		    $raw .=  $prefix . "}\n";
		}
	    } elsif (!ref($v)) {
		die "got undefined value for key '$k'!\n" if !defined($v);
		$raw .= $prefix . "$k: $v\n";
	    } else {
		die "unexpected reference in config hash: $k => ". ref($v) ."\n";
	    }
	}

	return $raw;
    };
    my $dump_section = $dump_section_weak;
    weaken($dump_section_weak);

    my $raw = $dump_section->($c, '');

    return $raw;
}

# read only - use atomic_write_conf method to write
PVE::Cluster::cfs_register_file('corosync.conf', \&parse_conf);
# this is read/write
PVE::Cluster::cfs_register_file('corosync.conf.new', \&parse_conf,
				\&write_conf);

sub check_conf_exists {
    my ($noerr) = @_;

    my $exists = -f "$basedir/corosync.conf";

    die "Error: Corosync config '$basedir/corosync.conf' does not exist - is this node part of a cluster?\n"
	if !$noerr && !$exists;

    return $exists;
}

sub update_nodelist {
    my ($conf, $nodelist) = @_;

    $conf->{main}->{nodelist}->{node} = $nodelist;

    atomic_write_conf($conf);
}

sub nodelist {
    my ($conf) = @_;
    return clone($conf->{main}->{nodelist}->{node});
}

sub totem_config {
    my ($conf) = @_;
    return clone($conf->{main}->{totem});
}

# caller must hold corosync.conf cfs lock if used in read-modify-write cycle
sub atomic_write_conf {
    my ($conf, $no_increase_version) = @_;

    if (!$no_increase_version) {
	die "invalid corosync config: unable to read config version\n"
	    if !defined($conf->{main}->{totem}->{config_version});
	$conf->{main}->{totem}->{config_version}++;
    }

    PVE::Cluster::cfs_write_file("corosync.conf.new", $conf);

    rename("/etc/pve/corosync.conf.new", "/etc/pve/corosync.conf")
	|| die "activating corosync.conf.new failed - $!\n";
}

# for creating a new cluster with the current node
# params are those from the API/CLI cluster create call
sub create_conf {
    my ($nodename, $param) = @_;

    my $clustername = $param->{clustername};
    my $nodeid = $param->{nodeid} || 1;
    my $votes = $param->{votes} || 1;

    my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);

    my $links = extract_corosync_link_args($param);

    # if no links given, fall back to local IP as link0
    $links->{0} = { address => $local_ip_address }
	if !%$links;

    my $conf = {
	totem => {
	    version => 2, # protocol version
	    secauth => 'on',
	    cluster_name => $clustername,
	    config_version => 0,
	    ip_version => 'ipv4-6',
	    link_mode => 'passive',
	    interface => {},
	},
	nodelist => {
	    node => {
		$nodename => {
		    name => $nodename,
		    nodeid => $nodeid,
		    quorum_votes => $votes,
		},
	    },
	},
	quorum => {
	    provider => 'corosync_votequorum',
	},
	logging => {
	    to_syslog => 'yes',
	    debug => 'off',
	},
    };
    my $totem = $conf->{totem};
    my $node = $conf->{nodelist}->{node}->{$nodename};

    foreach my $lnum (keys %$links) {
	my $link = $links->{$lnum};

	$totem->{interface}->{$lnum} = { linknumber => $lnum };

	my $prio = $link->{priority};
	$totem->{interface}->{$lnum}->{knet_link_priority} = $prio if $prio;

	$node->{"ring${lnum}_addr"} = $link->{address};
    }

    return { main => $conf };
}

# returns (\@errors, \@warnings) to the caller, does *not* 'die' or 'warn'
# verification was successful if \@errors is empty
sub verify_conf {
    my ($conf) = @_;

    my @errors = ();
    my @warnings = ();

    my $nodelist = nodelist($conf);
    if (!$nodelist) {
	push @errors, "no nodes found";
	return (\@errors, \@warnings);
    }

    my $totem = $conf->{main}->{totem};
    if (!$totem) {
	push @errors, "no totem found";
	return (\@errors, \@warnings);
    }

    if ((!defined($totem->{secauth}) || $totem->{secauth} ne 'on') &&
	(!defined($totem->{crypto_cipher}) || $totem->{crypto_cipher} eq 'none')) {
	push @warnings, "warning: authentication/encryption is not explicitly enabled"
	    . " (secauth / crypto_cipher / crypto_hash)";
    }

    my $interfaces = $totem->{interface};

    my $verify_link_ip = sub {
	my ($key, $link, $node) = @_;
	my ($resolved_ip, undef) = resolve_hostname_like_corosync($link, $conf);
	if (!defined($resolved_ip)) {
	    push @warnings, "warning: unable to resolve $key '$link' for node '$node'"
		. " to an IP address according to Corosync's resolve strategy -"
		. " cluster could fail on restart!";
	} elsif ($resolved_ip ne $link) {
	    push @warnings, "warning: $key '$link' for node '$node' resolves to"
		. " '$resolved_ip' - consider replacing it with the currently"
		. " resolved IP address for stability";
	}
    };

    # sort for output order stability
    my @node_names = sort keys %$nodelist;

    my $node_links = {};
    foreach my $node (@node_names) {
	my $options = $nodelist->{$node};
	foreach my $opt (keys %$options) {
	    my ($linktype, $linkid) = parse_link_entry($opt);
	    next if !defined($linktype);
	    $node_links->{$node}->{$linkid} = {
		name => "${linktype}${linkid}_addr",
		addr => $options->{$opt},
	    };
	}
    }

    if (%$interfaces) {
	# if interfaces are defined, *all* links must have a matching interface
	# definition, and vice versa
	for my $link (0..MAX_LINK_INDEX) {
	    my $have_interface = defined($interfaces->{$link});
	    foreach my $node (@node_names) {
		my $linkdef = $node_links->{$node}->{$link};
		if (defined($linkdef)) {
		    $verify_link_ip->($linkdef->{name}, $linkdef->{addr}, $node);
		    if (!$have_interface) {
			push @errors, "node '$node' has '$linkdef->{name}', but"
			    . " there is no interface number $link configured";
		    }
		} else {
		    if ($have_interface) {
			push @errors, "node '$node' is missing address for"
			    . "interface number $link";
		    }
		}
	    }
	}
    } else {
	# without interfaces, only check that links are consistent among nodes
	for my $link (0..MAX_LINK_INDEX) {
	    my $nodes_with_link = {};
	    foreach my $node (@node_names) {
		my $linkdef = $node_links->{$node}->{$link};
		if (defined($linkdef)) {
		    $verify_link_ip->($linkdef->{name}, $linkdef->{addr}, $node);
		    $nodes_with_link->{$node} = 1;
		}
	    }

	    if (%$nodes_with_link) {
		foreach my $node (@node_names) {
		    if (!defined($nodes_with_link->{$node})) {
			push @errors, "node '$node' is missing link $link,"
			    . " which is configured on other nodes";
		    }
		}
	    }
	}
    }

    return (\@errors, \@warnings);
}

# returns ($linktype, $linkid) with $linktype being 'ring' for now, and possibly
# 'link' with upcoming corosync versions
sub parse_link_entry {
    my ($opt) = @_;
    return (undef, undef) if $opt !~ $link_addr_re;
    return ($1, $2);
}

sub for_all_corosync_addresses {
    my ($corosync_conf, $ip_version, $func) = @_;

    my $nodelist = nodelist($corosync_conf);
    return if !defined($nodelist);

    # iterate sorted to make rules deterministic (for change detection)
    foreach my $node_name (sort keys %$nodelist) {
	my $node_config = $nodelist->{$node_name};
	foreach my $node_key (sort keys %$node_config) {
	    if ($node_key =~ $link_addr_re) {
		my $node_address = $node_config->{$node_key};

		my($ip, $version) = resolve_hostname_like_corosync($node_address, $corosync_conf);
		next if !defined($ip);
		next if defined($version) && defined($ip_version) && $version != $ip_version;

		$func->($node_name, $ip, $version, $node_key);
	    }
	}
    }
}

# NOTE: Corosync actually only resolves on startup or config change, but we
# currently do not have an easy way to synchronize our behaviour to that.
sub resolve_hostname_like_corosync {
    my ($hostname, $corosync_conf) = @_;

    my $corosync_strategy = $corosync_conf->{main}->{totem}->{ip_version};
    $corosync_strategy = lc ($corosync_strategy // "ipv6-4");

    my $match_ip_and_version = sub {
	my ($addr) = @_;

	return undef if !defined($addr);

	if ($addr =~ m/^$IPV4RE$/) {
	    return ($addr, 4);
	} elsif ($addr =~ m/^$IPV6RE$/) {
	    return ($addr, 6);
	}

	return undef;
    };

    my ($resolved_ip, $ip_version) = $match_ip_and_version->($hostname);

    return ($resolved_ip, $ip_version) if defined($resolved_ip);

    my $resolved_ip4;
    my $resolved_ip6;

    my @resolved_raw;
    eval { @resolved_raw = PVE::Tools::getaddrinfo_all($hostname); };

    return undef if ($@ || !@resolved_raw);

    foreach my $socket_info (@resolved_raw) {
	next if !$socket_info->{addr};

	my ($family, undef, $host) = PVE::Tools::unpack_sockaddr_in46($socket_info->{addr});

	if ($family == AF_INET && !defined($resolved_ip4)) {
	    $resolved_ip4 = inet_ntop(AF_INET, $host);
	} elsif ($family == AF_INET6 && !defined($resolved_ip6)) {
	    $resolved_ip6 = inet_ntop(AF_INET6, $host);
	}

	last if defined($resolved_ip4) && defined($resolved_ip6);
    }

    # corosync_strategy specifies the order in which IP addresses are resolved
    # by corosync. We need to match that order, to ensure we create firewall
    # rules for the correct address family.
    if ($corosync_strategy eq "ipv4") {
	$resolved_ip = $resolved_ip4;
    } elsif ($corosync_strategy eq "ipv6") {
	$resolved_ip = $resolved_ip6;
    } elsif ($corosync_strategy eq "ipv6-4") {
	$resolved_ip = $resolved_ip6 // $resolved_ip4;
    } elsif ($corosync_strategy eq "ipv4-6") {
	$resolved_ip = $resolved_ip4 // $resolved_ip6;
    }

    return $match_ip_and_version->($resolved_ip);
}

1;
Commit	Line	Data
b6973a89 TL	1	package PVE::Corosync;
	2
	3	use strict;
	4	use warnings;
	5
496de919	6	use Clone 'clone';
0e578bb7	7	use Digest::SHA;
b01bc84d	8	use Net::IP qw(ip_is_ipv6);
0e578bb7 TL	9	use Scalar::Util qw(weaken);
0e578bb7 TL	10	use Socket qw(AF_INET AF_INET6 inet_ntop);
b6973a89 TL	11
b6973a89 TL	12	use PVE::Cluster;
cde60d30	13	use PVE::JSONSchema;
5c82c8c8 SR	14	use PVE::Tools;
5c82c8c8 SR	15	use PVE::Tools qw($IPV4RE $IPV6RE);
b6973a89 TL	16
b6973a89 TL	17	my $basedir = "/etc/pve";
b2cae1f8	18	our $link_addr_re = qw/^(ring\|link)(\d+)_addr$/;
b6973a89	19
496de919 TL	20	my $conf_array_sections = {
	21	node => 1,
	22	interface => 1,
	23	};
	24
cde60d30 FG	25	my $corosync_link_format = {
	26	address => {
	27	default_key => 1,
	28	type => 'string', format => 'address',
	29	format_description => 'IP',
	30	description => "Hostname (or IP) of this corosync link address.",
	31	},
	32	priority => {
	33	optional => 1,
	34	type => 'integer',
	35	minimum => 0,
	36	maximum => 255,
	37	default => 0,
8ef581e4 SR	38	description => "The priority for the link when knet is used in 'passive'"
	39	. " mode (default). Lower value means higher priority. Only"
	40	. " valid for cluster create, ignored on node add.",
cde60d30 FG	41	},
	42	};
	43	my $corosync_link_desc = {
	44	type => 'string', format => $corosync_link_format,
8ef581e4 SR	45	description => "Address and priority information of a single corosync link."
8ef581e4 SR	46	. " (up to 8 links supported; link0..link7)",
cde60d30 FG	47	optional => 1,
	48	};
	49	PVE::JSONSchema::register_standard_option("corosync-link", $corosync_link_desc);
	50
	51	sub parse_corosync_link {
	52	my ($value) = @_;
	53
	54	return undef if !defined($value);
	55
	56	return PVE::JSONSchema::parse_property_string($corosync_link_format, $value);
	57	}
	58
8ef581e4 SR	59	sub print_corosync_link {
	60	my ($link) = @_;
	61
	62	return undef if !defined($link);
	63
	64	return PVE::JSONSchema::print_property_string($link, $corosync_link_format);
	65	}
	66
	67	use constant MAX_LINK_INDEX => 7;
	68
	69	sub add_corosync_link_properties {
	70	my ($prop) = @_;
	71
	72	for my $lnum (0..MAX_LINK_INDEX) {
	73	$prop->{"link$lnum"} = PVE::JSONSchema::get_standard_option("corosync-link");
	74	}
	75
	76	return $prop;
	77	}
	78
	79	sub extract_corosync_link_args {
	80	my ($args) = @_;
	81
	82	my $links = {};
	83	for my $lnum (0..MAX_LINK_INDEX) {
	84	$links->{$lnum} = parse_corosync_link($args->{"link$lnum"})
	85	if $args->{"link$lnum"};
	86	}
	87
	88	return $links;
	89	}
	90
b6973a89 TL	91	# a very simply parser ...
	92	sub parse_conf {
	93	my ($filename, $raw) = @_;
	94
	95	return {} if !$raw;
	96
	97	my $digest = Digest::SHA::sha1_hex(defined($raw) ? $raw : '');
	98
	99	$raw =~ s/#.*$//mg;
	100	$raw =~ s/\r?\n/ /g;
	101	$raw =~ s/\s+/ /g;
	102	$raw =~ s/^\s+//;
	103	$raw =~ s/\s*$//;
	104
	105	my @tokens = split(/\s/, $raw);
	106
496de919	107	my $conf = { 'main' => {} };
b6973a89 TL	108
b6973a89 TL	109	my $stack = [];
496de919	110	my $section = $conf->{main};
b6973a89 TL	111
	112	while (defined(my $token = shift @tokens)) {
	113	my $nexttok = $tokens[0];
	114
	115	if ($nexttok && ($nexttok eq '{')) {
	116	shift @tokens; # skip '{'
496de919 TL	117	my $new_section = {};
	118	if ($conf_array_sections->{$token}) {
	119	$section->{$token} = [] if !defined($section->{$token});
	120	push @{$section->{$token}}, $new_section;
	121	} elsif (!defined($section->{$token})) {
	122	$section->{$token} = $new_section;
	123	} else {
	124	die "section '$token' already exists and not marked as array!\n";
	125	}
b6973a89 TL	126	push @$stack, $section;
	127	$section = $new_section;
	128	next;
	129	}
	130
	131	if ($token eq '}') {
	132	$section = pop @$stack;
	133	die "parse error - uncexpected '}'\n" if !$section;
	134	next;
	135	}
	136
	137	my $key = $token;
	138	die "missing ':' after key '$key'\n" if ! ($key =~ s/:$//);
	139
	140	die "parse error - no value for '$key'\n" if !defined($nexttok);
	141	my $value = shift @tokens;
	142
496de919	143	$section->{$key} = $value;
b6973a89 TL	144	}
b6973a89 TL	145
e7ecad20 TL	146	# make working with the config way easier
e7ecad20 TL	147	my ($totem, $nodelist) = $conf->{main}->@{"totem", "nodelist"};
018bbcab TL	148
	149	$nodelist->{node} = {
	150	map {
	151	$_->{name} // $_->{ring0_addr} => $_
	152	} @{$nodelist->{node}}
	153	};
	154	$totem->{interface} = {
	155	map {
	156	$_->{linknumber} // $_->{ringnumber} => $_
	157	} @{$totem->{interface}}
	158	};
e7ecad20	159
b6973a89 TL	160	$conf->{digest} = $digest;
	161
	162	return $conf;
	163	}
	164
b6973a89 TL	165	sub write_conf {
	166	my ($filename, $conf) = @_;
	167
e7ecad20 TL	168	my $c = clone($conf->{main}) // die "no main section";
	169
	170	# retransform back for easier dumping
	171	my $hash_to_array = sub {
	172	my ($hash) = @_;
	173	return [ $hash->@{sort keys %$hash} ];
	174	};
b6973a89	175
e7ecad20 TL	176	$c->{nodelist}->{node} = &$hash_to_array($c->{nodelist}->{node});
	177	$c->{totem}->{interface} = &$hash_to_array($c->{totem}->{interface});
	178
0e578bb7 TL	179	my $dump_section_weak;
	180	$dump_section_weak = sub {
	181	my ($section, $prefix) = @_;
	182
	183	my $raw = '';
	184
	185	foreach my $k (sort keys %$section) {
	186	my $v = $section->{$k};
	187	if (ref($v) eq 'HASH') {
	188	$raw .= $prefix . "$k {\n";
	189	$raw .= $dump_section_weak->($v, "$prefix ");
	190	$raw .= $prefix . "}\n";
	191	$raw .= "\n" if !$prefix; # add extra newline at 1st level only
	192	} elsif (ref($v) eq 'ARRAY') {
	193	foreach my $child (@$v) {
	194	$raw .= $prefix . "$k {\n";
	195	$raw .= $dump_section_weak->($child, "$prefix ");
	196	$raw .= $prefix . "}\n";
	197	}
	198	} elsif (!ref($v)) {
	199	die "got undefined value for key '$k'!\n" if !defined($v);
	200	$raw .= $prefix . "$k: $v\n";
	201	} else {
	202	die "unexpected reference in config hash: $k => ". ref($v) ."\n";
	203	}
	204	}
	205
	206	return $raw;
	207	};
	208	my $dump_section = $dump_section_weak;
	209	weaken($dump_section_weak);
	210
	211	my $raw = $dump_section->($c, '');
b6973a89 TL	212
	213	return $raw;
	214	}
	215
2b28b160	216	# read only - use atomic_write_conf method to write
b6973a89 TL	217	PVE::Cluster::cfs_register_file('corosync.conf', \&parse_conf);
	218	# this is read/write
	219	PVE::Cluster::cfs_register_file('corosync.conf.new', \&parse_conf,
	220	\&write_conf);
	221
	222	sub check_conf_exists {
75a3d341	223	my ($noerr) = @_;
b6973a89 TL	224
	225	my $exists = -f "$basedir/corosync.conf";
	226
75a3d341 SR	227	die "Error: Corosync config '$basedir/corosync.conf' does not exist - is this node part of a cluster?\n"
75a3d341 SR	228	if !$noerr && !$exists;
b6973a89 TL	229
	230	return $exists;
	231	}
	232
	233	sub update_nodelist {
	234	my ($conf, $nodelist) = @_;
	235
e7ecad20	236	$conf->{main}->{nodelist}->{node} = $nodelist;
b6973a89	237
2b28b160	238	atomic_write_conf($conf);
b6973a89 TL	239	}
	240
	241	sub nodelist {
	242	my ($conf) = @_;
e7ecad20	243	return clone($conf->{main}->{nodelist}->{node});
b6973a89 TL	244	}
b6973a89 TL	245
b6973a89 TL	246	sub totem_config {
b6973a89 TL	247	my ($conf) = @_;
e7ecad20	248	return clone($conf->{main}->{totem});
b6973a89 TL	249	}
b6973a89 TL	250
2b28b160 TL	251	# caller must hold corosync.conf cfs lock if used in read-modify-write cycle
	252	sub atomic_write_conf {
	253	my ($conf, $no_increase_version) = @_;
	254
	255	if (!$no_increase_version) {
	256	die "invalid corosync config: unable to read config version\n"
	257	if !defined($conf->{main}->{totem}->{config_version});
	258	$conf->{main}->{totem}->{config_version}++;
	259	}
	260
	261	PVE::Cluster::cfs_write_file("corosync.conf.new", $conf);
	262
	263	rename("/etc/pve/corosync.conf.new", "/etc/pve/corosync.conf")
	264	\|\| die "activating corosync.conf.new failed - $!\n";
	265	}
	266
b01bc84d TL	267	# for creating a new cluster with the current node
	268	# params are those from the API/CLI cluster create call
	269	sub create_conf {
8ef581e4	270	my ($nodename, $param) = @_;
b01bc84d	271
8ef581e4 SR	272	my $clustername = $param->{clustername};
	273	my $nodeid = $param->{nodeid} \|\| 1;
	274	my $votes = $param->{votes} \|\| 1;
b01bc84d TL	275
b01bc84d TL	276	my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
b01bc84d	277
8ef581e4 SR	278	my $links = extract_corosync_link_args($param);
	279
	280	# if no links given, fall back to local IP as link0
	281	$links->{0} = { address => $local_ip_address }
	282	if !%$links;
b01bc84d TL	283
	284	my $conf = {
	285	totem => {
	286	version => 2, # protocol version
	287	secauth => 'on',
	288	cluster_name => $clustername,
	289	config_version => 0,
046173ce	290	ip_version => 'ipv4-6',
8ef581e4 SR	291	link_mode => 'passive',
8ef581e4 SR	292	interface => {},
b01bc84d TL	293	},
	294	nodelist => {
	295	node => {
	296	$nodename => {
	297	name => $nodename,
	298	nodeid => $nodeid,
	299	quorum_votes => $votes,
b01bc84d TL	300	},
	301	},
	302	},
	303	quorum => {
	304	provider => 'corosync_votequorum',
	305	},
	306	logging => {
	307	to_syslog => 'yes',
	308	debug => 'off',
	309	},
	310	};
e7f9c8cc	311	my $totem = $conf->{totem};
8ef581e4 SR	312	my $node = $conf->{nodelist}->{node}->{$nodename};
	313
	314	foreach my $lnum (keys %$links) {
	315	my $link = $links->{$lnum};
	316
	317	$totem->{interface}->{$lnum} = { linknumber => $lnum };
	318
	319	my $prio = $link->{priority};
	320	$totem->{interface}->{$lnum}->{knet_link_priority} = $prio if $prio;
b01bc84d	321
8ef581e4	322	$node->{"ring${lnum}_addr"} = $link->{address};
b01bc84d TL	323	}
	324
	325	return { main => $conf };
	326	}
	327
b2cae1f8 SR	328	# returns (\@errors, \@warnings) to the caller, does not 'die' or 'warn'
	329	# verification was successful if \@errors is empty
	330	sub verify_conf {
	331	my ($conf) = @_;
	332
	333	my @errors = ();
	334	my @warnings = ();
	335
	336	my $nodelist = nodelist($conf);
	337	if (!$nodelist) {
	338	push @errors, "no nodes found";
	339	return (\@errors, \@warnings);
	340	}
	341
	342	my $totem = $conf->{main}->{totem};
	343	if (!$totem) {
	344	push @errors, "no totem found";
	345	return (\@errors, \@warnings);
	346	}
	347
	348	if ((!defined($totem->{secauth}) \|\| $totem->{secauth} ne 'on') &&
	349	(!defined($totem->{crypto_cipher}) \|\| $totem->{crypto_cipher} eq 'none')) {
	350	push @warnings, "warning: authentication/encryption is not explicitly enabled"
	351	. " (secauth / crypto_cipher / crypto_hash)";
	352	}
	353
	354	my $interfaces = $totem->{interface};
	355
	356	my $verify_link_ip = sub {
	357	my ($key, $link, $node) = @_;
	358	my ($resolved_ip, undef) = resolve_hostname_like_corosync($link, $conf);
	359	if (!defined($resolved_ip)) {
	360	push @warnings, "warning: unable to resolve $key '$link' for node '$node'"
	361	. " to an IP address according to Corosync's resolve strategy -"
	362	. " cluster could fail on restart!";
	363	} elsif ($resolved_ip ne $link) {
	364	push @warnings, "warning: $key '$link' for node '$node' resolves to"
	365	. " '$resolved_ip' - consider replacing it with the currently"
	366	. " resolved IP address for stability";
	367	}
	368	};
	369
	370	# sort for output order stability
	371	my @node_names = sort keys %$nodelist;
	372
	373	my $node_links = {};
	374	foreach my $node (@node_names) {
	375	my $options = $nodelist->{$node};
	376	foreach my $opt (keys %$options) {
	377	my ($linktype, $linkid) = parse_link_entry($opt);
	378	next if !defined($linktype);
	379	$node_links->{$node}->{$linkid} = {
	380	name => "${linktype}${linkid}_addr",
	381	addr => $options->{$opt},
	382	};
	383	}
	384	}
	385
	386	if (%$interfaces) {
	387	# if interfaces are defined, all links must have a matching interface
	388	# definition, and vice versa
8ef581e4	389	for my $link (0..MAX_LINK_INDEX) {
b2cae1f8 SR	390	my $have_interface = defined($interfaces->{$link});
	391	foreach my $node (@node_names) {
	392	my $linkdef = $node_links->{$node}->{$link};
	393	if (defined($linkdef)) {
	394	$verify_link_ip->($linkdef->{name}, $linkdef->{addr}, $node);
	395	if (!$have_interface) {
	396	push @errors, "node '$node' has '$linkdef->{name}', but"
	397	. " there is no interface number $link configured";
	398	}
	399	} else {
	400	if ($have_interface) {
	401	push @errors, "node '$node' is missing address for"
	402	. "interface number $link";
	403	}
	404	}
	405	}
	406	}
	407	} else {
	408	# without interfaces, only check that links are consistent among nodes
8ef581e4	409	for my $link (0..MAX_LINK_INDEX) {
b2cae1f8 SR	410	my $nodes_with_link = {};
	411	foreach my $node (@node_names) {
	412	my $linkdef = $node_links->{$node}->{$link};
	413	if (defined($linkdef)) {
	414	$verify_link_ip->($linkdef->{name}, $linkdef->{addr}, $node);
	415	$nodes_with_link->{$node} = 1;
	416	}
	417	}
	418
	419	if (%$nodes_with_link) {
	420	foreach my $node (@node_names) {
	421	if (!defined($nodes_with_link->{$node})) {
	422	push @errors, "node '$node' is missing link $link,"
	423	. " which is configured on other nodes";
	424	}
	425	}
	426	}
	427	}
	428	}
	429
	430	return (\@errors, \@warnings);
	431	}
	432
	433	# returns ($linktype, $linkid) with $linktype being 'ring' for now, and possibly
	434	# 'link' with upcoming corosync versions
	435	sub parse_link_entry {
	436	my ($opt) = @_;
	437	return (undef, undef) if $opt !~ $link_addr_re;
	438	return ($1, $2);
	439	}
	440
5c82c8c8 SR	441	sub for_all_corosync_addresses {
	442	my ($corosync_conf, $ip_version, $func) = @_;
	443
	444	my $nodelist = nodelist($corosync_conf);
	445	return if !defined($nodelist);
	446
	447	# iterate sorted to make rules deterministic (for change detection)
	448	foreach my $node_name (sort keys %$nodelist) {
	449	my $node_config = $nodelist->{$node_name};
	450	foreach my $node_key (sort keys %$node_config) {
b2cae1f8	451	if ($node_key =~ $link_addr_re) {
5c82c8c8 SR	452	my $node_address = $node_config->{$node_key};
	453
	454	my($ip, $version) = resolve_hostname_like_corosync($node_address, $corosync_conf);
53d5168d	455	next if !defined($ip);
5c82c8c8 SR	456	next if defined($version) && defined($ip_version) && $version != $ip_version;
5c82c8c8 SR	457
53d5168d	458	$func->($node_name, $ip, $version, $node_key);
5c82c8c8 SR	459	}
	460	}
	461	}
	462	}
	463
	464	# NOTE: Corosync actually only resolves on startup or config change, but we
	465	# currently do not have an easy way to synchronize our behaviour to that.
	466	sub resolve_hostname_like_corosync {
	467	my ($hostname, $corosync_conf) = @_;
	468
	469	my $corosync_strategy = $corosync_conf->{main}->{totem}->{ip_version};
53d5168d	470	$corosync_strategy = lc ($corosync_strategy // "ipv6-4");
5c82c8c8	471
3e067ee3 FG	472	my $match_ip_and_version = sub {
	473	my ($addr) = @_;
	474
	475	return undef if !defined($addr);
	476
	477	if ($addr =~ m/^$IPV4RE$/) {
	478	return ($addr, 4);
	479	} elsif ($addr =~ m/^$IPV6RE$/) {
	480	return ($addr, 6);
	481	}
	482
	483	return undef;
	484	};
	485
	486	my ($resolved_ip, $ip_version) = $match_ip_and_version->($hostname);
	487
	488	return ($resolved_ip, $ip_version) if defined($resolved_ip);
	489
5c82c8c8 SR	490	my $resolved_ip4;
	491	my $resolved_ip6;
	492
	493	my @resolved_raw;
	494	eval { @resolved_raw = PVE::Tools::getaddrinfo_all($hostname); };
	495
3e067ee3	496	return undef if ($@ \|\| !@resolved_raw);
5c82c8c8 SR	497
	498	foreach my $socket_info (@resolved_raw) {
	499	next if !$socket_info->{addr};
	500
	501	my ($family, undef, $host) = PVE::Tools::unpack_sockaddr_in46($socket_info->{addr});
	502
	503	if ($family == AF_INET && !defined($resolved_ip4)) {
	504	$resolved_ip4 = inet_ntop(AF_INET, $host);
	505	} elsif ($family == AF_INET6 && !defined($resolved_ip6)) {
	506	$resolved_ip6 = inet_ntop(AF_INET6, $host);
	507	}
	508
	509	last if defined($resolved_ip4) && defined($resolved_ip6);
	510	}
	511
	512	# corosync_strategy specifies the order in which IP addresses are resolved
	513	# by corosync. We need to match that order, to ensure we create firewall
	514	# rules for the correct address family.
5c82c8c8 SR	515	if ($corosync_strategy eq "ipv4") {
	516	$resolved_ip = $resolved_ip4;
	517	} elsif ($corosync_strategy eq "ipv6") {
	518	$resolved_ip = $resolved_ip6;
	519	} elsif ($corosync_strategy eq "ipv6-4") {
	520	$resolved_ip = $resolved_ip6 // $resolved_ip4;
	521	} elsif ($corosync_strategy eq "ipv4-6") {
	522	$resolved_ip = $resolved_ip4 // $resolved_ip6;
	523	}
	524
3e067ee3	525	return $match_ip_and_version->($resolved_ip);
5c82c8c8 SR	526	}
5c82c8c8 SR	527
b6973a89	528	1;