]> git.proxmox.com Git - pve-cluster.git/blame - data/PVE/Corosync.pm
pvecm: move assert_joinable to avoid double call
[pve-cluster.git] / data / PVE / Corosync.pm
CommitLineData
b6973a89
TL
1package PVE::Corosync;
2
3use strict;
4use warnings;
5
496de919 6use Clone 'clone';
0e578bb7 7use Digest::SHA;
b01bc84d 8use Net::IP qw(ip_is_ipv6);
0e578bb7
TL
9use Scalar::Util qw(weaken);
10use Socket qw(AF_INET AF_INET6 inet_ntop);
b6973a89
TL
11
12use PVE::Cluster;
cde60d30 13use PVE::JSONSchema;
5c82c8c8
SR
14use PVE::Tools;
15use PVE::Tools qw($IPV4RE $IPV6RE);
b6973a89
TL
16
17my $basedir = "/etc/pve";
18
496de919
TL
19my $conf_array_sections = {
20 node => 1,
21 interface => 1,
22};
23
cde60d30
FG
24my $corosync_link_format = {
25 address => {
26 default_key => 1,
27 type => 'string', format => 'address',
28 format_description => 'IP',
29 description => "Hostname (or IP) of this corosync link address.",
30 },
31 priority => {
32 optional => 1,
33 type => 'integer',
34 minimum => 0,
35 maximum => 255,
36 default => 0,
37 description => "The priority for the link when knet is used in 'passive' mode. Lower value means higher priority.",
38 },
39};
40my $corosync_link_desc = {
41 type => 'string', format => $corosync_link_format,
42 description => "Address and priority information of a single corosync link.",
43 optional => 1,
44};
45PVE::JSONSchema::register_standard_option("corosync-link", $corosync_link_desc);
46
47sub parse_corosync_link {
48 my ($value) = @_;
49
50 return undef if !defined($value);
51
52 return PVE::JSONSchema::parse_property_string($corosync_link_format, $value);
53}
54
b6973a89
TL
55# a very simply parser ...
56sub parse_conf {
57 my ($filename, $raw) = @_;
58
59 return {} if !$raw;
60
61 my $digest = Digest::SHA::sha1_hex(defined($raw) ? $raw : '');
62
63 $raw =~ s/#.*$//mg;
64 $raw =~ s/\r?\n/ /g;
65 $raw =~ s/\s+/ /g;
66 $raw =~ s/^\s+//;
67 $raw =~ s/\s*$//;
68
69 my @tokens = split(/\s/, $raw);
70
496de919 71 my $conf = { 'main' => {} };
b6973a89
TL
72
73 my $stack = [];
496de919 74 my $section = $conf->{main};
b6973a89
TL
75
76 while (defined(my $token = shift @tokens)) {
77 my $nexttok = $tokens[0];
78
79 if ($nexttok && ($nexttok eq '{')) {
80 shift @tokens; # skip '{'
496de919
TL
81 my $new_section = {};
82 if ($conf_array_sections->{$token}) {
83 $section->{$token} = [] if !defined($section->{$token});
84 push @{$section->{$token}}, $new_section;
85 } elsif (!defined($section->{$token})) {
86 $section->{$token} = $new_section;
87 } else {
88 die "section '$token' already exists and not marked as array!\n";
89 }
b6973a89
TL
90 push @$stack, $section;
91 $section = $new_section;
92 next;
93 }
94
95 if ($token eq '}') {
96 $section = pop @$stack;
97 die "parse error - uncexpected '}'\n" if !$section;
98 next;
99 }
100
101 my $key = $token;
102 die "missing ':' after key '$key'\n" if ! ($key =~ s/:$//);
103
104 die "parse error - no value for '$key'\n" if !defined($nexttok);
105 my $value = shift @tokens;
106
496de919 107 $section->{$key} = $value;
b6973a89
TL
108 }
109
e7ecad20
TL
110 # make working with the config way easier
111 my ($totem, $nodelist) = $conf->{main}->@{"totem", "nodelist"};
018bbcab
TL
112
113 $nodelist->{node} = {
114 map {
115 $_->{name} // $_->{ring0_addr} => $_
116 } @{$nodelist->{node}}
117 };
118 $totem->{interface} = {
119 map {
120 $_->{linknumber} // $_->{ringnumber} => $_
121 } @{$totem->{interface}}
122 };
e7ecad20 123
b6973a89
TL
124 $conf->{digest} = $digest;
125
126 return $conf;
127}
128
b6973a89
TL
129sub write_conf {
130 my ($filename, $conf) = @_;
131
e7ecad20
TL
132 my $c = clone($conf->{main}) // die "no main section";
133
134 # retransform back for easier dumping
135 my $hash_to_array = sub {
136 my ($hash) = @_;
137 return [ $hash->@{sort keys %$hash} ];
138 };
b6973a89 139
e7ecad20
TL
140 $c->{nodelist}->{node} = &$hash_to_array($c->{nodelist}->{node});
141 $c->{totem}->{interface} = &$hash_to_array($c->{totem}->{interface});
142
0e578bb7
TL
143 my $dump_section_weak;
144 $dump_section_weak = sub {
145 my ($section, $prefix) = @_;
146
147 my $raw = '';
148
149 foreach my $k (sort keys %$section) {
150 my $v = $section->{$k};
151 if (ref($v) eq 'HASH') {
152 $raw .= $prefix . "$k {\n";
153 $raw .= $dump_section_weak->($v, "$prefix ");
154 $raw .= $prefix . "}\n";
155 $raw .= "\n" if !$prefix; # add extra newline at 1st level only
156 } elsif (ref($v) eq 'ARRAY') {
157 foreach my $child (@$v) {
158 $raw .= $prefix . "$k {\n";
159 $raw .= $dump_section_weak->($child, "$prefix ");
160 $raw .= $prefix . "}\n";
161 }
162 } elsif (!ref($v)) {
163 die "got undefined value for key '$k'!\n" if !defined($v);
164 $raw .= $prefix . "$k: $v\n";
165 } else {
166 die "unexpected reference in config hash: $k => ". ref($v) ."\n";
167 }
168 }
169
170 return $raw;
171 };
172 my $dump_section = $dump_section_weak;
173 weaken($dump_section_weak);
174
175 my $raw = $dump_section->($c, '');
b6973a89
TL
176
177 return $raw;
178}
179
2b28b160 180# read only - use atomic_write_conf method to write
b6973a89
TL
181PVE::Cluster::cfs_register_file('corosync.conf', \&parse_conf);
182# this is read/write
183PVE::Cluster::cfs_register_file('corosync.conf.new', \&parse_conf,
184 \&write_conf);
185
186sub check_conf_exists {
187 my ($silent) = @_;
188
189 $silent = $silent // 0;
190
191 my $exists = -f "$basedir/corosync.conf";
192
193 warn "Corosync config '$basedir/corosync.conf' does not exist - is this node part of a cluster?\n"
194 if !$silent && !$exists;
195
196 return $exists;
197}
198
199sub update_nodelist {
200 my ($conf, $nodelist) = @_;
201
e7ecad20 202 $conf->{main}->{nodelist}->{node} = $nodelist;
b6973a89 203
2b28b160 204 atomic_write_conf($conf);
b6973a89
TL
205}
206
207sub nodelist {
208 my ($conf) = @_;
e7ecad20 209 return clone($conf->{main}->{nodelist}->{node});
b6973a89
TL
210}
211
b6973a89
TL
212sub totem_config {
213 my ($conf) = @_;
e7ecad20 214 return clone($conf->{main}->{totem});
b6973a89
TL
215}
216
2b28b160
TL
217# caller must hold corosync.conf cfs lock if used in read-modify-write cycle
218sub atomic_write_conf {
219 my ($conf, $no_increase_version) = @_;
220
221 if (!$no_increase_version) {
222 die "invalid corosync config: unable to read config version\n"
223 if !defined($conf->{main}->{totem}->{config_version});
224 $conf->{main}->{totem}->{config_version}++;
225 }
226
227 PVE::Cluster::cfs_write_file("corosync.conf.new", $conf);
228
229 rename("/etc/pve/corosync.conf.new", "/etc/pve/corosync.conf")
230 || die "activating corosync.conf.new failed - $!\n";
231}
232
b01bc84d
TL
233# for creating a new cluster with the current node
234# params are those from the API/CLI cluster create call
235sub create_conf {
236 my ($nodename, %param) = @_;
237
238 my $clustername = $param{clustername};
239 my $nodeid = $param{nodeid} || 1;
240 my $votes = $param{votes} || 1;
241
242 my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
b01bc84d 243
046173ce
TL
244 my $link0 = PVE::Cluster::parse_corosync_link($param{link0});
245 $link0->{address} //= $local_ip_address;
b01bc84d
TL
246
247 my $conf = {
248 totem => {
249 version => 2, # protocol version
250 secauth => 'on',
251 cluster_name => $clustername,
252 config_version => 0,
046173ce 253 ip_version => 'ipv4-6',
b01bc84d
TL
254 interface => {
255 0 => {
018bbcab 256 linknumber => 0,
b01bc84d
TL
257 },
258 },
259 },
260 nodelist => {
261 node => {
262 $nodename => {
263 name => $nodename,
264 nodeid => $nodeid,
265 quorum_votes => $votes,
046173ce 266 ring0_addr => $link0->{address},
b01bc84d
TL
267 },
268 },
269 },
270 quorum => {
271 provider => 'corosync_votequorum',
272 },
273 logging => {
274 to_syslog => 'yes',
275 debug => 'off',
276 },
277 };
e7f9c8cc 278 my $totem = $conf->{totem};
b01bc84d 279
e7f9c8cc
TL
280 $totem->{interface}->{0}->{knet_link_priority} = $link0->{priority}
281 if defined($link0->{priority});
b01bc84d 282
e7f9c8cc 283 my $link1 = PVE::Cluster::parse_corosync_link($param{link1});
046173ce 284 if ($link1->{address}) {
b01bc84d 285 $conf->{totem}->{interface}->{1} = {
018bbcab 286 linknumber => 1,
b01bc84d 287 };
e7f9c8cc
TL
288 $totem->{link_mode} = 'passive';
289 $totem->{interface}->{1}->{knet_link_priority} = $link1->{priority}
290 if defined($link1->{priority});
046173ce 291 $conf->{nodelist}->{node}->{$nodename}->{ring1_addr} = $link1->{address};
b01bc84d
TL
292 }
293
294 return { main => $conf };
295}
296
5c82c8c8
SR
297sub for_all_corosync_addresses {
298 my ($corosync_conf, $ip_version, $func) = @_;
299
300 my $nodelist = nodelist($corosync_conf);
301 return if !defined($nodelist);
302
303 # iterate sorted to make rules deterministic (for change detection)
304 foreach my $node_name (sort keys %$nodelist) {
305 my $node_config = $nodelist->{$node_name};
306 foreach my $node_key (sort keys %$node_config) {
307 if ($node_key =~ /^(ring|link)\d+_addr$/) {
308 my $node_address = $node_config->{$node_key};
309
310 my($ip, $version) = resolve_hostname_like_corosync($node_address, $corosync_conf);
53d5168d 311 next if !defined($ip);
5c82c8c8
SR
312 next if defined($version) && defined($ip_version) && $version != $ip_version;
313
53d5168d 314 $func->($node_name, $ip, $version, $node_key);
5c82c8c8
SR
315 }
316 }
317 }
318}
319
320# NOTE: Corosync actually only resolves on startup or config change, but we
321# currently do not have an easy way to synchronize our behaviour to that.
322sub resolve_hostname_like_corosync {
323 my ($hostname, $corosync_conf) = @_;
324
325 my $corosync_strategy = $corosync_conf->{main}->{totem}->{ip_version};
53d5168d 326 $corosync_strategy = lc ($corosync_strategy // "ipv6-4");
5c82c8c8 327
3e067ee3
FG
328 my $match_ip_and_version = sub {
329 my ($addr) = @_;
330
331 return undef if !defined($addr);
332
333 if ($addr =~ m/^$IPV4RE$/) {
334 return ($addr, 4);
335 } elsif ($addr =~ m/^$IPV6RE$/) {
336 return ($addr, 6);
337 }
338
339 return undef;
340 };
341
342 my ($resolved_ip, $ip_version) = $match_ip_and_version->($hostname);
343
344 return ($resolved_ip, $ip_version) if defined($resolved_ip);
345
5c82c8c8
SR
346 my $resolved_ip4;
347 my $resolved_ip6;
348
349 my @resolved_raw;
350 eval { @resolved_raw = PVE::Tools::getaddrinfo_all($hostname); };
351
3e067ee3 352 return undef if ($@ || !@resolved_raw);
5c82c8c8
SR
353
354 foreach my $socket_info (@resolved_raw) {
355 next if !$socket_info->{addr};
356
357 my ($family, undef, $host) = PVE::Tools::unpack_sockaddr_in46($socket_info->{addr});
358
359 if ($family == AF_INET && !defined($resolved_ip4)) {
360 $resolved_ip4 = inet_ntop(AF_INET, $host);
361 } elsif ($family == AF_INET6 && !defined($resolved_ip6)) {
362 $resolved_ip6 = inet_ntop(AF_INET6, $host);
363 }
364
365 last if defined($resolved_ip4) && defined($resolved_ip6);
366 }
367
368 # corosync_strategy specifies the order in which IP addresses are resolved
369 # by corosync. We need to match that order, to ensure we create firewall
370 # rules for the correct address family.
5c82c8c8
SR
371 if ($corosync_strategy eq "ipv4") {
372 $resolved_ip = $resolved_ip4;
373 } elsif ($corosync_strategy eq "ipv6") {
374 $resolved_ip = $resolved_ip6;
375 } elsif ($corosync_strategy eq "ipv6-4") {
376 $resolved_ip = $resolved_ip6 // $resolved_ip4;
377 } elsif ($corosync_strategy eq "ipv4-6") {
378 $resolved_ip = $resolved_ip4 // $resolved_ip6;
379 }
380
3e067ee3 381 return $match_ip_and_version->($resolved_ip);
5c82c8c8
SR
382}
383
b6973a89 3841;