]>
Commit | Line | Data |
---|---|---|
b6973a89 TL |
1 | package PVE::Corosync; |
2 | ||
3 | use strict; | |
4 | use warnings; | |
5 | ||
496de919 | 6 | use Clone 'clone'; |
0e578bb7 | 7 | use Digest::SHA; |
b01bc84d | 8 | use Net::IP qw(ip_is_ipv6); |
0e578bb7 TL |
9 | use Scalar::Util qw(weaken); |
10 | use Socket qw(AF_INET AF_INET6 inet_ntop); | |
b6973a89 TL |
11 | |
12 | use PVE::Cluster; | |
cde60d30 | 13 | use PVE::JSONSchema; |
5c82c8c8 SR |
14 | use PVE::Tools; |
15 | use PVE::Tools qw($IPV4RE $IPV6RE); | |
b6973a89 TL |
16 | |
17 | my $basedir = "/etc/pve"; | |
18 | ||
496de919 TL |
19 | my $conf_array_sections = { |
20 | node => 1, | |
21 | interface => 1, | |
22 | }; | |
23 | ||
cde60d30 FG |
24 | my $corosync_link_format = { |
25 | address => { | |
26 | default_key => 1, | |
27 | type => 'string', format => 'address', | |
28 | format_description => 'IP', | |
29 | description => "Hostname (or IP) of this corosync link address.", | |
30 | }, | |
31 | priority => { | |
32 | optional => 1, | |
33 | type => 'integer', | |
34 | minimum => 0, | |
35 | maximum => 255, | |
36 | default => 0, | |
37 | description => "The priority for the link when knet is used in 'passive' mode. Lower value means higher priority.", | |
38 | }, | |
39 | }; | |
40 | my $corosync_link_desc = { | |
41 | type => 'string', format => $corosync_link_format, | |
42 | description => "Address and priority information of a single corosync link.", | |
43 | optional => 1, | |
44 | }; | |
45 | PVE::JSONSchema::register_standard_option("corosync-link", $corosync_link_desc); | |
46 | ||
47 | sub parse_corosync_link { | |
48 | my ($value) = @_; | |
49 | ||
50 | return undef if !defined($value); | |
51 | ||
52 | return PVE::JSONSchema::parse_property_string($corosync_link_format, $value); | |
53 | } | |
54 | ||
b6973a89 TL |
55 | # a very simply parser ... |
56 | sub parse_conf { | |
57 | my ($filename, $raw) = @_; | |
58 | ||
59 | return {} if !$raw; | |
60 | ||
61 | my $digest = Digest::SHA::sha1_hex(defined($raw) ? $raw : ''); | |
62 | ||
63 | $raw =~ s/#.*$//mg; | |
64 | $raw =~ s/\r?\n/ /g; | |
65 | $raw =~ s/\s+/ /g; | |
66 | $raw =~ s/^\s+//; | |
67 | $raw =~ s/\s*$//; | |
68 | ||
69 | my @tokens = split(/\s/, $raw); | |
70 | ||
496de919 | 71 | my $conf = { 'main' => {} }; |
b6973a89 TL |
72 | |
73 | my $stack = []; | |
496de919 | 74 | my $section = $conf->{main}; |
b6973a89 TL |
75 | |
76 | while (defined(my $token = shift @tokens)) { | |
77 | my $nexttok = $tokens[0]; | |
78 | ||
79 | if ($nexttok && ($nexttok eq '{')) { | |
80 | shift @tokens; # skip '{' | |
496de919 TL |
81 | my $new_section = {}; |
82 | if ($conf_array_sections->{$token}) { | |
83 | $section->{$token} = [] if !defined($section->{$token}); | |
84 | push @{$section->{$token}}, $new_section; | |
85 | } elsif (!defined($section->{$token})) { | |
86 | $section->{$token} = $new_section; | |
87 | } else { | |
88 | die "section '$token' already exists and not marked as array!\n"; | |
89 | } | |
b6973a89 TL |
90 | push @$stack, $section; |
91 | $section = $new_section; | |
92 | next; | |
93 | } | |
94 | ||
95 | if ($token eq '}') { | |
96 | $section = pop @$stack; | |
97 | die "parse error - uncexpected '}'\n" if !$section; | |
98 | next; | |
99 | } | |
100 | ||
101 | my $key = $token; | |
102 | die "missing ':' after key '$key'\n" if ! ($key =~ s/:$//); | |
103 | ||
104 | die "parse error - no value for '$key'\n" if !defined($nexttok); | |
105 | my $value = shift @tokens; | |
106 | ||
496de919 | 107 | $section->{$key} = $value; |
b6973a89 TL |
108 | } |
109 | ||
e7ecad20 TL |
110 | # make working with the config way easier |
111 | my ($totem, $nodelist) = $conf->{main}->@{"totem", "nodelist"}; | |
018bbcab TL |
112 | |
113 | $nodelist->{node} = { | |
114 | map { | |
115 | $_->{name} // $_->{ring0_addr} => $_ | |
116 | } @{$nodelist->{node}} | |
117 | }; | |
118 | $totem->{interface} = { | |
119 | map { | |
120 | $_->{linknumber} // $_->{ringnumber} => $_ | |
121 | } @{$totem->{interface}} | |
122 | }; | |
e7ecad20 | 123 | |
b6973a89 TL |
124 | $conf->{digest} = $digest; |
125 | ||
126 | return $conf; | |
127 | } | |
128 | ||
b6973a89 TL |
129 | sub write_conf { |
130 | my ($filename, $conf) = @_; | |
131 | ||
e7ecad20 TL |
132 | my $c = clone($conf->{main}) // die "no main section"; |
133 | ||
134 | # retransform back for easier dumping | |
135 | my $hash_to_array = sub { | |
136 | my ($hash) = @_; | |
137 | return [ $hash->@{sort keys %$hash} ]; | |
138 | }; | |
b6973a89 | 139 | |
e7ecad20 TL |
140 | $c->{nodelist}->{node} = &$hash_to_array($c->{nodelist}->{node}); |
141 | $c->{totem}->{interface} = &$hash_to_array($c->{totem}->{interface}); | |
142 | ||
0e578bb7 TL |
143 | my $dump_section_weak; |
144 | $dump_section_weak = sub { | |
145 | my ($section, $prefix) = @_; | |
146 | ||
147 | my $raw = ''; | |
148 | ||
149 | foreach my $k (sort keys %$section) { | |
150 | my $v = $section->{$k}; | |
151 | if (ref($v) eq 'HASH') { | |
152 | $raw .= $prefix . "$k {\n"; | |
153 | $raw .= $dump_section_weak->($v, "$prefix "); | |
154 | $raw .= $prefix . "}\n"; | |
155 | $raw .= "\n" if !$prefix; # add extra newline at 1st level only | |
156 | } elsif (ref($v) eq 'ARRAY') { | |
157 | foreach my $child (@$v) { | |
158 | $raw .= $prefix . "$k {\n"; | |
159 | $raw .= $dump_section_weak->($child, "$prefix "); | |
160 | $raw .= $prefix . "}\n"; | |
161 | } | |
162 | } elsif (!ref($v)) { | |
163 | die "got undefined value for key '$k'!\n" if !defined($v); | |
164 | $raw .= $prefix . "$k: $v\n"; | |
165 | } else { | |
166 | die "unexpected reference in config hash: $k => ". ref($v) ."\n"; | |
167 | } | |
168 | } | |
169 | ||
170 | return $raw; | |
171 | }; | |
172 | my $dump_section = $dump_section_weak; | |
173 | weaken($dump_section_weak); | |
174 | ||
175 | my $raw = $dump_section->($c, ''); | |
b6973a89 TL |
176 | |
177 | return $raw; | |
178 | } | |
179 | ||
2b28b160 | 180 | # read only - use atomic_write_conf method to write |
b6973a89 TL |
181 | PVE::Cluster::cfs_register_file('corosync.conf', \&parse_conf); |
182 | # this is read/write | |
183 | PVE::Cluster::cfs_register_file('corosync.conf.new', \&parse_conf, | |
184 | \&write_conf); | |
185 | ||
186 | sub check_conf_exists { | |
75a3d341 | 187 | my ($noerr) = @_; |
b6973a89 TL |
188 | |
189 | my $exists = -f "$basedir/corosync.conf"; | |
190 | ||
75a3d341 SR |
191 | die "Error: Corosync config '$basedir/corosync.conf' does not exist - is this node part of a cluster?\n" |
192 | if !$noerr && !$exists; | |
b6973a89 TL |
193 | |
194 | return $exists; | |
195 | } | |
196 | ||
197 | sub update_nodelist { | |
198 | my ($conf, $nodelist) = @_; | |
199 | ||
e7ecad20 | 200 | $conf->{main}->{nodelist}->{node} = $nodelist; |
b6973a89 | 201 | |
2b28b160 | 202 | atomic_write_conf($conf); |
b6973a89 TL |
203 | } |
204 | ||
205 | sub nodelist { | |
206 | my ($conf) = @_; | |
e7ecad20 | 207 | return clone($conf->{main}->{nodelist}->{node}); |
b6973a89 TL |
208 | } |
209 | ||
b6973a89 TL |
210 | sub totem_config { |
211 | my ($conf) = @_; | |
e7ecad20 | 212 | return clone($conf->{main}->{totem}); |
b6973a89 TL |
213 | } |
214 | ||
2b28b160 TL |
215 | # caller must hold corosync.conf cfs lock if used in read-modify-write cycle |
216 | sub atomic_write_conf { | |
217 | my ($conf, $no_increase_version) = @_; | |
218 | ||
219 | if (!$no_increase_version) { | |
220 | die "invalid corosync config: unable to read config version\n" | |
221 | if !defined($conf->{main}->{totem}->{config_version}); | |
222 | $conf->{main}->{totem}->{config_version}++; | |
223 | } | |
224 | ||
225 | PVE::Cluster::cfs_write_file("corosync.conf.new", $conf); | |
226 | ||
227 | rename("/etc/pve/corosync.conf.new", "/etc/pve/corosync.conf") | |
228 | || die "activating corosync.conf.new failed - $!\n"; | |
229 | } | |
230 | ||
b01bc84d TL |
231 | # for creating a new cluster with the current node |
232 | # params are those from the API/CLI cluster create call | |
233 | sub create_conf { | |
234 | my ($nodename, %param) = @_; | |
235 | ||
236 | my $clustername = $param{clustername}; | |
237 | my $nodeid = $param{nodeid} || 1; | |
238 | my $votes = $param{votes} || 1; | |
239 | ||
240 | my $local_ip_address = PVE::Cluster::remote_node_ip($nodename); | |
b01bc84d | 241 | |
046173ce TL |
242 | my $link0 = PVE::Cluster::parse_corosync_link($param{link0}); |
243 | $link0->{address} //= $local_ip_address; | |
b01bc84d TL |
244 | |
245 | my $conf = { | |
246 | totem => { | |
247 | version => 2, # protocol version | |
248 | secauth => 'on', | |
249 | cluster_name => $clustername, | |
250 | config_version => 0, | |
046173ce | 251 | ip_version => 'ipv4-6', |
b01bc84d TL |
252 | interface => { |
253 | 0 => { | |
018bbcab | 254 | linknumber => 0, |
b01bc84d TL |
255 | }, |
256 | }, | |
257 | }, | |
258 | nodelist => { | |
259 | node => { | |
260 | $nodename => { | |
261 | name => $nodename, | |
262 | nodeid => $nodeid, | |
263 | quorum_votes => $votes, | |
046173ce | 264 | ring0_addr => $link0->{address}, |
b01bc84d TL |
265 | }, |
266 | }, | |
267 | }, | |
268 | quorum => { | |
269 | provider => 'corosync_votequorum', | |
270 | }, | |
271 | logging => { | |
272 | to_syslog => 'yes', | |
273 | debug => 'off', | |
274 | }, | |
275 | }; | |
e7f9c8cc | 276 | my $totem = $conf->{totem}; |
b01bc84d | 277 | |
e7f9c8cc TL |
278 | $totem->{interface}->{0}->{knet_link_priority} = $link0->{priority} |
279 | if defined($link0->{priority}); | |
b01bc84d | 280 | |
e7f9c8cc | 281 | my $link1 = PVE::Cluster::parse_corosync_link($param{link1}); |
046173ce | 282 | if ($link1->{address}) { |
b01bc84d | 283 | $conf->{totem}->{interface}->{1} = { |
018bbcab | 284 | linknumber => 1, |
b01bc84d | 285 | }; |
e7f9c8cc TL |
286 | $totem->{link_mode} = 'passive'; |
287 | $totem->{interface}->{1}->{knet_link_priority} = $link1->{priority} | |
288 | if defined($link1->{priority}); | |
046173ce | 289 | $conf->{nodelist}->{node}->{$nodename}->{ring1_addr} = $link1->{address}; |
b01bc84d TL |
290 | } |
291 | ||
292 | return { main => $conf }; | |
293 | } | |
294 | ||
5c82c8c8 SR |
295 | sub for_all_corosync_addresses { |
296 | my ($corosync_conf, $ip_version, $func) = @_; | |
297 | ||
298 | my $nodelist = nodelist($corosync_conf); | |
299 | return if !defined($nodelist); | |
300 | ||
301 | # iterate sorted to make rules deterministic (for change detection) | |
302 | foreach my $node_name (sort keys %$nodelist) { | |
303 | my $node_config = $nodelist->{$node_name}; | |
304 | foreach my $node_key (sort keys %$node_config) { | |
305 | if ($node_key =~ /^(ring|link)\d+_addr$/) { | |
306 | my $node_address = $node_config->{$node_key}; | |
307 | ||
308 | my($ip, $version) = resolve_hostname_like_corosync($node_address, $corosync_conf); | |
53d5168d | 309 | next if !defined($ip); |
5c82c8c8 SR |
310 | next if defined($version) && defined($ip_version) && $version != $ip_version; |
311 | ||
53d5168d | 312 | $func->($node_name, $ip, $version, $node_key); |
5c82c8c8 SR |
313 | } |
314 | } | |
315 | } | |
316 | } | |
317 | ||
318 | # NOTE: Corosync actually only resolves on startup or config change, but we | |
319 | # currently do not have an easy way to synchronize our behaviour to that. | |
320 | sub resolve_hostname_like_corosync { | |
321 | my ($hostname, $corosync_conf) = @_; | |
322 | ||
323 | my $corosync_strategy = $corosync_conf->{main}->{totem}->{ip_version}; | |
53d5168d | 324 | $corosync_strategy = lc ($corosync_strategy // "ipv6-4"); |
5c82c8c8 | 325 | |
3e067ee3 FG |
326 | my $match_ip_and_version = sub { |
327 | my ($addr) = @_; | |
328 | ||
329 | return undef if !defined($addr); | |
330 | ||
331 | if ($addr =~ m/^$IPV4RE$/) { | |
332 | return ($addr, 4); | |
333 | } elsif ($addr =~ m/^$IPV6RE$/) { | |
334 | return ($addr, 6); | |
335 | } | |
336 | ||
337 | return undef; | |
338 | }; | |
339 | ||
340 | my ($resolved_ip, $ip_version) = $match_ip_and_version->($hostname); | |
341 | ||
342 | return ($resolved_ip, $ip_version) if defined($resolved_ip); | |
343 | ||
5c82c8c8 SR |
344 | my $resolved_ip4; |
345 | my $resolved_ip6; | |
346 | ||
347 | my @resolved_raw; | |
348 | eval { @resolved_raw = PVE::Tools::getaddrinfo_all($hostname); }; | |
349 | ||
3e067ee3 | 350 | return undef if ($@ || !@resolved_raw); |
5c82c8c8 SR |
351 | |
352 | foreach my $socket_info (@resolved_raw) { | |
353 | next if !$socket_info->{addr}; | |
354 | ||
355 | my ($family, undef, $host) = PVE::Tools::unpack_sockaddr_in46($socket_info->{addr}); | |
356 | ||
357 | if ($family == AF_INET && !defined($resolved_ip4)) { | |
358 | $resolved_ip4 = inet_ntop(AF_INET, $host); | |
359 | } elsif ($family == AF_INET6 && !defined($resolved_ip6)) { | |
360 | $resolved_ip6 = inet_ntop(AF_INET6, $host); | |
361 | } | |
362 | ||
363 | last if defined($resolved_ip4) && defined($resolved_ip6); | |
364 | } | |
365 | ||
366 | # corosync_strategy specifies the order in which IP addresses are resolved | |
367 | # by corosync. We need to match that order, to ensure we create firewall | |
368 | # rules for the correct address family. | |
5c82c8c8 SR |
369 | if ($corosync_strategy eq "ipv4") { |
370 | $resolved_ip = $resolved_ip4; | |
371 | } elsif ($corosync_strategy eq "ipv6") { | |
372 | $resolved_ip = $resolved_ip6; | |
373 | } elsif ($corosync_strategy eq "ipv6-4") { | |
374 | $resolved_ip = $resolved_ip6 // $resolved_ip4; | |
375 | } elsif ($corosync_strategy eq "ipv4-6") { | |
376 | $resolved_ip = $resolved_ip4 // $resolved_ip6; | |
377 | } | |
378 | ||
3e067ee3 | 379 | return $match_ip_and_version->($resolved_ip); |
5c82c8c8 SR |
380 | } |
381 | ||
b6973a89 | 382 | 1; |