data/PVE/Corosync.pm

   1 package PVE::Corosync;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Clone 'clone';
   7 use Digest::SHA;
   8 use Net::IP qw(ip_is_ipv6);
   9 use Scalar::Util qw(weaken);
  10 use Socket qw(AF_INET AF_INET6 inet_ntop);
  11
  12 use PVE::Cluster;
  13 use PVE::JSONSchema;
  14 use PVE::Tools;
  15 use PVE::Tools qw($IPV4RE $IPV6RE);
  16
  17 my $basedir = "/etc/pve";
  18
  19 my $conf_array_sections = {
  20     node => 1,
  21     interface => 1,
  22 };
  23
  24 my $corosync_link_format = {
  25     address => {
  26         default_key => 1,
  27         type => 'string', format => 'address',
  28         format_description => 'IP',
  29         description => "Hostname (or IP) of this corosync link address.",
  30     },
  31     priority => {
  32         optional => 1,
  33         type => 'integer',
  34         minimum => 0,
  35         maximum => 255,
  36         default => 0,
  37         description => "The priority for the link when knet is used in 'passive' mode. Lower value means higher priority.",
  38     },
  39 };
  40 my $corosync_link_desc = {
  41     type => 'string', format => $corosync_link_format,
  42     description => "Address and priority information of a single corosync link.",
  43     optional => 1,
  44 };
  45 PVE::JSONSchema::register_standard_option("corosync-link", $corosync_link_desc);
  46
  47 sub parse_corosync_link {
  48     my ($value) = @_;
  49
  50     return undef if !defined($value);
  51
  52     return PVE::JSONSchema::parse_property_string($corosync_link_format, $value);
  53 }
  54
  55 # a very simply parser ...
  56 sub parse_conf {
  57     my ($filename, $raw) = @_;
  58
  59     return {} if !$raw;
  60
  61     my $digest = Digest::SHA::sha1_hex(defined($raw) ? $raw : '');
  62
  63     $raw =~ s/#.*$//mg;
  64     $raw =~ s/\r?\n/ /g;
  65     $raw =~ s/\s+/ /g;
  66     $raw =~ s/^\s+//;
  67     $raw =~ s/\s*$//;
  68
  69     my @tokens = split(/\s/, $raw);
  70
  71     my $conf = { 'main' => {} };
  72
  73     my $stack = [];
  74     my $section = $conf->{main};
  75
  76     while (defined(my $token = shift @tokens)) {
  77         my $nexttok = $tokens[0];
  78
  79         if ($nexttok && ($nexttok eq '{')) {
  80             shift @tokens; # skip '{'
  81             my $new_section = {};
  82             if ($conf_array_sections->{$token}) {
  83                 $section->{$token} = [] if !defined($section->{$token});
  84                 push @{$section->{$token}}, $new_section;
  85             } elsif (!defined($section->{$token})) {
  86                 $section->{$token} = $new_section;
  87             } else {
  88                 die "section '$token' already exists and not marked as array!\n";
  89             }
  90             push @$stack, $section;
  91             $section = $new_section;
  92             next;
  93         }
  94
  95         if ($token eq '}') {
  96             $section = pop @$stack;
  97             die "parse error - uncexpected '}'\n" if !$section;
  98             next;
  99         }
 100
 101         my $key = $token;
 102         die "missing ':' after key '$key'\n" if ! ($key =~ s/:$//);
 103
 104         die "parse error - no value for '$key'\n" if !defined($nexttok);
 105         my $value = shift @tokens;
 106
 107         $section->{$key} = $value;
 108     }
 109
 110     # make working with the config way easier
 111     my ($totem, $nodelist) = $conf->{main}->@{"totem", "nodelist"};
 112
 113     $nodelist->{node} = {
 114         map {
 115             $_->{name} // $_->{ring0_addr} => $_
 116         } @{$nodelist->{node}}
 117     };
 118     $totem->{interface} = {
 119         map {
 120             $_->{linknumber} // $_->{ringnumber} => $_
 121         } @{$totem->{interface}}
 122     };
 123
 124     $conf->{digest} = $digest;
 125
 126     return $conf;
 127 }
 128
 129 sub write_conf {
 130     my ($filename, $conf) = @_;
 131
 132     my $c = clone($conf->{main}) // die "no main section";
 133
 134     # retransform back for easier dumping
 135     my $hash_to_array = sub {
 136         my ($hash) = @_;
 137         return [ $hash->@{sort keys %$hash} ];
 138     };
 139
 140     $c->{nodelist}->{node} = &$hash_to_array($c->{nodelist}->{node});
 141     $c->{totem}->{interface} = &$hash_to_array($c->{totem}->{interface});
 142
 143     my $dump_section_weak;
 144     $dump_section_weak = sub {
 145         my ($section, $prefix) = @_;
 146
 147         my $raw = '';
 148
 149         foreach my $k (sort keys %$section) {
 150             my $v = $section->{$k};
 151             if (ref($v) eq 'HASH') {
 152                 $raw .= $prefix . "$k {\n";
 153                 $raw .= $dump_section_weak->($v, "$prefix  ");
 154                 $raw .=  $prefix . "}\n";
 155                 $raw .= "\n" if !$prefix; # add extra newline at 1st level only
 156             } elsif (ref($v) eq 'ARRAY') {
 157                 foreach my $child (@$v) {
 158                     $raw .= $prefix . "$k {\n";
 159                     $raw .= $dump_section_weak->($child, "$prefix  ");
 160                     $raw .=  $prefix . "}\n";
 161                 }
 162             } elsif (!ref($v)) {
 163                 die "got undefined value for key '$k'!\n" if !defined($v);
 164                 $raw .= $prefix . "$k: $v\n";
 165             } else {
 166                 die "unexpected reference in config hash: $k => ". ref($v) ."\n";
 167             }
 168         }
 169
 170         return $raw;
 171     };
 172     my $dump_section = $dump_section_weak;
 173     weaken($dump_section_weak);
 174
 175     my $raw = $dump_section->($c, '');
 176
 177     return $raw;
 178 }
 179
 180 # read only - use atomic_write_conf method to write
 181 PVE::Cluster::cfs_register_file('corosync.conf', \&parse_conf);
 182 # this is read/write
 183 PVE::Cluster::cfs_register_file('corosync.conf.new', \&parse_conf,
 184                                 \&write_conf);
 185
 186 sub check_conf_exists {
 187     my ($silent) = @_;
 188
 189     $silent = $silent // 0;
 190
 191     my $exists = -f "$basedir/corosync.conf";
 192
 193     warn "Corosync config '$basedir/corosync.conf' does not exist - is this node part of a cluster?\n"
 194         if !$silent && !$exists;
 195
 196     return $exists;
 197 }
 198
 199 sub update_nodelist {
 200     my ($conf, $nodelist) = @_;
 201
 202     $conf->{main}->{nodelist}->{node} = $nodelist;
 203
 204     atomic_write_conf($conf);
 205 }
 206
 207 sub nodelist {
 208     my ($conf) = @_;
 209     return clone($conf->{main}->{nodelist}->{node});
 210 }
 211
 212 sub totem_config {
 213     my ($conf) = @_;
 214     return clone($conf->{main}->{totem});
 215 }
 216
 217 # caller must hold corosync.conf cfs lock if used in read-modify-write cycle
 218 sub atomic_write_conf {
 219     my ($conf, $no_increase_version) = @_;
 220
 221     if (!$no_increase_version) {
 222         die "invalid corosync config: unable to read config version\n"
 223             if !defined($conf->{main}->{totem}->{config_version});
 224         $conf->{main}->{totem}->{config_version}++;
 225     }
 226
 227     PVE::Cluster::cfs_write_file("corosync.conf.new", $conf);
 228
 229     rename("/etc/pve/corosync.conf.new", "/etc/pve/corosync.conf")
 230         || die "activating corosync.conf.new failed - $!\n";
 231 }
 232
 233 # for creating a new cluster with the current node
 234 # params are those from the API/CLI cluster create call
 235 sub create_conf {
 236     my ($nodename, %param) = @_;
 237
 238     my $clustername = $param{clustername};
 239     my $nodeid = $param{nodeid} || 1;
 240     my $votes = $param{votes} || 1;
 241
 242     my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 243
 244     my $link0 = PVE::Cluster::parse_corosync_link($param{link0});
 245     $link0->{address} //= $local_ip_address;
 246
 247     my $conf = {
 248         totem => {
 249             version => 2, # protocol version
 250             secauth => 'on',
 251             cluster_name => $clustername,
 252             config_version => 0,
 253             ip_version => 'ipv4-6',
 254             interface => {
 255                 0 => {
 256                     linknumber => 0,
 257                 },
 258             },
 259         },
 260         nodelist => {
 261             node => {
 262                 $nodename => {
 263                     name => $nodename,
 264                     nodeid => $nodeid,
 265                     quorum_votes => $votes,
 266                     ring0_addr => $link0->{address},
 267                 },
 268             },
 269         },
 270         quorum => {
 271             provider => 'corosync_votequorum',
 272         },
 273         logging => {
 274             to_syslog => 'yes',
 275             debug => 'off',
 276         },
 277     };
 278     my $totem = $conf->{totem};
 279
 280     $totem->{interface}->{0}->{knet_link_priority} = $link0->{priority}
 281         if defined($link0->{priority});
 282
 283     my $link1 = PVE::Cluster::parse_corosync_link($param{link1});
 284     if ($link1->{address}) {
 285         $conf->{totem}->{interface}->{1} = {
 286             linknumber => 1,
 287         };
 288         $totem->{link_mode} = 'passive';
 289         $totem->{interface}->{1}->{knet_link_priority} = $link1->{priority}
 290             if defined($link1->{priority});
 291         $conf->{nodelist}->{node}->{$nodename}->{ring1_addr} = $link1->{address};
 292     }
 293
 294     return { main => $conf };
 295 }
 296
 297 sub for_all_corosync_addresses {
 298     my ($corosync_conf, $ip_version, $func) = @_;
 299
 300     my $nodelist = nodelist($corosync_conf);
 301     return if !defined($nodelist);
 302
 303     # iterate sorted to make rules deterministic (for change detection)
 304     foreach my $node_name (sort keys %$nodelist) {
 305         my $node_config = $nodelist->{$node_name};
 306         foreach my $node_key (sort keys %$node_config) {
 307             if ($node_key =~ /^(ring|link)\d+_addr$/) {
 308                 my $node_address = $node_config->{$node_key};
 309
 310                 my($ip, $version) = resolve_hostname_like_corosync($node_address, $corosync_conf);
 311                 next if !defined($ip);
 312                 next if defined($version) && defined($ip_version) && $version != $ip_version;
 313
 314                 $func->($node_name, $ip, $version, $node_key);
 315             }
 316         }
 317     }
 318 }
 319
 320 # NOTE: Corosync actually only resolves on startup or config change, but we
 321 # currently do not have an easy way to synchronize our behaviour to that.
 322 sub resolve_hostname_like_corosync {
 323     my ($hostname, $corosync_conf) = @_;
 324
 325     my $corosync_strategy = $corosync_conf->{main}->{totem}->{ip_version};
 326     $corosync_strategy = lc ($corosync_strategy // "ipv6-4");
 327
 328     my $match_ip_and_version = sub {
 329         my ($addr) = @_;
 330
 331         return undef if !defined($addr);
 332
 333         if ($addr =~ m/^$IPV4RE$/) {
 334             return ($addr, 4);
 335         } elsif ($addr =~ m/^$IPV6RE$/) {
 336             return ($addr, 6);
 337         }
 338
 339         return undef;
 340     };
 341
 342     my ($resolved_ip, $ip_version) = $match_ip_and_version->($hostname);
 343
 344     return ($resolved_ip, $ip_version) if defined($resolved_ip);
 345
 346     my $resolved_ip4;
 347     my $resolved_ip6;
 348
 349     my @resolved_raw;
 350     eval { @resolved_raw = PVE::Tools::getaddrinfo_all($hostname); };
 351
 352     return undef if ($@ || !@resolved_raw);
 353
 354     foreach my $socket_info (@resolved_raw) {
 355         next if !$socket_info->{addr};
 356
 357         my ($family, undef, $host) = PVE::Tools::unpack_sockaddr_in46($socket_info->{addr});
 358
 359         if ($family == AF_INET && !defined($resolved_ip4)) {
 360             $resolved_ip4 = inet_ntop(AF_INET, $host);
 361         } elsif ($family == AF_INET6 && !defined($resolved_ip6)) {
 362             $resolved_ip6 = inet_ntop(AF_INET6, $host);
 363         }
 364
 365         last if defined($resolved_ip4) && defined($resolved_ip6);
 366     }
 367
 368     # corosync_strategy specifies the order in which IP addresses are resolved
 369     # by corosync. We need to match that order, to ensure we create firewall
 370     # rules for the correct address family.
 371     if ($corosync_strategy eq "ipv4") {
 372         $resolved_ip = $resolved_ip4;
 373     } elsif ($corosync_strategy eq "ipv6") {
 374         $resolved_ip = $resolved_ip6;
 375     } elsif ($corosync_strategy eq "ipv6-4") {
 376         $resolved_ip = $resolved_ip6 // $resolved_ip4;
 377     } elsif ($corosync_strategy eq "ipv4-6") {
 378         $resolved_ip = $resolved_ip4 // $resolved_ip6;
 379     }
 380
 381     return $match_ip_and_version->($resolved_ip);
 382 }
 383
 384 1;