data/PVE/Corosync.pm

   1 package PVE::Corosync;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Clone 'clone';
   7 use Digest::SHA;
   8 use Net::IP qw(ip_is_ipv6);
   9 use Scalar::Util qw(weaken);
  10 use Socket qw(AF_INET AF_INET6 inet_ntop);
  11
  12 use PVE::Cluster;
  13 use PVE::JSONSchema;
  14 use PVE::Tools;
  15 use PVE::Tools qw($IPV4RE $IPV6RE);
  16
  17 my $basedir = "/etc/pve";
  18 our $link_addr_re = qw/^(ring|link)(\d+)_addr$/;
  19
  20 my $conf_array_sections = {
  21     node => 1,
  22     interface => 1,
  23 };
  24
  25 my $corosync_link_format = {
  26     address => {
  27         default_key => 1,
  28         type => 'string', format => 'address',
  29         format_description => 'IP',
  30         description => "Hostname (or IP) of this corosync link address.",
  31     },
  32     priority => {
  33         optional => 1,
  34         type => 'integer',
  35         minimum => 0,
  36         maximum => 255,
  37         default => 0,
  38         description => "The priority for the link when knet is used in 'passive'"
  39                      . " mode (default). Lower value means higher priority. Only"
  40                      . " valid for cluster create, ignored on node add.",
  41     },
  42 };
  43 my $corosync_link_desc = {
  44     type => 'string', format => $corosync_link_format,
  45     description => "Address and priority information of a single corosync link."
  46                  . " (up to 8 links supported; link0..link7)",
  47     optional => 1,
  48 };
  49 PVE::JSONSchema::register_standard_option("corosync-link", $corosync_link_desc);
  50
  51 sub parse_corosync_link {
  52     my ($value) = @_;
  53
  54     return undef if !defined($value);
  55
  56     return PVE::JSONSchema::parse_property_string($corosync_link_format, $value);
  57 }
  58
  59 sub print_corosync_link {
  60     my ($link) = @_;
  61
  62     return undef if !defined($link);
  63
  64     return PVE::JSONSchema::print_property_string($link, $corosync_link_format);
  65 }
  66
  67 use constant MAX_LINK_INDEX => 7;
  68
  69 sub add_corosync_link_properties {
  70     my ($prop) = @_;
  71
  72     for my $lnum (0..MAX_LINK_INDEX) {
  73         $prop->{"link$lnum"} = PVE::JSONSchema::get_standard_option("corosync-link");
  74     }
  75
  76     return $prop;
  77 }
  78
  79 sub extract_corosync_link_args {
  80     my ($args) = @_;
  81
  82     my $links = {};
  83     for my $lnum (0..MAX_LINK_INDEX) {
  84         $links->{$lnum} = parse_corosync_link($args->{"link$lnum"})
  85             if $args->{"link$lnum"};
  86     }
  87
  88     return $links;
  89 }
  90
  91 # a very simply parser ...
  92 sub parse_conf {
  93     my ($filename, $raw) = @_;
  94
  95     return {} if !$raw;
  96
  97     my $digest = Digest::SHA::sha1_hex(defined($raw) ? $raw : '');
  98
  99     $raw =~ s/#.*$//mg;
 100     $raw =~ s/\r?\n/ /g;
 101     $raw =~ s/\s+/ /g;
 102     $raw =~ s/^\s+//;
 103     $raw =~ s/\s*$//;
 104
 105     my @tokens = split(/\s/, $raw);
 106
 107     my $conf = { 'main' => {} };
 108
 109     my $stack = [];
 110     my $section = $conf->{main};
 111
 112     while (defined(my $token = shift @tokens)) {
 113         my $nexttok = $tokens[0];
 114
 115         if ($nexttok && ($nexttok eq '{')) {
 116             shift @tokens; # skip '{'
 117             my $new_section = {};
 118             if ($conf_array_sections->{$token}) {
 119                 $section->{$token} = [] if !defined($section->{$token});
 120                 push @{$section->{$token}}, $new_section;
 121             } elsif (!defined($section->{$token})) {
 122                 $section->{$token} = $new_section;
 123             } else {
 124                 die "section '$token' already exists and not marked as array!\n";
 125             }
 126             push @$stack, $section;
 127             $section = $new_section;
 128             next;
 129         }
 130
 131         if ($token eq '}') {
 132             $section = pop @$stack;
 133             die "parse error - uncexpected '}'\n" if !$section;
 134             next;
 135         }
 136
 137         my $key = $token;
 138         die "missing ':' after key '$key'\n" if ! ($key =~ s/:$//);
 139
 140         die "parse error - no value for '$key'\n" if !defined($nexttok);
 141         my $value = shift @tokens;
 142
 143         $section->{$key} = $value;
 144     }
 145
 146     # make working with the config way easier
 147     my ($totem, $nodelist) = $conf->{main}->@{"totem", "nodelist"};
 148
 149     $nodelist->{node} = {
 150         map {
 151             $_->{name} // $_->{ring0_addr} => $_
 152         } @{$nodelist->{node}}
 153     };
 154     $totem->{interface} = {
 155         map {
 156             $_->{linknumber} // $_->{ringnumber} => $_
 157         } @{$totem->{interface}}
 158     };
 159
 160     $conf->{digest} = $digest;
 161
 162     return $conf;
 163 }
 164
 165 sub write_conf {
 166     my ($filename, $conf) = @_;
 167
 168     my $c = clone($conf->{main}) // die "no main section";
 169
 170     # retransform back for easier dumping
 171     my $hash_to_array = sub {
 172         my ($hash) = @_;
 173         return [ $hash->@{sort keys %$hash} ];
 174     };
 175
 176     $c->{nodelist}->{node} = &$hash_to_array($c->{nodelist}->{node});
 177     $c->{totem}->{interface} = &$hash_to_array($c->{totem}->{interface});
 178
 179     my $dump_section_weak;
 180     $dump_section_weak = sub {
 181         my ($section, $prefix) = @_;
 182
 183         my $raw = '';
 184
 185         foreach my $k (sort keys %$section) {
 186             my $v = $section->{$k};
 187             if (ref($v) eq 'HASH') {
 188                 $raw .= $prefix . "$k {\n";
 189                 $raw .= $dump_section_weak->($v, "$prefix  ");
 190                 $raw .=  $prefix . "}\n";
 191                 $raw .= "\n" if !$prefix; # add extra newline at 1st level only
 192             } elsif (ref($v) eq 'ARRAY') {
 193                 foreach my $child (@$v) {
 194                     $raw .= $prefix . "$k {\n";
 195                     $raw .= $dump_section_weak->($child, "$prefix  ");
 196                     $raw .=  $prefix . "}\n";
 197                 }
 198             } elsif (!ref($v)) {
 199                 die "got undefined value for key '$k'!\n" if !defined($v);
 200                 $raw .= $prefix . "$k: $v\n";
 201             } else {
 202                 die "unexpected reference in config hash: $k => ". ref($v) ."\n";
 203             }
 204         }
 205
 206         return $raw;
 207     };
 208     my $dump_section = $dump_section_weak;
 209     weaken($dump_section_weak);
 210
 211     my $raw = $dump_section->($c, '');
 212
 213     return $raw;
 214 }
 215
 216 # read only - use atomic_write_conf method to write
 217 PVE::Cluster::cfs_register_file('corosync.conf', \&parse_conf);
 218 # this is read/write
 219 PVE::Cluster::cfs_register_file('corosync.conf.new', \&parse_conf,
 220                                 \&write_conf);
 221
 222 sub check_conf_exists {
 223     my ($noerr) = @_;
 224
 225     my $exists = -f "$basedir/corosync.conf";
 226
 227     die "Error: Corosync config '$basedir/corosync.conf' does not exist - is this node part of a cluster?\n"
 228         if !$noerr && !$exists;
 229
 230     return $exists;
 231 }
 232
 233 sub update_nodelist {
 234     my ($conf, $nodelist) = @_;
 235
 236     $conf->{main}->{nodelist}->{node} = $nodelist;
 237
 238     atomic_write_conf($conf);
 239 }
 240
 241 sub nodelist {
 242     my ($conf) = @_;
 243     return clone($conf->{main}->{nodelist}->{node});
 244 }
 245
 246 sub totem_config {
 247     my ($conf) = @_;
 248     return clone($conf->{main}->{totem});
 249 }
 250
 251 # caller must hold corosync.conf cfs lock if used in read-modify-write cycle
 252 sub atomic_write_conf {
 253     my ($conf, $no_increase_version) = @_;
 254
 255     if (!$no_increase_version) {
 256         die "invalid corosync config: unable to read config version\n"
 257             if !defined($conf->{main}->{totem}->{config_version});
 258         $conf->{main}->{totem}->{config_version}++;
 259     }
 260
 261     PVE::Cluster::cfs_write_file("corosync.conf.new", $conf);
 262
 263     rename("/etc/pve/corosync.conf.new", "/etc/pve/corosync.conf")
 264         || die "activating corosync.conf.new failed - $!\n";
 265 }
 266
 267 # for creating a new cluster with the current node
 268 # params are those from the API/CLI cluster create call
 269 sub create_conf {
 270     my ($nodename, $param) = @_;
 271
 272     my $clustername = $param->{clustername};
 273     my $nodeid = $param->{nodeid} || 1;
 274     my $votes = $param->{votes} || 1;
 275
 276     my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 277
 278     my $links = extract_corosync_link_args($param);
 279
 280     # if no links given, fall back to local IP as link0
 281     $links->{0} = { address => $local_ip_address }
 282         if !%$links;
 283
 284     my $conf = {
 285         totem => {
 286             version => 2, # protocol version
 287             secauth => 'on',
 288             cluster_name => $clustername,
 289             config_version => 0,
 290             ip_version => 'ipv4-6',
 291             link_mode => 'passive',
 292             interface => {},
 293         },
 294         nodelist => {
 295             node => {
 296                 $nodename => {
 297                     name => $nodename,
 298                     nodeid => $nodeid,
 299                     quorum_votes => $votes,
 300                 },
 301             },
 302         },
 303         quorum => {
 304             provider => 'corosync_votequorum',
 305         },
 306         logging => {
 307             to_syslog => 'yes',
 308             debug => 'off',
 309         },
 310     };
 311     my $totem = $conf->{totem};
 312     my $node = $conf->{nodelist}->{node}->{$nodename};
 313
 314     foreach my $lnum (keys %$links) {
 315         my $link = $links->{$lnum};
 316
 317         $totem->{interface}->{$lnum} = { linknumber => $lnum };
 318
 319         my $prio = $link->{priority};
 320         $totem->{interface}->{$lnum}->{knet_link_priority} = $prio if $prio;
 321
 322         $node->{"ring${lnum}_addr"} = $link->{address};
 323     }
 324
 325     return { main => $conf };
 326 }
 327
 328 # returns (\@errors, \@warnings) to the caller, does *not* 'die' or 'warn'
 329 # verification was successful if \@errors is empty
 330 sub verify_conf {
 331     my ($conf) = @_;
 332
 333     my @errors = ();
 334     my @warnings = ();
 335
 336     my $nodelist = nodelist($conf);
 337     if (!$nodelist) {
 338         push @errors, "no nodes found";
 339         return (\@errors, \@warnings);
 340     }
 341
 342     my $totem = $conf->{main}->{totem};
 343     if (!$totem) {
 344         push @errors, "no totem found";
 345         return (\@errors, \@warnings);
 346     }
 347
 348     if ((!defined($totem->{secauth}) || $totem->{secauth} ne 'on') &&
 349         (!defined($totem->{crypto_cipher}) || $totem->{crypto_cipher} eq 'none')) {
 350         push @warnings, "warning: authentication/encryption is not explicitly enabled"
 351             . " (secauth / crypto_cipher / crypto_hash)";
 352     }
 353
 354     my $interfaces = $totem->{interface};
 355
 356     my $verify_link_ip = sub {
 357         my ($key, $link, $node) = @_;
 358         my ($resolved_ip, undef) = resolve_hostname_like_corosync($link, $conf);
 359         if (!defined($resolved_ip)) {
 360             push @warnings, "warning: unable to resolve $key '$link' for node '$node'"
 361                 . " to an IP address according to Corosync's resolve strategy -"
 362                 . " cluster could fail on restart!";
 363         } elsif ($resolved_ip ne $link) {
 364             push @warnings, "warning: $key '$link' for node '$node' resolves to"
 365                 . " '$resolved_ip' - consider replacing it with the currently"
 366                 . " resolved IP address for stability";
 367         }
 368     };
 369
 370     # sort for output order stability
 371     my @node_names = sort keys %$nodelist;
 372
 373     my $node_links = {};
 374     foreach my $node (@node_names) {
 375         my $options = $nodelist->{$node};
 376         foreach my $opt (keys %$options) {
 377             my ($linktype, $linkid) = parse_link_entry($opt);
 378             next if !defined($linktype);
 379             $node_links->{$node}->{$linkid} = {
 380                 name => "${linktype}${linkid}_addr",
 381                 addr => $options->{$opt},
 382             };
 383         }
 384     }
 385
 386     if (%$interfaces) {
 387         # if interfaces are defined, *all* links must have a matching interface
 388         # definition, and vice versa
 389         for my $link (0..MAX_LINK_INDEX) {
 390             my $have_interface = defined($interfaces->{$link});
 391             foreach my $node (@node_names) {
 392                 my $linkdef = $node_links->{$node}->{$link};
 393                 if (defined($linkdef)) {
 394                     $verify_link_ip->($linkdef->{name}, $linkdef->{addr}, $node);
 395                     if (!$have_interface) {
 396                         push @errors, "node '$node' has '$linkdef->{name}', but"
 397                             . " there is no interface number $link configured";
 398                     }
 399                 } else {
 400                     if ($have_interface) {
 401                         push @errors, "node '$node' is missing address for"
 402                             . "interface number $link";
 403                     }
 404                 }
 405             }
 406         }
 407     } else {
 408         # without interfaces, only check that links are consistent among nodes
 409         for my $link (0..MAX_LINK_INDEX) {
 410             my $nodes_with_link = {};
 411             foreach my $node (@node_names) {
 412                 my $linkdef = $node_links->{$node}->{$link};
 413                 if (defined($linkdef)) {
 414                     $verify_link_ip->($linkdef->{name}, $linkdef->{addr}, $node);
 415                     $nodes_with_link->{$node} = 1;
 416                 }
 417             }
 418
 419             if (%$nodes_with_link) {
 420                 foreach my $node (@node_names) {
 421                     if (!defined($nodes_with_link->{$node})) {
 422                         push @errors, "node '$node' is missing link $link,"
 423                             . " which is configured on other nodes";
 424                     }
 425                 }
 426             }
 427         }
 428     }
 429
 430     return (\@errors, \@warnings);
 431 }
 432
 433 # returns ($linktype, $linkid) with $linktype being 'ring' for now, and possibly
 434 # 'link' with upcoming corosync versions
 435 sub parse_link_entry {
 436     my ($opt) = @_;
 437     return (undef, undef) if $opt !~ $link_addr_re;
 438     return ($1, $2);
 439 }
 440
 441 sub for_all_corosync_addresses {
 442     my ($corosync_conf, $ip_version, $func) = @_;
 443
 444     my $nodelist = nodelist($corosync_conf);
 445     return if !defined($nodelist);
 446
 447     # iterate sorted to make rules deterministic (for change detection)
 448     foreach my $node_name (sort keys %$nodelist) {
 449         my $node_config = $nodelist->{$node_name};
 450         foreach my $node_key (sort keys %$node_config) {
 451             if ($node_key =~ $link_addr_re) {
 452                 my $node_address = $node_config->{$node_key};
 453
 454                 my($ip, $version) = resolve_hostname_like_corosync($node_address, $corosync_conf);
 455                 next if !defined($ip);
 456                 next if defined($version) && defined($ip_version) && $version != $ip_version;
 457
 458                 $func->($node_name, $ip, $version, $node_key);
 459             }
 460         }
 461     }
 462 }
 463
 464 # NOTE: Corosync actually only resolves on startup or config change, but we
 465 # currently do not have an easy way to synchronize our behaviour to that.
 466 sub resolve_hostname_like_corosync {
 467     my ($hostname, $corosync_conf) = @_;
 468
 469     my $corosync_strategy = $corosync_conf->{main}->{totem}->{ip_version};
 470     $corosync_strategy = lc ($corosync_strategy // "ipv6-4");
 471
 472     my $match_ip_and_version = sub {
 473         my ($addr) = @_;
 474
 475         return undef if !defined($addr);
 476
 477         if ($addr =~ m/^$IPV4RE$/) {
 478             return ($addr, 4);
 479         } elsif ($addr =~ m/^$IPV6RE$/) {
 480             return ($addr, 6);
 481         }
 482
 483         return undef;
 484     };
 485
 486     my ($resolved_ip, $ip_version) = $match_ip_and_version->($hostname);
 487
 488     return ($resolved_ip, $ip_version) if defined($resolved_ip);
 489
 490     my $resolved_ip4;
 491     my $resolved_ip6;
 492
 493     my @resolved_raw;
 494     eval { @resolved_raw = PVE::Tools::getaddrinfo_all($hostname); };
 495
 496     return undef if ($@ || !@resolved_raw);
 497
 498     foreach my $socket_info (@resolved_raw) {
 499         next if !$socket_info->{addr};
 500
 501         my ($family, undef, $host) = PVE::Tools::unpack_sockaddr_in46($socket_info->{addr});
 502
 503         if ($family == AF_INET && !defined($resolved_ip4)) {
 504             $resolved_ip4 = inet_ntop(AF_INET, $host);
 505         } elsif ($family == AF_INET6 && !defined($resolved_ip6)) {
 506             $resolved_ip6 = inet_ntop(AF_INET6, $host);
 507         }
 508
 509         last if defined($resolved_ip4) && defined($resolved_ip6);
 510     }
 511
 512     # corosync_strategy specifies the order in which IP addresses are resolved
 513     # by corosync. We need to match that order, to ensure we create firewall
 514     # rules for the correct address family.
 515     if ($corosync_strategy eq "ipv4") {
 516         $resolved_ip = $resolved_ip4;
 517     } elsif ($corosync_strategy eq "ipv6") {
 518         $resolved_ip = $resolved_ip6;
 519     } elsif ($corosync_strategy eq "ipv6-4") {
 520         $resolved_ip = $resolved_ip6 // $resolved_ip4;
 521     } elsif ($corosync_strategy eq "ipv4-6") {
 522         $resolved_ip = $resolved_ip4 // $resolved_ip6;
 523     }
 524
 525     return $match_ip_and_version->($resolved_ip);
 526 }
 527
 528 1;