data/PVE/Corosync.pm

   1 package PVE::Corosync;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Digest::SHA;
   7 use Clone 'clone';
   8 use Socket qw(AF_INET AF_INET6 inet_ntop);
   9 use Net::IP qw(ip_is_ipv6);
  10
  11 use PVE::Cluster;
  12 use PVE::JSONSchema;
  13 use PVE::Tools;
  14 use PVE::Tools qw($IPV4RE $IPV6RE);
  15
  16 my $basedir = "/etc/pve";
  17
  18 my $conf_array_sections = {
  19     node => 1,
  20     interface => 1,
  21 };
  22
  23 my $corosync_link_format = {
  24     address => {
  25         default_key => 1,
  26         type => 'string', format => 'address',
  27         format_description => 'IP',
  28         description => "Hostname (or IP) of this corosync link address.",
  29     },
  30     priority => {
  31         optional => 1,
  32         type => 'integer',
  33         minimum => 0,
  34         maximum => 255,
  35         default => 0,
  36         description => "The priority for the link when knet is used in 'passive' mode. Lower value means higher priority.",
  37     },
  38 };
  39 my $corosync_link_desc = {
  40     type => 'string', format => $corosync_link_format,
  41     description => "Address and priority information of a single corosync link.",
  42     optional => 1,
  43 };
  44 PVE::JSONSchema::register_standard_option("corosync-link", $corosync_link_desc);
  45
  46 sub parse_corosync_link {
  47     my ($value) = @_;
  48
  49     return undef if !defined($value);
  50
  51     return PVE::JSONSchema::parse_property_string($corosync_link_format, $value);
  52 }
  53
  54 # a very simply parser ...
  55 sub parse_conf {
  56     my ($filename, $raw) = @_;
  57
  58     return {} if !$raw;
  59
  60     my $digest = Digest::SHA::sha1_hex(defined($raw) ? $raw : '');
  61
  62     $raw =~ s/#.*$//mg;
  63     $raw =~ s/\r?\n/ /g;
  64     $raw =~ s/\s+/ /g;
  65     $raw =~ s/^\s+//;
  66     $raw =~ s/\s*$//;
  67
  68     my @tokens = split(/\s/, $raw);
  69
  70     my $conf = { 'main' => {} };
  71
  72     my $stack = [];
  73     my $section = $conf->{main};
  74
  75     while (defined(my $token = shift @tokens)) {
  76         my $nexttok = $tokens[0];
  77
  78         if ($nexttok && ($nexttok eq '{')) {
  79             shift @tokens; # skip '{'
  80             my $new_section = {};
  81             if ($conf_array_sections->{$token}) {
  82                 $section->{$token} = [] if !defined($section->{$token});
  83                 push @{$section->{$token}}, $new_section;
  84             } elsif (!defined($section->{$token})) {
  85                 $section->{$token} = $new_section;
  86             } else {
  87                 die "section '$token' already exists and not marked as array!\n";
  88             }
  89             push @$stack, $section;
  90             $section = $new_section;
  91             next;
  92         }
  93
  94         if ($token eq '}') {
  95             $section = pop @$stack;
  96             die "parse error - uncexpected '}'\n" if !$section;
  97             next;
  98         }
  99
 100         my $key = $token;
 101         die "missing ':' after key '$key'\n" if ! ($key =~ s/:$//);
 102
 103         die "parse error - no value for '$key'\n" if !defined($nexttok);
 104         my $value = shift @tokens;
 105
 106         $section->{$key} = $value;
 107     }
 108
 109     # make working with the config way easier
 110     my ($totem, $nodelist) = $conf->{main}->@{"totem", "nodelist"};
 111
 112     $nodelist->{node} = {
 113         map {
 114             $_->{name} // $_->{ring0_addr} => $_
 115         } @{$nodelist->{node}}
 116     };
 117     $totem->{interface} = {
 118         map {
 119             $_->{linknumber} // $_->{ringnumber} => $_
 120         } @{$totem->{interface}}
 121     };
 122
 123     $conf->{digest} = $digest;
 124
 125     return $conf;
 126 }
 127
 128 my $dump_section;
 129 $dump_section = sub {
 130     my ($section, $prefix) = @_;
 131
 132     my $raw = '';
 133
 134     foreach my $k (sort keys %$section) {
 135         my $v = $section->{$k};
 136         if (ref($v) eq 'HASH') {
 137             $raw .= $prefix . "$k {\n";
 138             $raw .= &$dump_section($v, "$prefix  ");
 139             $raw .=  $prefix . "}\n";
 140             $raw .= "\n" if !$prefix; # add extra newline at 1st level only
 141         } elsif (ref($v) eq 'ARRAY') {
 142             foreach my $child (@$v) {
 143                 $raw .= $prefix . "$k {\n";
 144                 $raw .= &$dump_section($child, "$prefix  ");
 145                 $raw .=  $prefix . "}\n";
 146             }
 147         } elsif (!ref($v)) {
 148             die "got undefined value for key '$k'!\n" if !defined($v);
 149             $raw .= $prefix . "$k: $v\n";
 150         } else {
 151             die "unexpected reference in config hash: $k => ". ref($v) ."\n";
 152         }
 153     }
 154
 155     return $raw;
 156 };
 157
 158 sub write_conf {
 159     my ($filename, $conf) = @_;
 160
 161     my $c = clone($conf->{main}) // die "no main section";
 162
 163     # retransform back for easier dumping
 164     my $hash_to_array = sub {
 165         my ($hash) = @_;
 166         return [ $hash->@{sort keys %$hash} ];
 167     };
 168
 169     $c->{nodelist}->{node} = &$hash_to_array($c->{nodelist}->{node});
 170     $c->{totem}->{interface} = &$hash_to_array($c->{totem}->{interface});
 171
 172     my $raw = &$dump_section($c, '');
 173
 174     return $raw;
 175 }
 176
 177 # read only - use atomic_write_conf method to write
 178 PVE::Cluster::cfs_register_file('corosync.conf', \&parse_conf);
 179 # this is read/write
 180 PVE::Cluster::cfs_register_file('corosync.conf.new', \&parse_conf,
 181                                 \&write_conf);
 182
 183 sub check_conf_exists {
 184     my ($silent) = @_;
 185
 186     $silent = $silent // 0;
 187
 188     my $exists = -f "$basedir/corosync.conf";
 189
 190     warn "Corosync config '$basedir/corosync.conf' does not exist - is this node part of a cluster?\n"
 191         if !$silent && !$exists;
 192
 193     return $exists;
 194 }
 195
 196 sub update_nodelist {
 197     my ($conf, $nodelist) = @_;
 198
 199     $conf->{main}->{nodelist}->{node} = $nodelist;
 200
 201     atomic_write_conf($conf);
 202 }
 203
 204 sub nodelist {
 205     my ($conf) = @_;
 206     return clone($conf->{main}->{nodelist}->{node});
 207 }
 208
 209 sub totem_config {
 210     my ($conf) = @_;
 211     return clone($conf->{main}->{totem});
 212 }
 213
 214 # caller must hold corosync.conf cfs lock if used in read-modify-write cycle
 215 sub atomic_write_conf {
 216     my ($conf, $no_increase_version) = @_;
 217
 218     if (!$no_increase_version) {
 219         die "invalid corosync config: unable to read config version\n"
 220             if !defined($conf->{main}->{totem}->{config_version});
 221         $conf->{main}->{totem}->{config_version}++;
 222     }
 223
 224     PVE::Cluster::cfs_write_file("corosync.conf.new", $conf);
 225
 226     rename("/etc/pve/corosync.conf.new", "/etc/pve/corosync.conf")
 227         || die "activating corosync.conf.new failed - $!\n";
 228 }
 229
 230 # for creating a new cluster with the current node
 231 # params are those from the API/CLI cluster create call
 232 sub create_conf {
 233     my ($nodename, %param) = @_;
 234
 235     my $clustername = $param{clustername};
 236     my $nodeid = $param{nodeid} || 1;
 237     my $votes = $param{votes} || 1;
 238
 239     my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
 240
 241     my $link0 = PVE::Cluster::parse_corosync_link($param{link0});
 242     $link0->{address} //= $local_ip_address;
 243
 244     my $conf = {
 245         totem => {
 246             version => 2, # protocol version
 247             secauth => 'on',
 248             cluster_name => $clustername,
 249             config_version => 0,
 250             ip_version => 'ipv4-6',
 251             interface => {
 252                 0 => {
 253                     linknumber => 0,
 254                 },
 255             },
 256         },
 257         nodelist => {
 258             node => {
 259                 $nodename => {
 260                     name => $nodename,
 261                     nodeid => $nodeid,
 262                     quorum_votes => $votes,
 263                     ring0_addr => $link0->{address},
 264                 },
 265             },
 266         },
 267         quorum => {
 268             provider => 'corosync_votequorum',
 269         },
 270         logging => {
 271             to_syslog => 'yes',
 272             debug => 'off',
 273         },
 274     };
 275     my $totem = $conf->{totem};
 276
 277     $totem->{interface}->{0}->{knet_link_priority} = $link0->{priority}
 278         if defined($link0->{priority});
 279
 280     my $link1 = PVE::Cluster::parse_corosync_link($param{link1});
 281     if ($link1->{address}) {
 282         $conf->{totem}->{interface}->{1} = {
 283             linknumber => 1,
 284         };
 285         $totem->{link_mode} = 'passive';
 286         $totem->{interface}->{1}->{knet_link_priority} = $link1->{priority}
 287             if defined($link1->{priority});
 288         $conf->{nodelist}->{node}->{$nodename}->{ring1_addr} = $link1->{address};
 289     }
 290
 291     return { main => $conf };
 292 }
 293
 294 sub for_all_corosync_addresses {
 295     my ($corosync_conf, $ip_version, $func) = @_;
 296
 297     my $nodelist = nodelist($corosync_conf);
 298     return if !defined($nodelist);
 299
 300     # iterate sorted to make rules deterministic (for change detection)
 301     foreach my $node_name (sort keys %$nodelist) {
 302         my $node_config = $nodelist->{$node_name};
 303         foreach my $node_key (sort keys %$node_config) {
 304             if ($node_key =~ /^(ring|link)\d+_addr$/) {
 305                 my $node_address = $node_config->{$node_key};
 306
 307                 my($ip, $version) = resolve_hostname_like_corosync($node_address, $corosync_conf);
 308                 next if !defined($ip);
 309                 next if defined($version) && defined($ip_version) && $version != $ip_version;
 310
 311                 $func->($node_name, $ip, $version, $node_key);
 312             }
 313         }
 314     }
 315 }
 316
 317 # NOTE: Corosync actually only resolves on startup or config change, but we
 318 # currently do not have an easy way to synchronize our behaviour to that.
 319 sub resolve_hostname_like_corosync {
 320     my ($hostname, $corosync_conf) = @_;
 321
 322     my $corosync_strategy = $corosync_conf->{main}->{totem}->{ip_version};
 323     $corosync_strategy = lc ($corosync_strategy // "ipv6-4");
 324
 325     my $match_ip_and_version = sub {
 326         my ($addr) = @_;
 327
 328         return undef if !defined($addr);
 329
 330         if ($addr =~ m/^$IPV4RE$/) {
 331             return ($addr, 4);
 332         } elsif ($addr =~ m/^$IPV6RE$/) {
 333             return ($addr, 6);
 334         }
 335
 336         return undef;
 337     };
 338
 339     my ($resolved_ip, $ip_version) = $match_ip_and_version->($hostname);
 340
 341     return ($resolved_ip, $ip_version) if defined($resolved_ip);
 342
 343     my $resolved_ip4;
 344     my $resolved_ip6;
 345
 346     my @resolved_raw;
 347     eval { @resolved_raw = PVE::Tools::getaddrinfo_all($hostname); };
 348
 349     return undef if ($@ || !@resolved_raw);
 350
 351     foreach my $socket_info (@resolved_raw) {
 352         next if !$socket_info->{addr};
 353
 354         my ($family, undef, $host) = PVE::Tools::unpack_sockaddr_in46($socket_info->{addr});
 355
 356         if ($family == AF_INET && !defined($resolved_ip4)) {
 357             $resolved_ip4 = inet_ntop(AF_INET, $host);
 358         } elsif ($family == AF_INET6 && !defined($resolved_ip6)) {
 359             $resolved_ip6 = inet_ntop(AF_INET6, $host);
 360         }
 361
 362         last if defined($resolved_ip4) && defined($resolved_ip6);
 363     }
 364
 365     # corosync_strategy specifies the order in which IP addresses are resolved
 366     # by corosync. We need to match that order, to ensure we create firewall
 367     # rules for the correct address family.
 368     if ($corosync_strategy eq "ipv4") {
 369         $resolved_ip = $resolved_ip4;
 370     } elsif ($corosync_strategy eq "ipv6") {
 371         $resolved_ip = $resolved_ip6;
 372     } elsif ($corosync_strategy eq "ipv6-4") {
 373         $resolved_ip = $resolved_ip6 // $resolved_ip4;
 374     } elsif ($corosync_strategy eq "ipv4-6") {
 375         $resolved_ip = $resolved_ip4 // $resolved_ip6;
 376     }
 377
 378     return $match_ip_and_version->($resolved_ip);
 379 }
 380
 381 1;