PMG/Cluster.pm

   1 package PMG::Cluster;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Socket;
   7 use PVE::Tools;
   8 use PVE::INotify;
   9
  10 # this is also used to get the IP of the local node
  11 sub lookup_node_ip {
  12     my ($nodename, $noerr) = @_;
  13
  14     my ($family, $packed_ip);
  15
  16     eval {
  17         my @res = PVE::Tools::getaddrinfo_all($nodename);
  18         $family = $res[0]->{family};
  19         $packed_ip = (PVE::Tools::unpack_sockaddr_in46($res[0]->{addr}))[2];
  20     };
  21
  22     if ($@) {
  23         die "hostname lookup failed:\n$@" if !$noerr;
  24         return undef;
  25     }
  26
  27     my $ip = Socket::inet_ntop($family, $packed_ip);
  28     if ($ip =~ m/^127\.|^::1$/) {
  29         die "hostname lookup failed - got local IP address ($nodename = $ip)\n" if !$noerr;
  30         return undef;
  31     }
  32
  33     return wantarray ? ($ip, $family) : $ip;
  34 }
  35
  36 sub remote_node_ip {
  37     my ($nodename, $noerr) = @_;
  38
  39     my $cinfo = PVE::INotify::read_file("cluster.conf");
  40
  41     foreach my $entry (@{$cinfo->{nodes}}) {
  42         if ($entry->{name} eq $nodename) {
  43             my $ip = $entry->{ip};
  44             return $ip if !wantarray;
  45             my $family = PVE::Tools::get_host_address_family($ip);
  46             return ($ip, $family);
  47         }
  48     }
  49
  50     # fallback: try to get IP by other means
  51     return lookup_node_ip($nodename, $noerr);
  52 }
  53
  54 sub get_master_node {
  55     my ($cinfo) = @_;
  56
  57     $cinfo //= PVE::INotify::read_file("cluster.conf");
  58
  59     return $cinfo->{master}->{name} if defined($cinfo->{master});
  60
  61     return 'localhost';
  62 }
  63
  64 # X509 Certificate cache helper
  65
  66 my $cert_cache_nodes = {};
  67 my $cert_cache_timestamp = time();
  68 my $cert_cache_fingerprints = {};
  69
  70 sub update_cert_cache {
  71     my ($update_node, $clear) = @_;
  72
  73     syslog('info', "Clearing outdated entries from certificate cache")
  74         if $clear;
  75
  76     $cert_cache_timestamp = time() if !defined($update_node);
  77
  78     my $node_list = defined($update_node) ?
  79         [ $update_node ] : [ keys %$cert_cache_nodes ];
  80
  81     my $clear_node = sub {
  82         my ($node) = @_;
  83         if (my $old_fp = $cert_cache_nodes->{$node}) {
  84             # distrust old fingerprint
  85             delete $cert_cache_fingerprints->{$old_fp};
  86             # ensure reload on next proxied request
  87             delete $cert_cache_nodes->{$node};
  88         }
  89     };
  90
  91     my $nodename = PVE::INotify::nodename();
  92
  93     foreach my $node (@$node_list) {
  94
  95         if ($node ne $nodename) {
  96             &$clear_node($node) if $clear;
  97             next;
  98         }
  99
 100         my $cert_path = "/etc/proxmox/pmg-api.pem";
 101
 102         my $cert;
 103         eval {
 104             my $bio = Net::SSLeay::BIO_new_file($cert_path, 'r');
 105             $cert = Net::SSLeay::PEM_read_bio_X509($bio);
 106             Net::SSLeay::BIO_free($bio);
 107         };
 108         my $err = $@;
 109         if ($err || !defined($cert)) {
 110             &$clear_node($node) if $clear;
 111             next;
 112         }
 113
 114         my $fp;
 115         eval {
 116             $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
 117         };
 118         $err = $@;
 119         if ($err || !defined($fp) || $fp eq '') {
 120             &$clear_node($node) if $clear;
 121             next;
 122         }
 123
 124         my $old_fp = $cert_cache_nodes->{$node};
 125         $cert_cache_fingerprints->{$fp} = 1;
 126         $cert_cache_nodes->{$node} = $fp;
 127
 128         if (defined($old_fp) && $fp ne $old_fp) {
 129             delete $cert_cache_fingerprints->{$old_fp};
 130         }
 131     }
 132 }
 133
 134 # load and cache cert fingerprint once
 135 sub initialize_cert_cache {
 136     my ($node) = @_;
 137
 138     update_cert_cache($node)
 139         if defined($node) && !defined($cert_cache_nodes->{$node});
 140 }
 141
 142 sub check_cert_fingerprint {
 143     my ($cert) = @_;
 144
 145     # clear cache every 30 minutes at least
 146     update_cert_cache(undef, 1) if time() - $cert_cache_timestamp >= 60*30;
 147
 148     # get fingerprint of server certificate
 149     my $fp;
 150     eval {
 151         $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
 152     };
 153     return 0 if $@ || !defined($fp) || $fp eq ''; # error
 154
 155     my $check = sub {
 156         for my $expected (keys %$cert_cache_fingerprints) {
 157             return 1 if $fp eq $expected;
 158         }
 159         return 0;
 160     };
 161
 162     return 1 if &$check();
 163
 164     # clear cache and retry at most once every minute
 165     if (time() - $cert_cache_timestamp >= 60) {
 166         syslog ('info', "Could not verify remote node certificate '$fp' with list of pinned certificates, refreshing cache");
 167         update_cert_cache();
 168         return &$check();
 169     }
 170
 171     return 0;
 172 }
 173
 174 sub read_cluster_conf {
 175     my ($filename, $fh) = @_;
 176
 177     my $localname = PVE::INotify::nodename();
 178     my $localip = lookup_node_ip($localname);
 179
 180     my $level = 0;
 181     my $maxcid = 0;
 182
 183     my $cinfo;
 184
 185     $cinfo->{nodes} = [];
 186     $cinfo->{remnodes} = [];
 187
 188     $cinfo->{local} = {
 189         role => '-',
 190         cid => 0,
 191         ip => $localip,
 192         name => $localname,
 193         configport => 83,
 194         dbport => 5432,
 195     };
 196
 197     # fixme: add test is local node is part of node list
 198     if (defined($fh)) {
 199
 200         $cinfo->{exists} = 1; # cluster configuratin file exists and is readable
 201
 202         while (defined(my $line = <$fh>)) {
 203             chomp $line;
 204
 205             next if $line =~ m/^\s*$/; # skip empty lines
 206
 207             if ($line =~ m/^maxcid\s+(\d+)\s*$/i) {
 208                 $maxcid = $1 > $maxcid ? $1 : $maxcid;
 209                 next;
 210             }
 211
 212             if ($line =~ m/^(master|node)\s+(\d+)\s+\{\s*$/i) {
 213                 $level++;
 214                 my ($t, $cid) = (lc($1), $2);
 215
 216                 $maxcid = $cid > $maxcid ? $cid : $maxcid;
 217
 218                 my $res = {
 219                     role => $t eq 'master' ? 'M' : 'N',
 220                     cid => $cid
 221                 };
 222
 223                 while (defined($line = <$fh>)) {
 224                     chomp $line;
 225                     next if $line =~ m/^\s*$/; # skip empty lines
 226                     if ($line =~ m/^\}\s*$/) {
 227                         $level--;
 228                         last;
 229                     }
 230
 231                     if ($line =~ m/^\s*(\S+)\s*:\s*(\S+)\s*$/) {
 232                         my ($n, $v) = (lc $1, $2);
 233
 234                         # fixme: do syntax checks
 235                         if ($n eq 'ip') {
 236                             $res->{$n} = $v;
 237                         } elsif ($n eq 'name') {
 238                             $res->{$n} = $v;
 239                         } elsif ($n eq 'hostrsapubkey') {
 240                             $res->{$n} = $v;
 241                         } elsif ($n eq 'rootrsapubkey') {
 242                             $res->{$n} = $v;
 243                         } else {
 244                             die "syntax error in configuration file\n";
 245                         }
 246                     } else {
 247                         die "syntax error in configuration file\n";
 248                     }
 249                 }
 250
 251                 die "missing ip address for node '$cid'\n" if !$res->{ip};
 252                 die "missing name for node '$cid'\n" if !$res->{name};
 253                 #die "missing host RSA key for node '$cid'\n" if !$res->{hostrsapubkey};
 254                 #die "missing user RSA key for node '$cid'\n" if !$res->{rootrsapubkey};
 255
 256                 push @{$cinfo->{nodes}}, $res;
 257
 258                 if ($res->{role} eq 'M') {
 259                     $cinfo->{master} = $res;
 260                 }
 261
 262                 if ($res->{ip} eq $localname) {
 263                     $cinfo->{local} = $res;
 264                 }
 265             } else {
 266                 die "syntax error in configuration file\n";
 267             }
 268         }
 269     }
 270
 271     die "syntax error in configuration file\n" if $level;
 272
 273     $cinfo->{maxcid} = $maxcid;
 274
 275     my @cidlist = ();
 276     foreach my $ni (@{$cinfo->{nodes}}) {
 277         next if $cinfo->{local}->{cid} == $ni->{cid}; # skip local CID
 278         push @cidlist, $ni->{cid};
 279     }
 280
 281     my $ind = 0;
 282     my $portid = {};
 283     foreach my $cid (sort @cidlist) {
 284         $portid->{$cid} = $ind;
 285         $ind++;
 286     }
 287
 288     foreach my $ni (@{$cinfo->{nodes}}) {
 289         # fixme: do we still need those ports?
 290         $ni->{configport} = $ni->{cid} == $cinfo->{local}->{cid} ? 83 : 50000 + $portid->{$ni->{cid}};
 291         $ni->{dbport} = $ni->{cid} == $cinfo->{local}->{cid} ? 5432 : 50100 + $portid->{$ni->{cid}};
 292     }
 293
 294     foreach my $ni (@{$cinfo->{nodes}}) {
 295         next if $ni->{cid} == $cinfo->{local}->{cid};
 296         push @{$cinfo->{remnodes}}, $ni->{cid};
 297     }
 298
 299     return $cinfo;
 300 }
 301
 302 sub write_cluster_conf {
 303     my ($filename, $fh, $cinfo) = @_;
 304
 305     my $raw = "maxcid $cinfo->{maxcid}\n\n";
 306
 307     foreach my $ni (@{$cinfo->{nodes}}) {
 308
 309         if ($ni->{role} eq 'M') {
 310             $raw .= "master $ni->{cid} {\n";
 311             $raw .= " IP: $ni->{ip}\n";
 312             $raw .= " NAME: $ni->{name}\n";
 313             $raw .= " HOSTRSAPUBKEY: $ni->{hostrsapubkey}\n";
 314             $raw .= " ROOTRSAPUBKEY: $ni->{rootrsapubkey}\n";
 315             $raw .= "}\n\n";
 316         } elsif ($ni->{role} eq 'N') {
 317             $raw .= "node $ni->{cid} {\n";
 318             $raw .= " IP: $ni->{ip}\n";
 319             $raw .= " NAME: $ni->{name}\n";
 320             $raw .= " HOSTRSAPUBKEY: $ni->{hostrsapubkey}\n";
 321             $raw .= " ROOTRSAPUBKEY: $ni->{rootrsapubkey}\n";
 322             $raw .= "}\n\n";
 323         }
 324     }
 325
 326     PVE::Tools::safe_print($filename, $fh, $raw);
 327 }
 328
 329 PVE::INotify::register_file('cluster.conf', "/etc/proxmox/cluster.conf",
 330                             \&read_cluster_conf,
 331                             \&write_cluster_conf,
 332                             undef,
 333                             always_call_parser => 1);
 334
 335 1;