PMG/Cluster.pm

   1 package PMG::Cluster;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Socket;
   7 use PVE::Tools;
   8 use PVE::INotify;
   9
  10 # this is also used to get the IP of the local node
  11 sub remote_node_ip {
  12     my ($nodename, $noerr) = @_;
  13
  14     # todo: implement cluster node list
  15
  16     # fallback: try to get IP by other means
  17     my ($family, $packed_ip);
  18
  19     eval {
  20         my @res = PVE::Tools::getaddrinfo_all($nodename);
  21         $family = $res[0]->{family};
  22         $packed_ip = (PVE::Tools::unpack_sockaddr_in46($res[0]->{addr}))[2];
  23     };
  24
  25     if ($@) {
  26         die "hostname lookup failed:\n$@" if !$noerr;
  27         return undef;
  28     }
  29
  30     my $ip = Socket::inet_ntop($family, $packed_ip);
  31     if ($ip =~ m/^127\.|^::1$/) {
  32         die "hostname lookup failed - got local IP address ($nodename = $ip)\n" if !$noerr;
  33         return undef;
  34     }
  35
  36     return wantarray ? ($ip, $family) : $ip;
  37 }
  38
  39 # X509 Certificate cache helper
  40
  41 my $cert_cache_nodes = {};
  42 my $cert_cache_timestamp = time();
  43 my $cert_cache_fingerprints = {};
  44
  45 sub update_cert_cache {
  46     my ($update_node, $clear) = @_;
  47
  48     syslog('info', "Clearing outdated entries from certificate cache")
  49         if $clear;
  50
  51     $cert_cache_timestamp = time() if !defined($update_node);
  52
  53     my $node_list = defined($update_node) ?
  54         [ $update_node ] : [ keys %$cert_cache_nodes ];
  55
  56     my $clear_node = sub {
  57         my ($node) = @_;
  58         if (my $old_fp = $cert_cache_nodes->{$node}) {
  59             # distrust old fingerprint
  60             delete $cert_cache_fingerprints->{$old_fp};
  61             # ensure reload on next proxied request
  62             delete $cert_cache_nodes->{$node};
  63         }
  64     };
  65
  66     my $nodename = PVE::INotify::nodename();
  67
  68     foreach my $node (@$node_list) {
  69
  70         if ($node ne $nodename) {
  71             &$clear_node($node) if $clear;
  72             next;
  73         }
  74
  75         my $cert_path = "/etc/proxmox/pmg-api.pem";
  76
  77         my $cert;
  78         eval {
  79             my $bio = Net::SSLeay::BIO_new_file($cert_path, 'r');
  80             $cert = Net::SSLeay::PEM_read_bio_X509($bio);
  81             Net::SSLeay::BIO_free($bio);
  82         };
  83         my $err = $@;
  84         if ($err || !defined($cert)) {
  85             &$clear_node($node) if $clear;
  86             next;
  87         }
  88
  89         my $fp;
  90         eval {
  91             $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
  92         };
  93         $err = $@;
  94         if ($err || !defined($fp) || $fp eq '') {
  95             &$clear_node($node) if $clear;
  96             next;
  97         }
  98
  99         my $old_fp = $cert_cache_nodes->{$node};
 100         $cert_cache_fingerprints->{$fp} = 1;
 101         $cert_cache_nodes->{$node} = $fp;
 102
 103         if (defined($old_fp) && $fp ne $old_fp) {
 104             delete $cert_cache_fingerprints->{$old_fp};
 105         }
 106     }
 107 }
 108
 109 # load and cache cert fingerprint once
 110 sub initialize_cert_cache {
 111     my ($node) = @_;
 112
 113     update_cert_cache($node)
 114         if defined($node) && !defined($cert_cache_nodes->{$node});
 115 }
 116
 117 sub check_cert_fingerprint {
 118     my ($cert) = @_;
 119
 120     # clear cache every 30 minutes at least
 121     update_cert_cache(undef, 1) if time() - $cert_cache_timestamp >= 60*30;
 122
 123     # get fingerprint of server certificate
 124     my $fp;
 125     eval {
 126         $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
 127     };
 128     return 0 if $@ || !defined($fp) || $fp eq ''; # error
 129
 130     my $check = sub {
 131         for my $expected (keys %$cert_cache_fingerprints) {
 132             return 1 if $fp eq $expected;
 133         }
 134         return 0;
 135     };
 136
 137     return 1 if &$check();
 138
 139     # clear cache and retry at most once every minute
 140     if (time() - $cert_cache_timestamp >= 60) {
 141         syslog ('info', "Could not verify remote node certificate '$fp' with list of pinned certificates, refreshing cache");
 142         update_cert_cache();
 143         return &$check();
 144     }
 145
 146     return 0;
 147 }
 148
 149 sub read_cluster_conf {
 150     my ($filename, $fh) = @_;
 151
 152     my $localname = PVE::INotify::nodename();
 153     my $localip = remote_node_ip($localname);
 154
 155     my $level = 0;
 156     my $maxcid = 0;
 157
 158     my $cinfo;
 159
 160     $cinfo->{nodes} = [];
 161     $cinfo->{remnodes} = [];
 162
 163     $cinfo->{local} = {
 164         role => '-',
 165         cid => 0,
 166         ip => $localip,
 167         name => $localname,
 168         configport => 83,
 169         dbport => 5432,
 170     };
 171
 172     if (defined($fh)) {
 173
 174         $cinfo->{exists} = 1; # cluster configuratin file exists and is readable
 175
 176         while (defined(my $line = <$fh>)) {
 177             chomp $line;
 178
 179             next if $line =~ m/^\s*$/; # skip empty lines
 180
 181             if ($line =~ m/^maxcid\s+(\d+)\s*$/i) {
 182                 $maxcid = $1 > $maxcid ? $1 : $maxcid;
 183                 next;
 184             }
 185
 186             if ($line =~ m/^(master|node)\s+(\d+)\s+\{\s*$/i) {
 187                 $level++;
 188                 my ($t, $cid) = (lc($1), $2);
 189
 190                 $maxcid = $cid > $maxcid ? $cid : $maxcid;
 191
 192                 my $res = {
 193                     role => $t eq 'master' ? 'M' : 'N',
 194                     cid => $cid
 195                 };
 196
 197                 while (defined($line = <$fh>)) {
 198                     chomp $line;
 199                     next if $line =~ m/^\s*$/; # skip empty lines
 200                     if ($line =~ m/^\}\s*$/) {
 201                         $level--;
 202                         last;
 203                     }
 204
 205                     if ($line =~ m/^\s*(\S+)\s*:\s*(\S+)\s*$/) {
 206                         my ($n, $v) = (lc $1, $2);
 207
 208                         # fixme: do syntax checks
 209                         if ($n eq 'ip') {
 210                             $res->{$n} = $v;
 211                         } elsif ($n eq 'name') {
 212                             $res->{$n} = $v;
 213                         } elsif ($n eq 'hostrsapubkey') {
 214                             $res->{$n} = $v;
 215                         } elsif ($n eq 'rootrsapubkey') {
 216                             $res->{$n} = $v;
 217                         } else {
 218                             die "syntax error in configuration file\n";
 219                         }
 220                     } else {
 221                         die "syntax error in configuration file\n";
 222                     }
 223                 }
 224
 225                 die "missing ip address for node '$cid'\n" if !$res->{ip};
 226                 die "missing name for node '$cid'\n" if !$res->{name};
 227                 #die "missing host RSA key for node '$cid'\n" if !$res->{hostrsapubkey};
 228                 #die "missing user RSA key for node '$cid'\n" if !$res->{rootrsapubkey};
 229
 230                 push @{$cinfo->{nodes}}, $res;
 231
 232                 if ($res->{role} eq 'M') {
 233                     $cinfo->{master} = $res;
 234                 }
 235
 236                 if ($res->{ip} eq $localip) {
 237                     $cinfo->{local} = $res;
 238                 }
 239             } else {
 240                 die "syntax error in configuration file\n";
 241             }
 242         }
 243     }
 244
 245     die "syntax error in configuration file\n" if $level;
 246
 247     $cinfo->{maxcid} = $maxcid;
 248
 249     my @cidlist = ();
 250     foreach my $ni (@{$cinfo->{nodes}}) {
 251         next if $cinfo->{local}->{cid} == $ni->{cid}; # skip local CID
 252         push @cidlist, $ni->{cid};
 253     }
 254
 255     my $ind = 0;
 256     my $portid = {};
 257     foreach my $cid (sort @cidlist) {
 258         $portid->{$cid} = $ind;
 259         $ind++;
 260     }
 261
 262     foreach my $ni (@{$cinfo->{nodes}}) {
 263         # fixme: do we still need those ports?
 264         $ni->{configport} = $ni->{cid} == $cinfo->{local}->{cid} ? 83 : 50000 + $portid->{$ni->{cid}};
 265         $ni->{dbport} = $ni->{cid} == $cinfo->{local}->{cid} ? 5432 : 50100 + $portid->{$ni->{cid}};
 266     }
 267
 268     foreach my $ni (@{$cinfo->{nodes}}) {
 269         next if $ni->{cid} == $cinfo->{local}->{cid};
 270         push @{$cinfo->{remnodes}}, $ni->{cid};
 271     }
 272
 273     return $cinfo;
 274 }
 275
 276 sub write_cluster_conf {
 277     my ($filename, $fh, $cinfo) = @_;
 278
 279     my $raw = "maxcid $cinfo->{maxcid}\n\n";
 280
 281     foreach my $ni (@{$cinfo->{nodes}}) {
 282
 283         if ($ni->{role} eq 'M') {
 284             $raw .= "master $ni->{cid} {\n";
 285             $raw .= " IP: $ni->{ip}\n";
 286             $raw .= " NAME: $ni->{name}\n";
 287             $raw .= " HOSTRSAPUBKEY: $ni->{hostrsapubkey}\n";
 288             $raw .= " ROOTRSAPUBKEY: $ni->{rootrsapubkey}\n";
 289             $raw .= "}\n\n";
 290         } elsif ($ni->{role} eq 'N') {
 291             $raw .= "node $ni->{cid} {\n";
 292             $raw .= " IP: $ni->{ip}\n";
 293             $raw .= " NAME: $ni->{name}\n";
 294             $raw .= " HOSTRSAPUBKEY: $ni->{hostrsapubkey}\n";
 295             $raw .= " ROOTRSAPUBKEY: $ni->{rootrsapubkey}\n";
 296             $raw .= "}\n\n";
 297         }
 298     }
 299
 300     PVE::Tools::safe_print($filename, $fh, $raw);
 301 }
 302
 303 PVE::INotify::register_file('cluster.conf', "/etc/proxmox/cluster.conf",
 304                             \&read_cluster_conf,
 305                             \&write_cluster_conf,
 306                             undef,
 307                             always_call_parser => 1);
 308
 309 1;