PMG/Cluster.pm

   1 package PMG::Cluster;
   2
   3 use strict;
   4 use warnings;
   5 use Data::Dumper;
   6 use Socket;
   7 use File::Path;
   8
   9 use PVE::SafeSyslog;
  10 use PVE::Tools;
  11 use PVE::INotify;
  12
  13 use PMG::Utils;
  14 use PMG::Config;
  15 use PMG::ClusterConfig;
  16 use PMG::RuleDB;
  17 use PMG::RuleCache;
  18 use PVE::APIClient::LWP;
  19
  20 our $spooldir = "/var/spool/proxmox";
  21
  22 sub create_needed_dirs {
  23     my ($lcid, $cleanup) = @_;
  24
  25     # if requested, remove any stale date
  26     File::Path::remove_tree("$spooldir/cluster", "$spooldir/virus", "$spooldir/spam") if $cleanup;
  27
  28     mkdir "$spooldir/spam";
  29     mkdir "$spooldir/virus";
  30
  31     if ($lcid) {
  32         mkpath "$spooldir/cluster/$lcid/virus";
  33         mkpath "$spooldir/cluster/$lcid/spam";
  34     }
  35 }
  36
  37 sub remote_node_ip {
  38     my ($nodename, $noerr) = @_;
  39
  40     my $cinfo = PMG::ClusterConfig->new();
  41
  42     foreach my $entry (values %{$cinfo->{ids}}) {
  43         if ($entry->{name} eq $nodename) {
  44             my $ip = $entry->{ip};
  45             return $ip if !wantarray;
  46             my $family = PVE::Tools::get_host_address_family($ip);
  47             return ($ip, $family);
  48         }
  49     }
  50
  51     # fallback: try to get IP by other means
  52     return PMG::Utils::lookup_node_ip($nodename, $noerr);
  53 }
  54
  55 sub get_master_node {
  56     my ($cinfo) = @_;
  57
  58     $cinfo = PMG::ClusterConfig->new() if !$cinfo;
  59
  60     return $cinfo->{master}->{name} if defined($cinfo->{master});
  61
  62     return 'localhost';
  63 }
  64
  65 sub read_local_ssl_cert_fingerprint {
  66     my $cert_path = "/etc/pmg/pmg-api.pem";
  67
  68     my $cert;
  69     eval {
  70         my $bio = Net::SSLeay::BIO_new_file($cert_path, 'r');
  71         $cert = Net::SSLeay::PEM_read_bio_X509($bio);
  72         Net::SSLeay::BIO_free($bio);
  73     };
  74     if (my $err = $@) {
  75         die "unable to read certificate '$cert_path' - $err\n";
  76     }
  77
  78     if (!defined($cert)) {
  79         die "unable to read certificate '$cert_path' - got empty value\n";
  80     }
  81
  82     my $fp;
  83     eval {
  84         $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
  85     };
  86     if (my $err = $@) {
  87         die "unable to get fingerprint for '$cert_path' - $err\n";
  88     }
  89
  90     if (!defined($fp) || $fp eq '') {
  91         die "unable to get fingerprint for '$cert_path' - got empty value\n";
  92     }
  93
  94     return $fp;
  95 }
  96
  97 my $hostrsapubkey_fn = '/etc/ssh/ssh_host_rsa_key.pub';
  98 my $rootrsakey_fn = '/root/.ssh/id_rsa';
  99 my $rootrsapubkey_fn = '/root/.ssh/id_rsa.pub';
 100
 101 sub read_local_cluster_info {
 102
 103     my $res = {};
 104
 105     my $hostrsapubkey = PVE::Tools::file_read_firstline($hostrsapubkey_fn);
 106     $hostrsapubkey =~ s/^.*ssh-rsa\s+//i;
 107     $hostrsapubkey =~ s/\s+root\@\S+\s*$//i;
 108
 109     die "unable to parse ${hostrsapubkey_fn}\n"
 110         if $hostrsapubkey !~ m/^[A-Za-z0-9\.\/\+]{200,}$/;
 111
 112     my $nodename = PVE::INotify::nodename();
 113
 114     $res->{name} = $nodename;
 115
 116     $res->{ip} = PMG::Utils::lookup_node_ip($nodename);
 117
 118     $res->{hostrsapubkey} = $hostrsapubkey;
 119
 120     if (! -f $rootrsapubkey_fn) {
 121         unlink $rootrsakey_fn;
 122         my $cmd = ['ssh-keygen', '-t', 'rsa', '-N', '', '-b', '2048',
 123                    '-f', $rootrsakey_fn];
 124         PMG::Utils::run_silent_cmd($cmd);
 125     }
 126
 127     my $rootrsapubkey = PVE::Tools::file_read_firstline($rootrsapubkey_fn);
 128     $rootrsapubkey =~ s/^.*ssh-rsa\s+//i;
 129     $rootrsapubkey =~ s/\s+root\@\S+\s*$//i;
 130
 131     die "unable to parse ${rootrsapubkey_fn}\n"
 132         if $rootrsapubkey !~ m/^[A-Za-z0-9\.\/\+]{200,}$/;
 133
 134     $res->{rootrsapubkey} = $rootrsapubkey;
 135
 136     $res->{fingerprint} = read_local_ssl_cert_fingerprint();
 137
 138     return $res;
 139 }
 140
 141 # X509 Certificate cache helper
 142
 143 my $cert_cache_nodes = {};
 144 my $cert_cache_timestamp = time();
 145 my $cert_cache_fingerprints = {};
 146
 147 sub update_cert_cache {
 148
 149     $cert_cache_timestamp = time();
 150
 151     $cert_cache_fingerprints = {};
 152     $cert_cache_nodes = {};
 153
 154     my $cinfo = PMG::ClusterConfig->new();
 155
 156     foreach my $entry (values %{$cinfo->{ids}}) {
 157         my $node = $entry->{name};
 158         my $fp = $entry->{fingerprint};
 159         if ($node && $fp) {
 160             $cert_cache_fingerprints->{$fp} = 1;
 161             $cert_cache_nodes->{$node} = $fp;
 162         }
 163     }
 164 }
 165
 166 # load and cache cert fingerprint once
 167 sub initialize_cert_cache {
 168     my ($node) = @_;
 169
 170     update_cert_cache()
 171         if defined($node) && !defined($cert_cache_nodes->{$node});
 172 }
 173
 174 sub check_cert_fingerprint {
 175     my ($cert) = @_;
 176
 177     # clear cache every 30 minutes at least
 178     update_cert_cache() if time() - $cert_cache_timestamp >= 60*30;
 179
 180     # get fingerprint of server certificate
 181     my $fp;
 182     eval {
 183         $fp = Net::SSLeay::X509_get_fingerprint($cert, 'sha256');
 184     };
 185     return 0 if $@ || !defined($fp) || $fp eq ''; # error
 186
 187     my $check = sub {
 188         for my $expected (keys %$cert_cache_fingerprints) {
 189             return 1 if $fp eq $expected;
 190         }
 191         return 0;
 192     };
 193
 194     return 1 if $check->();
 195
 196     # clear cache and retry at most once every minute
 197     if (time() - $cert_cache_timestamp >= 60) {
 198         syslog ('info', "Could not verify remote node certificate '$fp' with list of pinned certificates, refreshing cache");
 199         update_cert_cache();
 200         return $check->();
 201     }
 202
 203     return 0;
 204 }
 205
 206 my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts2";
 207 my $rootsshauthkeys = "/root/.ssh/authorized_keys";
 208 my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
 209
 210 sub update_ssh_keys {
 211     my ($cinfo) = @_;
 212
 213     my $data = '';
 214     foreach my $node (values %{$cinfo->{ids}}) {
 215         $data .= "$node->{ip} ssh-rsa $node->{hostrsapubkey}\n";
 216         $data .= "$node->{name} ssh-rsa $node->{hostrsapubkey}\n";
 217     }
 218
 219     PVE::Tools::file_set_contents($sshglobalknownhosts, $data);
 220
 221     $data = '';
 222
 223     # always add ourself
 224     if (-f $ssh_rsa_id) {
 225         my $pub = PVE::Tools::file_get_contents($ssh_rsa_id);
 226         chomp($pub);
 227         $data .= "$pub\n";
 228     }
 229
 230     foreach my $node (values %{$cinfo->{ids}}) {
 231         $data .= "ssh-rsa $node->{rootrsapubkey} root\@$node->{name}\n";
 232     }
 233
 234     if (-f $rootsshauthkeys) {
 235         my $mykey = PVE::Tools::file_get_contents($rootsshauthkeys, 128*1024);
 236         chomp($mykey);
 237         $data .= "$mykey\n";
 238     }
 239
 240     my $newdata = "";
 241     my $vhash = {};
 242     my @lines = split(/\n/, $data);
 243     foreach my $line (@lines) {
 244         if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
 245             next if $vhash->{$3}++;
 246         }
 247         $newdata .= "$line\n";
 248     }
 249
 250     PVE::Tools::file_set_contents($rootsshauthkeys, $newdata, 0600);
 251 }
 252
 253 my $cfgdir = '/etc/pmg';
 254 my $syncdir = "$cfgdir/master";
 255
 256 my $cond_commit_synced_file = sub {
 257     my ($filename, $dstfn) = @_;
 258
 259     $dstfn = "$cfgdir/$filename" if !defined($dstfn);
 260     my $srcfn = "$syncdir/$filename";
 261
 262     if (! -f $srcfn) {
 263         unlink $dstfn;
 264         return;
 265     }
 266
 267     my $new = PVE::Tools::file_get_contents($srcfn, 1024*1024);
 268
 269     if (-f $dstfn) {
 270         my $old = PVE::Tools::file_get_contents($dstfn, 1024*1024);
 271         return 0 if $new eq $old;
 272     }
 273
 274     rename($srcfn, $dstfn) ||
 275         die "cond_rename_file '$filename' failed - $!\n";
 276
 277     print STDERR "updated $dstfn\n";
 278
 279     return 1;
 280 };
 281
 282 my $rsync_command = sub {
 283     my ($host_key_alias, @args) = @_;
 284
 285     my $ssh_cmd = '--rsh=ssh -l root -o BatchMode=yes';
 286     $ssh_cmd .=  " -o HostKeyAlias=${host_key_alias}" if $host_key_alias;
 287
 288     my $cmd = ['rsync', $ssh_cmd,  '-q', @args];
 289
 290     return $cmd;
 291 };
 292
 293 sub sync_quarantine_files {
 294     my ($host_ip, $host_name, $flistname) = @_;
 295
 296     my $cmd = $rsync_command->(
 297         $host_name, '--timeout', '10', "${host_ip}:$spooldir", $spooldir,
 298         '--files-from', $flistname);
 299
 300     Proxmox::Utils::run_command($cmd);
 301 }
 302
 303 sub sync_spooldir {
 304     my ($host_ip, $host_name, $rcid) = @_;
 305
 306     mkdir "$spooldir/cluster/";
 307     my $syncdir = "$spooldir/cluster/$rcid";
 308     mkdir $syncdir;
 309
 310     my $cmd = $rsync_command->(
 311         $host_name, '-aq', '--timeout', '10', "${host_ip}:$syncdir/", $syncdir);
 312
 313     foreach my $incl (('spam/', 'spam/*', 'spam/*/*', 'virus/', 'virus/*', 'virus/*/*')) {
 314         push @$cmd, '--include', $incl;
 315     }
 316
 317     push @$cmd, '--exclude', '*';
 318
 319     PVE::Tools::run_command($cmd);
 320 }
 321
 322 sub sync_master_quar {
 323     my ($host_ip, $host_name) = @_;
 324
 325     my $syncdir = "$spooldir/cluster/";
 326     mkdir $syncdir;
 327
 328     my $cmd = $rsync_command->(
 329         $host_name, '-aq', '--timeout', '10', "${host_ip}:$syncdir", $syncdir);
 330
 331     PVE::Tools::run_command($cmd);
 332 }
 333
 334 sub sync_config_from_master {
 335     my ($cinfo, $master_name, $master_ip, $noreload) = @_;
 336
 337     my $local_ip = $cinfo->{local}->{ip};
 338     my $local_name = $cinfo->{local}->{name};
 339
 340     if ($local_ip eq $master_ip) {
 341         print STDERR "local node is master - nothing to do\n";
 342         return;
 343     }
 344
 345     mkdir $syncdir;
 346     File::Path::remove_tree($syncdir, {keep_root => 1});
 347
 348     my $sa_conf_dir = "/etc/mail/spamassassin";
 349     my $sa_custom_cf = "custom.cf";
 350
 351     my $cmd = $rsync_command->(
 352         $master_name, '-lpgoq',
 353         "${master_ip}:$cfgdir/* ${sa_conf_dir}/${sa_custom_cf}",
 354         "$syncdir/",
 355         '--exclude', '*~',
 356         '--exclude', '*.db',
 357         '--exclude', 'pmg-api.pem',
 358         '--exclude', 'pmg-tls.pem',
 359         );
 360
 361     my $errmsg = "syncing master configuration from '${master_ip}' failed";
 362     PVE::Tools::run_command($cmd, errmsg => $errmsg);
 363
 364     # verify that the remote host is cluster master
 365     open (my $fh, '<', "$syncdir/cluster.conf") ||
 366         die "unable to open synced cluster.conf - $!\n";
 367     my $newcinfo = PMG::ClusterConfig::read_cluster_conf('cluster.conf', $fh);
 368
 369     if (!$newcinfo->{master} || ($newcinfo->{master}->{ip} ne $master_ip)) {
 370         die "host '$master_ip' is not cluster master\n";
 371     }
 372
 373     my $role = $newcinfo->{'local'}->{type} // '-';
 374     die "local node '$newcinfo->{local}->{name}' not part of cluster\n"
 375         if $role eq '-';
 376
 377     die "local node '$newcinfo->{local}->{name}' is new cluster master\n"
 378         if $role eq 'master';
 379
 380
 381     $cond_commit_synced_file->('cluster.conf');
 382     $cinfo = $newcinfo;
 383
 384     my $files = [
 385         'pmg-authkey.key',
 386         'pmg-authkey.pub',
 387         'pmg-csrf.key',
 388         'ldap.conf',
 389         'user.conf',
 390         ];
 391
 392     foreach my $filename (@$files) {
 393         $cond_commit_synced_file->($filename);
 394     }
 395
 396     my $force_restart = {};
 397
 398     if ($cond_commit_synced_file->($sa_custom_cf, "${sa_conf_dir}/${sa_custom_cf}")) {
 399         $force_restart->{spam} = 1;
 400     }
 401
 402     $cond_commit_synced_file->('pmg.conf');
 403
 404     my $cfg = PMG::Config->new();
 405
 406     $cfg->rewrite_config(1, $force_restart);
 407 }
 408
 409 sub sync_ruledb_from_master {
 410     my ($ldb, $rdb, $ni, $ticket) = @_;
 411
 412     my $ruledb = PMG::RuleDB->new($ldb);
 413     my $rulecache = PMG::RuleCache->new($ruledb);
 414
 415     my $conn = PVE::APIClient::LWP->new(
 416         ticket => $ticket,
 417         cookie_name => 'PMGAuthCookie',
 418         host => $ni->{ip},
 419         cached_fingerprints => {
 420             $ni->{fingerprint} => 1,
 421         });
 422
 423     my $digest = $conn->get("/config/ruledb/digest", {});
 424
 425     return if $digest eq $rulecache->{digest}; # no changes
 426
 427     syslog('info', "detected rule database changes - starting sync from '$ni->{ip}'");
 428
 429     eval {
 430         $ldb->begin_work;
 431
 432         $ldb->do("DELETE FROM Rule");
 433         $ldb->do("DELETE FROM RuleGroup");
 434         $ldb->do("DELETE FROM ObjectGroup");
 435         $ldb->do("DELETE FROM Object");
 436         $ldb->do("DELETE FROM Attribut");
 437
 438         eval {
 439             $rdb->begin_work;
 440
 441             # read a consistent snapshot
 442             $rdb->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");
 443
 444             PMG::DBTools::copy_table($ldb, $rdb, "Rule");
 445             PMG::DBTools::copy_table($ldb, $rdb, "RuleGroup");
 446             PMG::DBTools::copy_table($ldb, $rdb, "ObjectGroup");
 447             PMG::DBTools::copy_table($ldb, $rdb, "Object", 'value');
 448             PMG::DBTools::copy_table($ldb, $rdb, "Attribut", 'value');
 449         };
 450
 451         $rdb->rollback; # end transaction
 452
 453         die $@ if $@;
 454
 455         # update sequences
 456
 457         $ldb->do("SELECT setval('rule_id_seq', max(id)+1) FROM Rule");
 458         $ldb->do("SELECT setval('object_id_seq', max(id)+1) FROM Object");
 459         $ldb->do("SELECT setval('objectgroup_id_seq', max(id)+1) FROM ObjectGroup");
 460
 461         $ldb->commit;
 462     };
 463     if (my $err = $@) {
 464         $ldb->rollback;
 465         die $err;
 466     }
 467
 468     syslog('info', "finished rule database sync from host '$ni->{ip}'");
 469 }
 470
 471 1;