src/bin/pmg-smtp-filter

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5
   6 use Carp;
   7 use Encode qw(encode);
   8 use Getopt::Long;
   9 use Time::HiRes qw (usleep gettimeofday tv_interval);
  10 use POSIX qw(:sys_wait_h errno_h signal_h);
  11
  12 use MIME::Parser;
  13 use MIME::WordDecoder qw(mime_to_perl_string);
  14 use File::Path;
  15 use Net::Server::PreFork;
  16 use Net::Server::SIG qw(register_sig check_sigs);
  17 use Net::SMTP;
  18
  19 use Fcntl ':flock';
  20 use File::Basename;
  21 use Xdgmime;
  22
  23 use PVE::SafeSyslog;
  24 use PVE::ProcFSTools;
  25 use PVE::INotify;
  26
  27 use Mail::SpamAssassin;
  28 use Mail::SpamAssassin::NetSet;
  29
  30 use PMG::pmgcfg;
  31 use PMG::Utils;
  32 use PMG::Cluster;
  33 use PMG::ClusterConfig;
  34
  35 use PMG::DBTools;
  36 use PMG::RuleDB;
  37 use PMG::RuleCache;
  38 use PMG::ModGroup;
  39 use PMG::AtomicFile;
  40 use PMG::LDAPConfig;
  41 use PMG::LDAPSet;
  42 use PMG::Config;
  43 use PMG::MailQueue;
  44 use PMG::Unpack;
  45 use PMG::SMTP;
  46
  47 use PMG::Unpack;
  48 use PMG::Statistic;
  49
  50 use base qw(Net::Server::PreFork);
  51
  52 $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
  53
  54 delete @ENV{qw(IFS CDPATH ENV BASH_ENV)};
  55
  56 my $opt_commandline = [$0, @ARGV];
  57 my $opt_max_dequeue = 1;
  58 my $opt_dequeue_time = 30;
  59
  60 my $opt_ext_port = 10024;
  61 my $opt_int_port = 10023;
  62 my $opt_inject_port = 10025;
  63
  64 my $opt_testmode;
  65 my $opt_untrusted;
  66 my $opt_pidfile;
  67 my $opt_database;
  68
  69 my $prog_name = 'pmg-smtp-filter';
  70
  71 initlog($prog_name, 'mail');
  72
  73 if (!GetOptions ('testmode=s' => \$opt_testmode,
  74                  'pidfile=s' => \$opt_pidfile,
  75                  'untrusted' => \$opt_untrusted,
  76                  'database=s' => \$opt_database)) {
  77     die "usage error\n";
  78     exit (-1);
  79 }
  80
  81 $opt_pidfile = "/run/${prog_name}.pid" if !$opt_pidfile;
  82
  83 my $max_servers = 1;
  84 my $min_servers = 1;
  85 my $min_spare_servers = 0;
  86 my $max_spare_servers = 0;
  87 my $max_requests = 1;
  88
  89 if (!$opt_testmode) {
  90     my $pmg_cfg = PMG::Config->new();
  91
  92     my $demo = $pmg_cfg->get('admin', 'demo');
  93
  94     if ($demo) {
  95         syslog ('info', 'demo mode detected - not starting server');
  96         exit (0);
  97     }
  98
  99     $max_servers = $pmg_cfg->get('mail', 'max_filters') + 2;
 100     $min_servers = 2;
 101     $min_spare_servers = 1;
 102     $max_spare_servers = 4;
 103     $max_requests = 20;
 104 }
 105
 106 $opt_max_dequeue = 0 if $opt_testmode;
 107
 108 my $daemonize = 1;
 109 if (defined ($ENV{BOUND_SOCKETS})) {
 110     $daemonize = undef;
 111 }
 112
 113 my $server_attr = {
 114     port => [ $opt_int_port, $opt_ext_port ],
 115     host => '127.0.0.1',
 116     min_servers => $min_servers,
 117     max_servers => $max_servers,
 118     min_spare_servers => $min_spare_servers,
 119     max_spare_servers => $max_spare_servers,
 120     max_requests => $max_requests,
 121     serialize => 'flock',
 122     max_dequeue => $opt_max_dequeue,
 123     check_for_dequeue => $opt_dequeue_time,
 124     log_level => 3,
 125     pid_file => $opt_pidfile,
 126     no_close_by_child => 1,
 127     no_client_stdout => 1,
 128     commandline => $opt_commandline,
 129 };
 130
 131 $server_attr->{setsid} = $daemonize if !$opt_testmode;
 132
 133 my $database;
 134
 135 if (defined($opt_database)) {
 136     $database = $opt_database;
 137 } else {
 138     $database = $opt_testmode ? "Proxmox_testdb" : "Proxmox_ruledb";
 139 }
 140
 141 $SIG{'__WARN__'} = sub {
 142     my $err = $@;
 143     my $t = $_[0];
 144     chomp $t;
 145     syslog('warning', "WARNING: %s", $t);
 146     $@ = $err;
 147 };
 148
 149 sub get_prox_vars {
 150     my ($self, $queue, $entity, $msginfo, $rule, $targets, $spaminfo) = @_;
 151
 152     $spaminfo = {
 153         sa_score => $queue->{sa_score},
 154         sa_hits => $queue->{sa_hits},
 155         sa_data => $queue->{sa_data},
 156         sa_max => $queue->{sa_max}
 157     } if !$spaminfo;
 158
 159     my $vars = {
 160         'SUBJECT' => PMG::Utils::decode_rfc1522($entity->head->get ('subject', 0) || 'No Subject'),
 161         'RULE' => $rule->{name},
 162         'RULE_INFO' => $msginfo->{rule_info},
 163         'SENDER' => $msginfo->{sender},
 164         'SENDER_IP' => $msginfo->{xforward}->{addr},
 165         'TARGETS' => join (', ', @$targets),
 166         'RECEIVERS' => join (', ', @{$msginfo->{targets}}),
 167         'SPAMLEVEL' => $spaminfo->{sa_score},
 168         'SPAMSTARS' => '*' x (($spaminfo->{sa_score} || 0) > 100 ? 100 : $spaminfo->{sa_score} || 0),
 169         'ADMIN' =>  $self->{pmg_cfg}->get('admin', 'email'),
 170         'HOST' =>  $msginfo->{hostname},
 171         'DOMAIN' =>  $msginfo->{domain},
 172         'FQDN' => $msginfo->{fqdn},
 173         'MSGID' => $queue->{msgid},
 174         'VERSION' => PMG::pmgcfg::package() . "/" . PMG::pmgcfg::release() . "/" . PMG::pmgcfg::repoid(),
 175     };
 176
 177     $vars->{__spaminfo} = $spaminfo;
 178
 179     if ($opt_testmode) {
 180         if ($queue->{vinfo_clam} || $queue->{vinfo_avast} || $queue->{vinfo_custom}) {
 181             $vars->{'VIRUS_INFO'} = "Virus Info:";
 182             $vars->{'VIRUS_INFO'} .= " clam: $queue->{vinfo_clam}" if $queue->{vinfo_clam};
 183             $vars->{'VIRUS_INFO'} .= " avast: $queue->{vinfo_avast}" if $queue->{vinfo_avast};
 184             $vars->{'VIRUS_INFO'} .= " custom: $queue->{vinfo_custom}" if $queue->{vinfo_custom};
 185         } else {
 186             $vars->{'VIRUS_INFO'} = '';
 187         }
 188     } else {
 189         if ($queue->{vinfo}) {
 190             $vars->{'VIRUS_INFO'} = "Virus Info: $queue->{vinfo}\n";
 191         } else {
 192             $vars->{'VIRUS_INFO'} = '';
 193         }
 194     }
 195
 196     $vars->{'SPAM_HITS'} = $spaminfo->{sa_hits};
 197
 198     $vars->{'SPAM_INFO'} = '';
 199     my $sscores = $spaminfo->{sa_data};
 200
 201     if (defined ($sscores) && @$sscores != -1) {
 202         my $sa_text;
 203         if ($opt_testmode) {
 204             $sa_text = "Spam detection results:  100\n";
 205         } else {
 206             $sa_text = "Spam detection results:  $spaminfo->{sa_score}\n";
 207         }
 208
 209         foreach my $s (@$sscores) {
 210             if ($opt_testmode) {
 211                 $sa_text .= sprintf ("%-22s %6s %s\n", $s->{rule}, 1, $s->{desc} || '-');
 212             } else {
 213                 $sa_text .= sprintf ("%-22s %6s %s\n", $s->{rule}, $s->{score}, $s->{desc} || '-');
 214             }
 215         }
 216         $vars->{'SPAM_INFO'} = $sa_text;
 217     }
 218
 219     if ($opt_testmode) {
 220         delete ($vars->{'ADMIN'});
 221         #delete ($vars->{'SPAM_INFO'});
 222     }
 223
 224     return $vars;
 225 }
 226
 227 sub apply_rules {
 228     my ($self, $queue, $msginfo, $entity, $ldap) = @_;
 229
 230     my $final;
 231     my %rule_targets;
 232     my %rule_actions;
 233     my %rule_marks;
 234     my $matching_rules = [];
 235
 236     my $rulecache = $self->{rulecache};
 237     my $rules = $rulecache->rules ();
 238     my $dbh = $self->{ruledb}->{dbh};
 239
 240     # first, we remove all conditional written 'X-' header attributes
 241     foreach my $rule (@$rules) {
 242         next if !$rule->{active};
 243         next if ($rule->{direction} == 0) &&  $msginfo->{trusted};
 244         next if ($rule->{direction} == 1) &&  !$msginfo->{trusted};
 245
 246         my $actions = $rulecache->get_actions ($rule->{id});
 247         if ($actions) {
 248             foreach my $action (@$actions) {
 249                 if ($action->isa ("PMG::RuleDB::ModField")) {
 250                     my $fname = $action->{field};
 251                     next if $fname !~ m/^X-/i;
 252                     $entity->head->delete($fname);
 253                 }
 254             }
 255         }
 256     }
 257
 258     foreach my $rule (@$rules) {
 259         next if !$rule->{active};
 260         next if ($rule->{direction} == 0) &&  $msginfo->{trusted};
 261         next if ($rule->{direction} == 1) &&  !$msginfo->{trusted};
 262
 263         # match from, when and what classes (not target dependent)
 264         if (!($rulecache->from_match ($rule->{id}, $msginfo->{sender}, $msginfo->{xforward}->{addr}, $ldap) &&
 265               $rulecache->when_match ($rule->{id}, time))) {
 266             next;
 267         }
 268
 269         $rule_marks{$rule->{id}} =
 270             $rulecache->what_match ($rule->{id}, $queue, $entity, $msginfo, $dbh);
 271
 272         $rule_actions{$rule->{id}} = $rulecache->get_actions ($rule->{id});
 273         my $fin = $rulecache->final ($rule->{id});
 274
 275         # match targets
 276         foreach my $target (@{$msginfo->{targets}}) {
 277             next if $final->{$target};
 278             next if !defined ($rule_marks{$rule->{id}});
 279             next if !defined ($rule_marks{$rule->{id}}->{$target});
 280             next if !defined ($rule_marks{$rule->{id}}->{$target}->{marks});
 281             next if !$rulecache->to_match ($rule->{id}, $target, $ldap);
 282
 283             $final->{$target} = $fin;
 284
 285             push @{$rule_targets{$rule->{id}}}, $target;
 286         }
 287     }
 288
 289     # Compute rule_info (summary about matching rule)
 290     # this can be used for debugging
 291     my $rule_info = "";
 292     foreach my $rule (@$rules) {
 293         next if !$rule_targets{$rule->{id}};
 294
 295         push @$matching_rules, $rule->{id};
 296
 297         $rule_info .= "Rule: $rule->{name}\n";
 298
 299         foreach my $target (@{$rule_targets{$rule->{id}}}) {
 300             $rule_info .= "  Receiver: $target\n";
 301         }
 302         foreach my $action (@{$rule_actions{$rule->{id}}}) {
 303             $rule_info .= "  Action: " . $action->short_desc () . "\n";
 304         }
 305
 306     }
 307     $msginfo->{rule_info} = $rule_info;
 308
 309     if ($msginfo->{testmode}) {
 310         my $vars = $self->get_prox_vars($queue, $entity, $msginfo, undef, [], undef);
 311         my $fh = $msginfo->{test_fh};
 312         print $fh $rule_info;
 313     }
 314
 315     # apply actions
 316
 317     my $mod_group = PMG::ModGroup->new($entity, $msginfo->{targets});
 318
 319     foreach my $rule (@$rules) {
 320         my $targets = $rule_targets{$rule->{id}};
 321         next if !$targets;
 322
 323         my $spaminfo;
 324         foreach my $t (@$targets) {
 325             if ($rule_marks{$rule->{id}}->{$t} && $rule_marks{$rule->{id}}->{$t}->{spaminfo}) {
 326                 $spaminfo = $rule_marks{$rule->{id}}->{$t}->{spaminfo};
 327                 # we assume spam info is the same for all matching targets
 328                 last;
 329             }
 330         }
 331
 332         my $vars = $self->get_prox_vars (
 333             $queue, $entity, $msginfo, $rule, $rule_targets{$rule->{id}}, $spaminfo);
 334
 335         my @sorted_actions = sort {$a->priority <=> $b->priority} @{$rule_actions{$rule->{id}}};
 336
 337         foreach my $action (@sorted_actions) {
 338             $action->execute(
 339                 $queue, $self->{ruledb}, $mod_group, $rule_targets{$rule->{id}}, $msginfo, $vars,
 340                 $rule_marks{$rule->{id}}->{marks}, $ldap
 341             );
 342             last if $action->final;
 343         }
 344     }
 345
 346     # we deliver all mail not matched by any rule (default action = accept)
 347     my $unmatched;
 348     foreach my $target (@{$msginfo->{targets}}) {
 349         next if $final->{$target};
 350
 351         push @$unmatched, $target;
 352     }
 353
 354     if ($unmatched) {
 355         my $accept = PMG::RuleDB::Accept->new ();
 356         $accept->execute ($queue, $self->{ruledb}, $mod_group, $unmatched,
 357                           $msginfo, { RULE => 'default-accept' }, undef);
 358     }
 359
 360     return $matching_rules;
 361 }
 362
 363 # reload ruledb and pmg config
 364 sub load_config {
 365     my $self = shift;
 366     my $prop = $self->{server};
 367
 368     if ($self->{ruledb}) {
 369         $self->log (0, "reloading configuration $database");
 370         $self->{ruledb}->close ();
 371     }
 372
 373     $self->{pmg_cfg} = PMG::Config->new();
 374     $self->{cinfo} = PVE::INotify::read_file("cluster.conf");
 375
 376     # create spool directories
 377     PMG::MailQueue::create_spooldirs($self->{cinfo}->{local}->{cid});
 378
 379     eval {
 380         my $dbh = PMG::DBTools::open_ruledb ($database);
 381         $self->{ruledb} = PMG::RuleDB->new ($dbh);
 382
 383         # load rulecache
 384         $self->{rulecache} = PMG::RuleCache->new ($self->{ruledb});
 385     };
 386
 387     my $err = $@;
 388
 389     if ($err) {
 390         sleep (10); # reduce restart rate when postgres is down
 391         die $err;
 392     }
 393
 394     # create LDAP object
 395     my $ldapconf = PVE::INotify::read_file('pmg-ldap.conf');
 396     $self->{ldap} = PMG::LDAPSet->new_from_ldap_cfg($ldapconf, 1);
 397
 398     $self->{reload_config} = 0;
 399 }
 400
 401 my $syslog_map = {
 402     0 => 'err',
 403     1 => 'warning',
 404     2 => 'notice',
 405     3 => 'info',
 406     4 => 'debug'
 407 };
 408
 409 sub log {
 410     my ($self, $level, $msg, @therest) = @_;
 411     my $prop = $self->{server};
 412
 413     return if $level =~ /^\d+$/ && $level > $prop->{log_level};
 414
 415     $level = $syslog_map->{$level} || $level;
 416     if (@therest) {
 417         syslog($level, $msg, @therest);
 418     } else {
 419         syslog ($level, $msg);
 420     }
 421 }
 422
 423 sub pre_loop_hook {
 424     my $self = shift;
 425     my $prop = $self->{server};
 426
 427     $prop->{log_level} = 3;
 428
 429     $self->log (0, "Filter daemon (re)started (max. $max_servers processes)");
 430
 431     eval {  PMG::MailQueue::cleanup_active(); };
 432     $self->log (0, "Cleanup failures: $@") if $@;
 433
 434     my $sig_set = POSIX::SigSet->new;
 435     $sig_set->addset (&POSIX::SIGHUP);
 436     $sig_set->addset (&POSIX::SIGCHLD);
 437     my $old_sig_set = POSIX::SigSet->new();
 438
 439     sigprocmask (SIG_UNBLOCK, $sig_set, $old_sig_set);
 440
 441     my ($backup_umask) = umask;
 442
 443     my $pmg_cfg = PMG::Config->new();
 444     $pmg_cfg->write_smtp_filter_config();
 445
 446     # Note: you need to restart the daemon when you change 'rbl_checks'
 447     my $rbl_checks = $pmg_cfg->get('spam', 'rbl_checks');
 448
 449     $self->{sa} = Mail::SpamAssassin->new ({
 450         debug => 0,
 451         local_tests_only => $opt_testmode || !$rbl_checks,
 452         home_dir_for_helpers => '/root',
 453         userstate_dir => '/root/.spamassassin',
 454         dont_copy_prefs   => 1,
 455         stop_at_threshold => 0,
 456     });
 457
 458     $self->{sa}->compile_now;
 459
 460     alarm (0); # SA forgets to clear alarm in some cases
 461     umask ($backup_umask);
 462     initlog ($prog_name, 'mail');
 463
 464     $SIG{'USR1'} = sub {
 465         # reloading server configuration
 466         if (defined $prop->{children}) {
 467             foreach my $pid (keys %{$prop->{children}}) {
 468                 kill (10, $pid); # SIGUSR1 children
 469             }
 470         }
 471     }
 472 }
 473
 474 sub child_init_hook {
 475     my $self = shift;
 476
 477     $0 = "$prog_name child";
 478
 479     # $self->log (3, "init child");
 480
 481     eval {
 482         $self->load_config ();
 483     };
 484
 485     if ($@) {
 486         $self->log (0, $@);
 487         $self->child_finish_hook;
 488         exit;
 489     }
 490
 491     $SIG{'USR1'} = sub {
 492         $self->{reload_config} = 1;
 493     }
 494 }
 495
 496 sub child_finish_hook {
 497     my $self = shift;
 498
 499     # $self->log (3, "finish child");
 500     $self->{ruledb}->close () if $self->{ruledb};
 501 }
 502
 503 my $last_dequeue_time = 0;
 504
 505 sub run_dequeue {
 506     my $self = shift;
 507
 508     my $err;
 509
 510     # do database maintenance here, this is called every 30 secends
 511
 512     eval { PMG::Utils::update_node_status_rrd() };
 513     if ($err = $@) {
 514         $self->log(0, "ERROR: $err");
 515         # continue
 516     }
 517
 518     my $ctime = time();
 519     my $tdiff = $ctime - $last_dequeue_time;
 520
 521     # return if tdiff less than 2 minutes
 522     return if $tdiff < 2*60;
 523     $last_dequeue_time = $ctime;
 524
 525     $self->log (2, "starting database maintenance");
 526
 527     my ($csec, $usec) = gettimeofday ();
 528
 529     my $cinfo = PVE::INotify::read_file("cluster.conf");
 530
 531     my $dbh = eval { PMG::DBTools::open_ruledb($database) };
 532     if ($err = $@) {
 533         $self->log (0, "ERROR: $err");
 534         return;
 535     }
 536
 537     eval { PMG::Statistic::update_stats($dbh, $cinfo) };
 538     $err = $@;
 539
 540     if ($err) {
 541         $self->log (0, $err);
 542     } else {
 543         my ($csec_end, $usec_end) = gettimeofday ();
 544         my $ptime = int (($csec_end - $csec) * 1000 + ($usec_end - $usec) / 1000);
 545         $self->log (2, "end database maintenance ($ptime ms)");
 546     }
 547
 548     $dbh->disconnect() if $dbh;
 549 }
 550
 551
 552 sub unpack_entity {
 553     my ($self, $unpack, $entity, $msginfo, $queue) = @_;
 554
 555     my ($magic, $path) = $entity->@{'PMX_magic_ct', 'PMX_decoded_path'};
 556
 557     if ($magic && $path) {
 558         my $filename = basename ($path);
 559
 560         if (PMG::Unpack::is_archive ($magic)) {
 561             $self->log (3, "$queue->{logid}: found archive '$filename' ($magic)");
 562
 563             my $start = [gettimeofday];
 564
 565             $unpack->{mime} = {};
 566
 567             eval { $unpack->unpack_archive ($path, $magic) };
 568             $self->log (3, "$queue->{logid}: unpack failed - $@") if $@;
 569
 570             $entity->{PMX_content_types} = $unpack->{mime};
 571             $entity->{PMX_filenames} = $unpack->{filenames};
 572
 573             if ($opt_testmode) {
 574                 my $types = join (", ", sort keys (%{$entity->{PMX_content_types}}));
 575                 my $fh = $msginfo->{test_fh};
 576                 $filename =~ s/\d+/X/g if $filename =~ m/^msg-\d+-\d+.msg/;
 577                 print $fh "Types:$filename: $types\n" if $types;
 578             }
 579
 580             my $elapsed = int(tv_interval($start) * 1000);
 581             $self->log (3, "$queue->{logid}: unpack archive '$filename' done ($elapsed ms)");
 582         }
 583     }
 584
 585     for my $part ($entity->parts)  {
 586         $self->unpack_entity ($unpack, $part, $msginfo, $queue);
 587     }
 588
 589 }
 590
 591 sub handle_smtp {
 592     my ($self, $smtp) = @_;
 593
 594     my ($csec, $usec) = gettimeofday ();
 595
 596     my $queue;
 597     my $msginfo = {};
 598     my $pmg_cfg = $self->{pmg_cfg};
 599     my $ldap = $self->{ldap};
 600     my $cinfo = $self->{cinfo};
 601     my $lcid = $cinfo->{local}->{cid};
 602
 603     $msginfo->{test_fh} = PMG::AtomicFile->new("testresult.out", "w")  if $opt_testmode;
 604
 605     $msginfo->{trusted} = $self->{trusted};
 606
 607
 608 # PHASE 1 - save incoming mail (already done)
 609 # on error: exit
 610
 611     $queue = $smtp->{queue};
 612     $queue->{sa} = $self->{sa};
 613     $queue->{clamav_heuristic_score} =
 614         $opt_testmode ? 100 : $pmg_cfg->get('spam', 'clamav_heuristic_score');
 615
 616     $queue->{lic_valid} = 1;
 617
 618     my $matching_rules;
 619
 620     eval {
 621         $msginfo->{testmode} = $opt_testmode;
 622         $msginfo->{sender} = $smtp->{from};
 623         $msginfo->{xforward} = $smtp->{xforward};
 624         $msginfo->{targets} = $smtp->{to};
 625         $msginfo->{param} = $smtp->{param};
 626
 627         my $dkim_sign = $msginfo->{trusted} && $pmg_cfg->get('admin', 'dkim_sign');
 628         if ($dkim_sign) {
 629             $msginfo->{dkim}->{sign} = $dkim_sign;
 630             $msginfo->{dkim}->{sign_all} = $pmg_cfg->get('admin', 'dkim_sign_all_mail');
 631             $msginfo->{dkim}->{selector} = $pmg_cfg->get('admin', 'dkim_selector');
 632         }
 633
 634         $msginfo->{hostname} = PVE::INotify::nodename();
 635         my $resolv = PVE::INotify::read_file('resolvconf');
 636
 637         $msginfo->{domain} = $resolv->{search};
 638         $msginfo->{fqdn} = $msginfo->{hostname};
 639         $msginfo->{fqdn} .= ".$msginfo->{domain}" if $msginfo->{domain};
 640         $msginfo->{lcid} = $lcid;
 641
 642         # $msginfo->{targets} is case sensitive,
 643         # but pmail is always lower case!
 644
 645         foreach my $t (@{$msginfo->{targets}}) {
 646             my $res;
 647             if ($ldap && ($res = $ldap->account_info ($t))) {
 648                 $msginfo->{pmail}->{$t} = $res->{pmail};
 649             } else {
 650                 $msginfo->{pmail}->{$t} = lc ($t);
 651             }
 652         }
 653
 654 # PHASE 2 - parse mail
 655 # on error: exit
 656
 657         my $maxfiles = $pmg_cfg->get('clamav', 'archivemaxfiles');
 658
 659         my $entity = $queue->parse_mail($maxfiles);
 660
 661         $self->log (3, "$queue->{logid}: new mail message-id=%s", $queue->{msgid});
 662
 663 # PHASE 3 - run external content analyzers
 664 # (SPAM analyzer is run on demand later)
 665 # on error: log error messages
 666
 667         # run custom script first
 668         my ($vinfo, $custom_spaminfo) = PMG::Utils::analyze_custom_check(
 669             $queue, $queue->{dataname}, $pmg_cfg, $opt_testmode);
 670
 671         # test for virus if none found
 672         if (!defined($vinfo)) {
 673             $vinfo = PMG::Utils::analyze_virus(
 674                 $queue, $queue->{dataname}, $pmg_cfg, $opt_testmode);
 675
 676             if ($vinfo && $vinfo =~ m/^Heuristics\.(.+)$/) {
 677                 my $hit = $1;
 678                 $queue->{clamav_heuristic} = $hit;
 679                 $vinfo = undef;
 680             }
 681         }
 682         $queue->{vinfo} = $vinfo;
 683
 684         # always add this headers to incoming mails
 685         # to enable user to report false negatives
 686         if (!$msginfo->{trusted}) {
 687             if ($queue->{vinfo}) {
 688                 $entity->head->replace('X-Proxmox-VInfo', $queue->{vinfo});
 689             }
 690         }
 691
 692         # we unpack after virus scanning, because this is more secure.
 693         # This way virus scanners gets the whole mail files and are able
 694         # to detect phishing signature for example - which would not work
 695         # if we decompose first and only analyze the decomposed attachments.
 696         # Disadvantage is that we need to unpack more than
 697         # once (bad performance).
 698
 699         # should we scan content types inside archives
 700
 701         my $rulecache = $self->{rulecache};
 702
 703         my $scan_archives = 0;
 704
 705         if (($rulecache->{archivefilter_in} && !$msginfo->{trusted}) ||
 706             ($rulecache->{archivefilter_out} && $msginfo->{trusted})) {
 707             $scan_archives = 1;
 708         }
 709
 710         if ($scan_archives && !$queue->{vinfo}) {
 711
 712             # unpack all archives - determine contained content types
 713
 714             my $decdir = $queue->{dumpdir} . "/__decoded_archives";
 715             mkdir $decdir;
 716
 717             my $start = [gettimeofday];
 718
 719             my $unpack;
 720             eval {
 721
 722                 # limits: We use clamav limit for maxfiles, and scan
 723                 # only 4 levels, timeout of 30 seconds
 724
 725                 $unpack = PMG::Unpack->new (
 726                     tmpdir => $decdir,
 727                     timeout => 30,
 728                     ctonly => 1, # only detect CTs
 729                     maxrec => -4,
 730                     maxfiles => $maxfiles);
 731
 732                 $self->unpack_entity ($unpack, $entity, $msginfo, $queue);
 733             };
 734
 735             my $err = $@;
 736
 737             $unpack->cleanup() if $unpack;
 738
 739             my $elapsed = int(tv_interval ($start) * 1000);
 740
 741             if ($err) {
 742                 $self->log (3, "$queue->{logid}: unpack archive failed ($elapsed ms) - $err");
 743             }
 744         }
 745
 746 # PHASE 4 - apply rules
 747 # on error: exit (cleanup process should do the rest)
 748         $msginfo->{maxspamsize} = $pmg_cfg->get('spam', 'maxspamsize');
 749         if ($msginfo->{maxspamsize} <= 1024*64) {
 750             $msginfo->{maxspamsize} = 1024*64;
 751         }
 752
 753         if ($msginfo->{trusted}) {
 754             my $hide = $pmg_cfg->get('mail', 'hide_received');
 755             $entity->head->delete("Received") if $hide;
 756         }
 757
 758         $matching_rules = $self->apply_rules($queue, $msginfo, $entity, $ldap);
 759     };
 760
 761     my $err = $@;
 762
 763     $self->{errors} = $queue->{errors};
 764
 765     # restart on error
 766     $self->{errors} = 1 if $err;
 767
 768     $queue->close ();
 769
 770     die $err if $err;
 771
 772     my ($csec_end, $usec_end) = gettimeofday ();
 773     my $time_total =
 774         int (($csec_end-$csec)*1000 + ($usec_end - $usec)/1000);
 775
 776 # PHASE 5 - log statistics
 777 # on error: log error messages
 778
 779     eval {
 780         my $dbh = $self->{ruledb}->{dbh};
 781
 782         my $where = "";
 783         foreach my $rid (@$matching_rules) {
 784             $where .= $where ? " OR ID = $rid" : "ID = $rid";
 785         }
 786
 787         $dbh->do ("UPDATE Rule SET Count = Count + 1 WHERE $where") if $where;
 788
 789         my $insert_cmds = "SELECT nextval ('cstatistic_id_seq');INSERT INTO CStatistic " .
 790             "(CID, RID, ID, Time, Bytes, Direction, Spamlevel, VirusInfo, PTime, Sender) VALUES (" .
 791             "$lcid, currval('cstatistic_id_seq'), currval('cstatistic_id_seq'),";
 792
 793         $insert_cmds .= $queue->{rtime} . ',';
 794         $insert_cmds .= $queue->{bytes} . ',';
 795         $insert_cmds .= $dbh->quote($msginfo->{trusted} ? 0 : 1) . ',';
 796         $insert_cmds .= ($queue->{sa_score} || 0) . ',';
 797         $insert_cmds .= $dbh->quote($queue->{vinfo}) . ',';
 798         $insert_cmds .= $time_total . ',';
 799         $insert_cmds .= $dbh->quote(encode('UTF-8', $msginfo->{sender})) . ');';
 800
 801         foreach my $r (@{$msginfo->{targets}}) {
 802             my $tmp = $dbh->quote(encode('UTF-8',$r));
 803             my $blocked = $queue->{status}->{$r} eq 'blocked' ? 1 : 0;
 804             $insert_cmds .= "INSERT INTO CReceivers (CStatistic_CID, CStatistic_RID, Receiver, Blocked) " .
 805                 "VALUES ($lcid, currval ('cstatistic_id_seq'), $tmp, '$blocked'); ";
 806         }
 807
 808         $dbh->do($insert_cmds);
 809     };
 810     # save $err (because log clears $@)
 811     $err = $@;
 812
 813     $time_total = $time_total/1000;
 814
 815     my $ptspam = ($queue->{ptime_spam} || 0)/1000;
 816     my $ptclam = ($queue->{ptime_clam} || 0)/1000;
 817     my $ptcustom = ($queue->{ptime_custom} || 0)/1000;
 818
 819     $self->log(3, "$queue->{logid}: processing time: ${time_total} seconds ($ptspam, $ptclam, $ptcustom)");
 820
 821     $msginfo->{test_fh}->close if $opt_testmode;
 822
 823     die $err if $err;
 824 }
 825
 826 my $initial_memory_usage;
 827
 828 sub process_request {
 829   my $self = shift;
 830   my $prop = $self->{server};
 831   my $sock = $prop->{client};
 832
 833   eval {
 834
 835       # make sure the ldap cache is up to date
 836       $self->{ldap}->update (1);
 837
 838       $self->load_config() if $self->{reload_config};
 839
 840       $self->{trusted} = 0;
 841       if ($prop->{sockport} == $opt_int_port && !$opt_untrusted) {
 842           $self->{trusted} = 1;
 843       }
 844
 845       my $smtp = PMG::SMTP->new ($sock);
 846
 847       my $maxcount = $max_requests - $prop->{requests};
 848
 849       my $count = $smtp->loop (\&handle_smtp, $self, $maxcount);
 850       if ($count > 1) {
 851           $prop->{requests} += $count - 1;
 852       }
 853   };
 854
 855   my $err = $@;
 856
 857   $self->log (0, $err) if $err;
 858
 859   kill(15, $prop->{ppid}) if $opt_testmode;
 860
 861   my $mem = PVE::ProcFSTools::read_memory_usage();
 862   if (!defined($initial_memory_usage) || ($prop->{requests} < 10)) {
 863       $initial_memory_usage = $mem->{resident};
 864   }
 865
 866   if ($opt_testmode) {
 867       $self->log (0, "memory usage: $mem->{size} bytes");
 868   } else {
 869       if ($self->{errors}) {
 870           $self->log (0, "fast exit because of errors (free $mem->{size} bytes)");
 871           $self->done (1);
 872       } else {
 873           my $diff = $mem->{resident} - $initial_memory_usage;
 874           if ($diff > 5*1024*1024) {
 875               $self->log (0, "fast exit to reduce server load (free $diff bytes)");
 876               $self->done (1);
 877           }
 878       }
 879   }
 880
 881   $self->done (1) if $err;
 882 }
 883
 884 # test sig_hup with: for ((;;)) ;do kill -HUP  `cat /run/${prog_name}.pid`; done;
 885 # wrapper to avoid multiple calls to sig_hup
 886 sub sig_hup {
 887   my $self = shift;
 888   my $prop = $self->{server};
 889
 890   return if defined ($prop->{_HUP}); # do not call twice
 891
 892   $self->SUPER::sig_hup();
 893 }
 894
 895 sub restart_close_hook {
 896   my $self = shift;
 897
 898   my $sig_set = POSIX::SigSet->new;
 899   $sig_set->addset (&POSIX::SIGHUP);
 900   $sig_set->addset (&POSIX::SIGCHLD); # to avoid zombies
 901   my $old_sig_set = POSIX::SigSet->new();
 902
 903   sigprocmask (SIG_BLOCK, $sig_set, $old_sig_set);
 904 }
 905
 906 sub pre_server_close_hook {
 907     my $self = shift;
 908     my $prop = $self->{server};
 909
 910     if (defined $prop->{_HUP}) {
 911         undef $prop->{pid_file_unlink};
 912     }
 913
 914     if (defined $prop->{children}) {
 915         foreach my $pid (keys %{$prop->{children}}) {
 916             kill (1, $pid); # HUP children
 917         }
 918     }
 919
 920     # nicely shutdown children (give them max 30 seconds to shut down)
 921     my $previous_alarm = alarm(30);
 922     eval {
 923         local $SIG{ALRM} = sub { die "Timed Out!\n" };
 924
 925         my $pid;
 926         1 while ((($pid = waitpid (-1, 0)) > 0) || ($! == EINTR));
 927
 928         alarm(0); # avoid race
 929     };
 930     alarm ($previous_alarm);
 931 }
 932
 933 # initialize mime system before fork
 934 xdg_mime_get_mime_type_for_file ($0);
 935
 936 my $server = bless {
 937     server => $server_attr,
 938 };
 939
 940 if (!$opt_testmode) {
 941     $server->run ();
 942 } else {
 943     if (fork) {
 944         $server->run();
 945     } else {
 946
 947         my $sender ='sender@proxtest.com';
 948         my $targets = ['target1@proxtest.com',
 949                        'target2@proxtest.com',
 950                        'target3@proxtest.com'];
 951
 952         my $smtp;
 953         while (!$smtp) {
 954             $smtp = Net::SMTP->new ('127.0.0.1', Port => 10023);
 955             last if $smtp;
 956             usleep(10);
 957         }
 958
 959         # syslog ('info', "connected to " . $smtp->domain);
 960
 961         $smtp->mail ($sender);
 962         $smtp->to (@$targets);
 963
 964         $smtp->data();
 965
 966         open (TMP, $opt_testmode) ||
 967             die "unable to upen file '$opt_testmode' - $! :ERROR";
 968         while (<TMP>) {
 969             $smtp->datasend ($_);
 970         }
 971         close TMP;
 972
 973         $smtp->datasend ("\n");
 974         $smtp->dataend ();
 975
 976         $smtp->quit;
 977     }
 978 }
 979
 980 exit (0);
 981
 982 __END__
 983
 984 =head1 NAME
 985
 986 pmg-smtp-filter - the Proxmox mail filter
 987
 988 =head1 SYNOPSIS
 989
 990 pmg-smtp-filter [-u] [-t testfile]
 991
 992 =head1 DESCRIPTION
 993
 994 Documentation is available at www.proxmox.com