src/bin/pmg-smtp-filter

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5
   6 use Carp;
   7 use Encode qw(encode);
   8 use Getopt::Long;
   9 use Time::HiRes qw (usleep gettimeofday tv_interval);
  10 use POSIX qw(:sys_wait_h errno_h signal_h);
  11
  12 use MIME::Parser;
  13 use MIME::WordDecoder qw(mime_to_perl_string);
  14 use File::Path;
  15 use Net::Server::PreFork;
  16 use Net::Server::SIG qw(register_sig check_sigs);
  17 use Net::SMTP;
  18
  19 use Fcntl ':flock';
  20 use File::Basename;
  21 use Xdgmime;
  22
  23 use PVE::SafeSyslog;
  24 use PVE::ProcFSTools;
  25 use PVE::INotify;
  26
  27 use Mail::SpamAssassin;
  28 use Mail::SpamAssassin::NetSet;
  29
  30 use PMG::pmgcfg;
  31 use PMG::Utils;
  32 use PMG::Cluster;
  33 use PMG::ClusterConfig;
  34
  35 use PMG::DBTools;
  36 use PMG::RuleDB;
  37 use PMG::RuleCache;
  38 use PMG::ModGroup;
  39 use PMG::AtomicFile;
  40 use PMG::LDAPConfig;
  41 use PMG::LDAPSet;
  42 use PMG::Config;
  43 use PMG::MailQueue;
  44 use PMG::Unpack;
  45 use PMG::SMTP;
  46
  47 use PMG::Unpack;
  48 use PMG::Statistic;
  49
  50 use base qw(Net::Server::PreFork);
  51
  52 $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
  53
  54 delete @ENV{qw(IFS CDPATH ENV BASH_ENV)};
  55
  56 my $opt_commandline = [$0, @ARGV];
  57 my $opt_max_dequeue = 1;
  58 my $opt_dequeue_time = 30;
  59
  60 my $opt_ext_port = 10024;
  61 my $opt_int_port = 10023;
  62 my $opt_inject_port = 10025;
  63
  64 my $opt_testmode;
  65 my $opt_untrusted;
  66 my $opt_pidfile;
  67 my $opt_database;
  68
  69 my $prog_name = 'pmg-smtp-filter';
  70
  71 initlog($prog_name, 'mail');
  72
  73 if (!GetOptions(
  74     'testmode=s' => \$opt_testmode,
  75     'pidfile=s' => \$opt_pidfile,
  76     'untrusted' => \$opt_untrusted,
  77     'database=s' => \$opt_database
  78 )) {
  79     die "usage error\n";
  80     exit (-1);
  81 }
  82
  83 $opt_pidfile = "/run/${prog_name}.pid" if !$opt_pidfile;
  84
  85 my $max_servers = 1;
  86 my $min_servers = 1;
  87 my $min_spare_servers = 0;
  88 my $max_spare_servers = 0;
  89 my $max_requests = 1;
  90
  91 if (!$opt_testmode) {
  92     my $pmg_cfg = PMG::Config->new();
  93
  94     my $demo = $pmg_cfg->get('admin', 'demo');
  95
  96     if ($demo) {
  97         syslog ('info', 'demo mode detected - not starting server');
  98         exit (0);
  99     }
 100
 101     my $memory = PMG::Config::Mail::physical_memory();
 102     if ($memory < 3840) {
 103         warn "total memory below 4 GiB, consider setting 'max_filters' manually to avoid OOM-kills\n"
 104             if !defined($pmg_cfg->get('mail', 'max_filters', 1));
 105     }
 106
 107     $max_servers = $pmg_cfg->get('mail', 'max_filters') + 2;
 108     $min_servers = 2;
 109     $min_spare_servers = 1;
 110     $max_spare_servers = 4;
 111     $max_requests = 20;
 112 }
 113
 114 print "using pre-fork workers with min=$min_servers, max=$max_servers, min_spare=$min_spare_servers"
 115     .", max_spare=$max_spare_servers, max_requests=$max_requests\n";
 116
 117 $opt_max_dequeue = 0 if $opt_testmode;
 118
 119 my $daemonize = 1;
 120 if (defined ($ENV{BOUND_SOCKETS})) {
 121     $daemonize = undef;
 122 }
 123
 124 my $server_attr = {
 125     port => [ $opt_int_port, $opt_ext_port ],
 126     host => '127.0.0.1',
 127     min_servers => $min_servers,
 128     max_servers => $max_servers,
 129     min_spare_servers => $min_spare_servers,
 130     max_spare_servers => $max_spare_servers,
 131     max_requests => $max_requests,
 132     serialize => 'flock',
 133     max_dequeue => $opt_max_dequeue,
 134     check_for_dequeue => $opt_dequeue_time,
 135     log_level => 3,
 136     pid_file => $opt_pidfile,
 137     no_close_by_child => 1,
 138     no_client_stdout => 1,
 139     commandline => $opt_commandline,
 140 };
 141
 142 $server_attr->{setsid} = $daemonize if !$opt_testmode;
 143
 144 my $database;
 145
 146 if (defined($opt_database)) {
 147     $database = $opt_database;
 148 } else {
 149     $database = $opt_testmode ? "Proxmox_testdb" : "Proxmox_ruledb";
 150 }
 151
 152 $SIG{'__WARN__'} = sub {
 153     my $err = $@;
 154     my $t = $_[0];
 155     chomp $t;
 156     syslog('warning', "WARNING: %s", $t);
 157     $@ = $err;
 158 };
 159
 160 sub get_prox_vars {
 161     my ($self, $queue, $entity, $msginfo, $rule, $targets, $spaminfo) = @_;
 162
 163     $spaminfo = {
 164         sa_score => $queue->{sa_score},
 165         sa_hits => $queue->{sa_hits},
 166         sa_data => $queue->{sa_data},
 167         sa_max => $queue->{sa_max}
 168     } if !$spaminfo;
 169
 170     my $vars = {
 171         'SUBJECT' => PMG::Utils::decode_rfc1522($entity->head->get ('subject', 0) || 'No Subject'),
 172         'RULE' => $rule->{name},
 173         'RULE_INFO' => $msginfo->{rule_info},
 174         'SENDER' => $msginfo->{sender},
 175         'SENDER_IP' => $msginfo->{xforward}->{addr},
 176         'TARGETS' => join (', ', @$targets),
 177         'RECEIVERS' => join (', ', @{$msginfo->{targets}}),
 178         'SPAMLEVEL' => $spaminfo->{sa_score},
 179         'SPAMSTARS' => '*' x (($spaminfo->{sa_score} || 0) > 100 ? 100 : $spaminfo->{sa_score} || 0),
 180         'ADMIN' =>  $self->{pmg_cfg}->get('admin', 'email'),
 181         'HOST' =>  $msginfo->{hostname},
 182         'DOMAIN' =>  $msginfo->{domain},
 183         'FQDN' => $msginfo->{fqdn},
 184         'MSGID' => $queue->{msgid},
 185         'VERSION' => PMG::pmgcfg::package() . "/" . PMG::pmgcfg::release() . "/" . PMG::pmgcfg::repoid(),
 186     };
 187
 188     $vars->{__spaminfo} = $spaminfo;
 189
 190     if ($opt_testmode) {
 191         if ($queue->{vinfo_clam} || $queue->{vinfo_avast} || $queue->{vinfo_custom}) {
 192             $vars->{'VIRUS_INFO'} = "Virus Info:";
 193             $vars->{'VIRUS_INFO'} .= " clam: $queue->{vinfo_clam}" if $queue->{vinfo_clam};
 194             $vars->{'VIRUS_INFO'} .= " avast: $queue->{vinfo_avast}" if $queue->{vinfo_avast};
 195             $vars->{'VIRUS_INFO'} .= " custom: $queue->{vinfo_custom}" if $queue->{vinfo_custom};
 196         } else {
 197             $vars->{'VIRUS_INFO'} = '';
 198         }
 199     } else {
 200         if ($queue->{vinfo}) {
 201             $vars->{'VIRUS_INFO'} = "Virus Info: $queue->{vinfo}\n";
 202         } else {
 203             $vars->{'VIRUS_INFO'} = '';
 204         }
 205     }
 206
 207     $vars->{'SPAM_HITS'} = $spaminfo->{sa_hits};
 208
 209     $vars->{'SPAM_INFO'} = '';
 210     my $sscores = $spaminfo->{sa_data};
 211
 212     if (defined ($sscores) && @$sscores != -1) {
 213         my $sa_text;
 214         if ($opt_testmode) {
 215             $sa_text = "Spam detection results:  100\n";
 216         } else {
 217             $sa_text = "Spam detection results:  $spaminfo->{sa_score}\n";
 218         }
 219
 220         foreach my $s (@$sscores) {
 221             if ($opt_testmode) {
 222                 $sa_text .= sprintf ("%-22s %6s %s\n", $s->{rule}, 1, $s->{desc} || '-');
 223             } else {
 224                 $sa_text .= sprintf ("%-22s %6s %s\n", $s->{rule}, $s->{score}, $s->{desc} || '-');
 225             }
 226         }
 227         $vars->{'SPAM_INFO'} = $sa_text;
 228     }
 229
 230     if ($opt_testmode) {
 231         delete ($vars->{'ADMIN'});
 232         #delete ($vars->{'SPAM_INFO'});
 233     }
 234
 235     return $vars;
 236 }
 237
 238 sub apply_rules {
 239     my ($self, $queue, $msginfo, $entity, $ldap) = @_;
 240
 241     my $final;
 242     my %rule_targets;
 243     my %rule_actions;
 244     my %rule_marks;
 245     my %rule_spaminfo;
 246     my $matching_rules = [];
 247
 248     my $rulecache = $self->{rulecache};
 249     my $rules = $rulecache->rules ();
 250     my $dbh = $self->{ruledb}->{dbh};
 251
 252     # first, we remove all conditional written 'X-' header attributes
 253     foreach my $rule (@$rules) {
 254         next if !$rule->{active};
 255         next if ($rule->{direction} == 0) &&  $msginfo->{trusted};
 256         next if ($rule->{direction} == 1) &&  !$msginfo->{trusted};
 257
 258         my $actions = $rulecache->get_actions ($rule->{id});
 259         if ($actions) {
 260             foreach my $action (@$actions) {
 261                 if ($action->isa ("PMG::RuleDB::ModField")) {
 262                     my $fname = $action->{field};
 263                     next if $fname !~ m/^X-/i;
 264                     $entity->head->delete($fname);
 265                 }
 266             }
 267         }
 268     }
 269
 270     foreach my $rule (@$rules) {
 271         next if !$rule->{active};
 272         next if ($rule->{direction} == 0) &&  $msginfo->{trusted};
 273         next if ($rule->{direction} == 1) &&  !$msginfo->{trusted};
 274
 275         # match from, when and what classes (not target dependent)
 276         if (!($rulecache->from_match ($rule->{id}, $msginfo->{sender}, $msginfo->{xforward}->{addr}, $ldap) &&
 277               $rulecache->when_match ($rule->{id}, time))) {
 278             next;
 279         }
 280
 281         my ($marks, $spaminfo) =
 282             $rulecache->what_match ($rule->{id}, $queue, $entity, $msginfo, $dbh);
 283
 284         $rule_marks{$rule->{id}} = $marks;
 285         $rule_spaminfo{$rule->{id}} = $spaminfo;
 286
 287         $rule_actions{$rule->{id}} = $rulecache->get_actions ($rule->{id});
 288         my $fin = $rulecache->final ($rule->{id});
 289
 290         # match targets
 291         foreach my $target (@{$msginfo->{targets}}) {
 292             next if $final->{$target};
 293             next if !defined ($rule_marks{$rule->{id}});
 294             next if !defined ($rule_marks{$rule->{id}}->{$target});
 295             next if !$rulecache->to_match ($rule->{id}, $target, $ldap);
 296
 297             $final->{$target} = $fin;
 298
 299             push @{$rule_targets{$rule->{id}}}, $target;
 300         }
 301     }
 302
 303     # Compute rule_info (summary about matching rule)
 304     # this can be used for debugging
 305     my $rule_info = "";
 306     foreach my $rule (@$rules) {
 307         next if !$rule_targets{$rule->{id}};
 308
 309         push @$matching_rules, $rule->{id};
 310
 311         $rule_info .= "Rule: $rule->{name}\n";
 312
 313         foreach my $target (@{$rule_targets{$rule->{id}}}) {
 314             $rule_info .= "  Receiver: $target\n";
 315         }
 316         foreach my $action (@{$rule_actions{$rule->{id}}}) {
 317             $rule_info .= "  Action: " . $action->short_desc () . "\n";
 318         }
 319
 320     }
 321     $msginfo->{rule_info} = $rule_info;
 322
 323     if ($msginfo->{testmode}) {
 324         my $vars = $self->get_prox_vars($queue, $entity, $msginfo, undef, [], undef);
 325         my $fh = $msginfo->{test_fh};
 326         print $fh $rule_info;
 327     }
 328
 329     # apply actions
 330
 331     my $mod_group = PMG::ModGroup->new($entity, $msginfo->{targets});
 332
 333     my $processing_time = int(tv_interval($msginfo->{starttime}));
 334     my $filter_timeout = $self->{pmg_cfg}->get('mail', 'filter-timeout');
 335     die "processing took ${processing_time}s, longer than the timeout (${filter_timeout}s)\n"
 336         if $processing_time > $filter_timeout;
 337
 338     foreach my $rule (@$rules) {
 339         my $targets = $rule_targets{$rule->{id}};
 340         next if !$targets;
 341
 342         my $vars = $self->get_prox_vars (
 343             $queue, $entity, $msginfo, $rule, $rule_targets{$rule->{id}}, $rule_spaminfo{$rule->{id}});
 344
 345         my @sorted_actions = sort {$a->priority <=> $b->priority} @{$rule_actions{$rule->{id}}};
 346
 347         foreach my $action (@sorted_actions) {
 348             $action->execute(
 349                 $queue, $self->{ruledb}, $mod_group, $rule_targets{$rule->{id}}, $msginfo, $vars,
 350                 $rule_marks{$rule->{id}}, $ldap
 351             );
 352             last if $action->final;
 353         }
 354     }
 355
 356     # we deliver all mail not matched by any rule (default action = accept)
 357     my $unmatched;
 358     foreach my $target (@{$msginfo->{targets}}) {
 359         next if $final->{$target};
 360
 361         push @$unmatched, $target;
 362     }
 363
 364     if ($unmatched) {
 365         my $accept = PMG::RuleDB::Accept->new ();
 366         $accept->execute ($queue, $self->{ruledb}, $mod_group, $unmatched,
 367                           $msginfo, { RULE => 'default-accept' }, undef);
 368     }
 369
 370     return $matching_rules;
 371 }
 372
 373 # reload ruledb and pmg config
 374 sub load_config {
 375     my $self = shift;
 376     my $prop = $self->{server};
 377
 378     if ($self->{ruledb}) {
 379         $self->log ('info', "reloading configuration $database");
 380         $self->{ruledb}->close ();
 381     }
 382
 383     $self->{pmg_cfg} = PMG::Config->new();
 384     $self->{cinfo} = PVE::INotify::read_file("cluster.conf");
 385
 386     # create spool directories
 387     PMG::MailQueue::create_spooldirs($self->{cinfo}->{local}->{cid});
 388
 389     eval {
 390         my $dbh = PMG::DBTools::open_ruledb ($database);
 391         $self->{ruledb} = PMG::RuleDB->new ($dbh);
 392
 393         # load rulecache
 394         $self->{rulecache} = PMG::RuleCache->new ($self->{ruledb});
 395     };
 396
 397     my $err = $@;
 398
 399     if ($err) {
 400         sleep (10); # reduce restart rate when postgres is down
 401         die $err;
 402     }
 403
 404     # create LDAP object
 405     my $ldapconf = PVE::INotify::read_file('pmg-ldap.conf');
 406     $self->{ldap} = PMG::LDAPSet->new_from_ldap_cfg($ldapconf, 1);
 407
 408     $self->{reload_config} = 0;
 409 }
 410
 411 my $syslog_map = {
 412     0 => 'err',
 413     1 => 'warning',
 414     2 => 'notice',
 415     3 => 'info',
 416     4 => 'debug'
 417 };
 418
 419 sub log {
 420     my ($self, $level, $msg, @therest) = @_;
 421     my $prop = $self->{server};
 422
 423     return if $level =~ /^\d+$/ && $level > $prop->{log_level};
 424
 425     $level = $syslog_map->{$level} || $level;
 426     if (@therest) {
 427         syslog($level, $msg, @therest);
 428     } else {
 429         syslog ($level, $msg);
 430     }
 431 }
 432
 433 sub pre_loop_hook {
 434     my $self = shift;
 435     my $prop = $self->{server};
 436
 437     $prop->{log_level} = 3;
 438
 439     $self->log ('info', "Filter daemon (re)started (max. $max_servers processes)");
 440
 441     eval {  PMG::MailQueue::cleanup_active(); };
 442     $self->log (0, "Cleanup failures: $@") if $@;
 443
 444     my $sig_set = POSIX::SigSet->new;
 445     $sig_set->addset (&POSIX::SIGHUP);
 446     $sig_set->addset (&POSIX::SIGCHLD);
 447     my $old_sig_set = POSIX::SigSet->new();
 448
 449     sigprocmask (SIG_UNBLOCK, $sig_set, $old_sig_set);
 450
 451     my ($backup_umask) = umask;
 452
 453     my $pmg_cfg = PMG::Config->new();
 454     $pmg_cfg->write_smtp_filter_config();
 455
 456     # Note: you need to restart the daemon when you change 'rbl_checks'
 457     my $rbl_checks = $pmg_cfg->get('spam', 'rbl_checks');
 458
 459     $self->{sa} = Mail::SpamAssassin->new ({
 460         debug => 0,
 461         local_tests_only => $opt_testmode || !$rbl_checks,
 462         home_dir_for_helpers => '/root',
 463         userstate_dir => '/root/.spamassassin',
 464         dont_copy_prefs   => 1,
 465         stop_at_threshold => 0,
 466     });
 467
 468     $self->{sa}->compile_now;
 469
 470     alarm (0); # SA forgets to clear alarm in some cases
 471     umask ($backup_umask);
 472     initlog ($prog_name, 'mail');
 473
 474     $SIG{'USR1'} = sub {
 475         # reloading server configuration
 476         if (defined $prop->{children}) {
 477             foreach my $pid (keys %{$prop->{children}}) {
 478                 kill (10, $pid); # SIGUSR1 children
 479             }
 480         }
 481     }
 482 }
 483
 484 sub child_init_hook {
 485     my $self = shift;
 486
 487     $0 = "$prog_name child";
 488
 489     # $self->log (3, "init child");
 490
 491     eval {
 492         $self->load_config ();
 493     };
 494
 495     if ($@) {
 496         $self->log (0, $@);
 497         $self->child_finish_hook;
 498         exit;
 499     }
 500
 501     $SIG{'USR1'} = sub {
 502         $self->{reload_config} = 1;
 503     }
 504 }
 505
 506 sub child_finish_hook {
 507     my $self = shift;
 508
 509     # $self->log (3, "finish child");
 510     $self->{ruledb}->close () if $self->{ruledb};
 511 }
 512
 513 my $last_dequeue_time = 0;
 514
 515 sub run_dequeue {
 516     my $self = shift;
 517
 518     my $err;
 519
 520     # do database maintenance here, this is called every 30 secends
 521
 522     eval { PMG::Utils::update_node_status_rrd() };
 523     if ($err = $@) {
 524         $self->log(0, "ERROR: $err");
 525         # continue
 526     }
 527
 528     my $ctime = time();
 529     my $tdiff = $ctime - $last_dequeue_time;
 530
 531     # return if tdiff less than 2 minutes
 532     return if $tdiff < 2*60;
 533     $last_dequeue_time = $ctime;
 534
 535     $self->log (2, "starting database maintenance");
 536
 537     my $starttime = [ gettimeofday() ];
 538
 539     my $cinfo = PVE::INotify::read_file("cluster.conf");
 540
 541     my $dbh = eval { PMG::DBTools::open_ruledb($database) };
 542     if ($err = $@) {
 543         $self->log (0, "ERROR: $err");
 544         return;
 545     }
 546
 547     eval { PMG::Statistic::update_stats($dbh, $cinfo) };
 548     $err = $@;
 549
 550     if ($err) {
 551         $self->log (0, $err);
 552     } else {
 553         my $ptime = int(tv_interval($starttime) * 1000);
 554         $self->log (2, "end database maintenance ($ptime ms)");
 555     }
 556
 557     $dbh->disconnect() if $dbh;
 558 }
 559
 560
 561 sub unpack_entity {
 562     my ($self, $unpack, $entity, $msginfo, $queue) = @_;
 563
 564     my ($magic, $path) = $entity->@{'PMX_magic_ct', 'PMX_decoded_path'};
 565
 566     if ($magic && $path) {
 567         my $filename = basename ($path);
 568
 569         if (PMG::Unpack::is_archive ($magic)) {
 570             $self->log (3, "$queue->{logid}: found archive '$filename' ($magic)");
 571
 572             my $start = [ gettimeofday() ];
 573
 574             $unpack->{mime} = {};
 575
 576             eval { $unpack->unpack_archive ($path, $magic) };
 577             $self->log (3, "$queue->{logid}: unpack failed - $@") if $@;
 578
 579             $entity->{PMX_content_types} = $unpack->{mime};
 580             $entity->{PMX_filenames} = $unpack->{filenames};
 581
 582             if ($opt_testmode) {
 583                 my $types = join (", ", sort keys (%{$entity->{PMX_content_types}}));
 584                 my $fh = $msginfo->{test_fh};
 585                 $filename =~ s/\d+/X/g if $filename =~ m/^msg-\d+-\d+.msg/;
 586                 print $fh "Types:$filename: $types\n" if $types;
 587             }
 588
 589             my $elapsed = int(tv_interval($start) * 1000);
 590             $self->log (3, "$queue->{logid}: unpack archive '$filename' done ($elapsed ms)");
 591         }
 592     }
 593
 594     for my $part ($entity->parts)  {
 595         $self->unpack_entity ($unpack, $part, $msginfo, $queue);
 596     }
 597
 598 }
 599
 600 sub handle_smtp {
 601     my ($self, $smtp) = @_;
 602
 603     my $starttime = [ gettimeofday() ];
 604
 605     my $queue;
 606     my $msginfo = {};
 607     my $pmg_cfg = $self->{pmg_cfg};
 608     my $ldap = $self->{ldap};
 609     my $cinfo = $self->{cinfo};
 610     my $lcid = $cinfo->{local}->{cid};
 611
 612     $msginfo->{test_fh} = PMG::AtomicFile->new("testresult.out", "w")  if $opt_testmode;
 613
 614     $msginfo->{trusted} = $self->{trusted};
 615
 616
 617 # PHASE 1 - save incoming mail (already done)
 618 # on error: exit
 619
 620     $queue = $smtp->{queue};
 621     $queue->{sa} = $self->{sa};
 622     $queue->{clamav_heuristic_score} =
 623         $opt_testmode ? 100 : $pmg_cfg->get('spam', 'clamav_heuristic_score');
 624
 625     $queue->{lic_valid} = 1;
 626
 627     my $matching_rules;
 628
 629     eval {
 630         $msginfo->{testmode} = $opt_testmode;
 631         $msginfo->{sender} = $smtp->{from};
 632         $msginfo->{xforward} = $smtp->{xforward};
 633         $msginfo->{targets} = $smtp->{to};
 634         $msginfo->{param} = $smtp->{param};
 635
 636         my $dkim_sign = $msginfo->{trusted} && $pmg_cfg->get('admin', 'dkim_sign');
 637         if ($dkim_sign) {
 638             $msginfo->{dkim}->{sign} = $dkim_sign;
 639             $msginfo->{dkim}->{use_domain} = $pmg_cfg->get('admin', 'dkim-use-domain');
 640             $msginfo->{dkim}->{sign_all} = $pmg_cfg->get('admin', 'dkim_sign_all_mail');
 641             $msginfo->{dkim}->{selector} = $pmg_cfg->get('admin', 'dkim_selector');
 642         }
 643
 644         $msginfo->{hostname} = PVE::INotify::nodename();
 645         my $resolv = PVE::INotify::read_file('resolvconf');
 646
 647         $msginfo->{domain} = $resolv->{search};
 648         $msginfo->{fqdn} = $msginfo->{hostname};
 649         $msginfo->{fqdn} .= ".$msginfo->{domain}" if $msginfo->{domain};
 650         $msginfo->{lcid} = $lcid;
 651         $msginfo->{starttime} = $starttime;
 652
 653         # $msginfo->{targets} is case sensitive,
 654         # but pmail is always lower case!
 655
 656         foreach my $t (@{$msginfo->{targets}}) {
 657             my $res;
 658             if ($ldap && ($res = $ldap->account_info ($t))) {
 659                 $msginfo->{pmail}->{$t} = $res->{pmail};
 660             } else {
 661                 $msginfo->{pmail}->{$t} = lc ($t);
 662             }
 663         }
 664
 665 # PHASE 2 - parse mail
 666 # on error: exit
 667
 668         my $maxfiles = $pmg_cfg->get('clamav', 'archivemaxfiles');
 669
 670         my ($entity, $max_aid) = $queue->parse_mail($maxfiles);
 671         $msginfo->{max_aid} = $max_aid;
 672
 673         $self->log (3, "$queue->{logid}: new mail message-id=%s", $queue->{msgid});
 674
 675 # PHASE 3 - run external content analyzers
 676 # (SPAM analyzer is run on demand later)
 677 # on error: log error messages
 678
 679         # run custom script first
 680         my ($vinfo, $custom_spaminfo) = PMG::Utils::analyze_custom_check(
 681             $queue, $queue->{dataname}, $pmg_cfg, $opt_testmode);
 682
 683         # test for virus if none found
 684         if (!defined($vinfo)) {
 685             $vinfo = PMG::Utils::analyze_virus(
 686                 $queue, $queue->{dataname}, $pmg_cfg, $opt_testmode);
 687
 688             if ($vinfo && $vinfo =~ m/^Heuristics\.(.+)$/) {
 689                 my $hit = $1;
 690                 $queue->{clamav_heuristic} = $hit;
 691                 $vinfo = undef;
 692             }
 693         }
 694         $queue->{vinfo} = $vinfo;
 695
 696         # always add this headers to incoming mails
 697         # to enable user to report false negatives
 698         if (!$msginfo->{trusted}) {
 699             if ($queue->{vinfo}) {
 700                 $entity->head->replace('X-Proxmox-VInfo', $queue->{vinfo});
 701             }
 702         }
 703
 704         # we unpack after virus scanning, because this is more secure.
 705         # This way virus scanners gets the whole mail files and are able
 706         # to detect phishing signature for example - which would not work
 707         # if we decompose first and only analyze the decomposed attachments.
 708         # Disadvantage is that we need to unpack more than
 709         # once (bad performance).
 710
 711         # should we scan content types inside archives
 712
 713         my $rulecache = $self->{rulecache};
 714
 715         my $scan_archives = 0;
 716
 717         if (($rulecache->{archivefilter_in} && !$msginfo->{trusted}) ||
 718             ($rulecache->{archivefilter_out} && $msginfo->{trusted})) {
 719             $scan_archives = 1;
 720         }
 721
 722         if ($scan_archives && !$queue->{vinfo}) {
 723
 724             # unpack all archives - determine contained content types
 725
 726             my $decdir = $queue->{dumpdir} . "/__decoded_archives";
 727             mkdir $decdir;
 728
 729             my $start = [ gettimeofday() ];
 730
 731             my $unpack;
 732             eval {
 733
 734                 # limits: We use clamav limit for maxfiles, and scan
 735                 # only 4 levels, timeout of 30 seconds
 736
 737                 $unpack = PMG::Unpack->new (
 738                     tmpdir => $decdir,
 739                     timeout => 30,
 740                     ctonly => 1, # only detect CTs
 741                     maxrec => -4,
 742                     maxfiles => $maxfiles);
 743
 744                 $self->unpack_entity ($unpack, $entity, $msginfo, $queue);
 745             };
 746
 747             my $err = $@;
 748
 749             $unpack->cleanup() if $unpack;
 750
 751             my $elapsed = int(tv_interval ($start) * 1000);
 752
 753             if ($err) {
 754                 $self->log (3, "$queue->{logid}: unpack archive failed ($elapsed ms) - $err");
 755             }
 756         }
 757
 758 # PHASE 4 - apply rules
 759 # on error: exit (cleanup process should do the rest)
 760         $msginfo->{maxspamsize} = $pmg_cfg->get('spam', 'maxspamsize');
 761         if ($msginfo->{maxspamsize} <= 1024*64) {
 762             $msginfo->{maxspamsize} = 1024*64;
 763         }
 764
 765         if ($msginfo->{trusted}) {
 766             my $hide = $pmg_cfg->get('mail', 'hide_received');
 767             $entity->head->delete("Received") if $hide;
 768         }
 769
 770         $matching_rules = $self->apply_rules($queue, $msginfo, $entity, $ldap);
 771     };
 772
 773     my $err = $@;
 774
 775     $self->{errors} = $queue->{errors};
 776
 777     # restart on error
 778     $self->{errors} = 1 if $err;
 779
 780     $queue->close ();
 781
 782     die $err if $err;
 783
 784     my $time_total = int(tv_interval($starttime) * 1000);
 785
 786 # PHASE 5 - log statistics
 787 # on error: log error messages
 788
 789     eval {
 790         my $dbh = $self->{ruledb}->{dbh};
 791
 792         my $where = "";
 793         foreach my $rid (@$matching_rules) {
 794             $where .= $where ? " OR ID = $rid" : "ID = $rid";
 795         }
 796
 797         $dbh->do ("UPDATE Rule SET Count = Count + 1 WHERE $where") if $where;
 798
 799         my $insert_cmds = "SELECT nextval ('cstatistic_id_seq');INSERT INTO CStatistic " .
 800             "(CID, RID, ID, Time, Bytes, Direction, Spamlevel, VirusInfo, PTime, Sender) VALUES (" .
 801             "$lcid, currval('cstatistic_id_seq'), currval('cstatistic_id_seq'),";
 802
 803         $insert_cmds .= $queue->{rtime} . ',';
 804         $insert_cmds .= $queue->{bytes} . ',';
 805         $insert_cmds .= $dbh->quote($msginfo->{trusted} ? 0 : 1) . ',';
 806         $insert_cmds .= ($queue->{sa_score} || 0) . ',';
 807         $insert_cmds .= $dbh->quote($queue->{vinfo}) . ',';
 808         $insert_cmds .= $time_total . ',';
 809         $insert_cmds .= $dbh->quote(encode('UTF-8', $msginfo->{sender})) . ');';
 810
 811         foreach my $r (@{$msginfo->{targets}}) {
 812             my $tmp = $dbh->quote(encode('UTF-8',$r));
 813             my $blocked = $queue->{status}->{$r} eq 'blocked' ? 1 : 0;
 814             $insert_cmds .= "INSERT INTO CReceivers (CStatistic_CID, CStatistic_RID, Receiver, Blocked) " .
 815                 "VALUES ($lcid, currval ('cstatistic_id_seq'), $tmp, '$blocked'); ";
 816         }
 817
 818         $dbh->do($insert_cmds);
 819     };
 820     # save $err (because log clears $@)
 821     $err = $@;
 822
 823     $time_total = $time_total/1000;
 824
 825     my $ptspam = ($queue->{ptime_spam} || 0)/1000;
 826     my $ptclam = ($queue->{ptime_clam} || 0)/1000;
 827     my $ptcustom = ($queue->{ptime_custom} || 0)/1000;
 828
 829     $self->log(3, "$queue->{logid}: processing time: ${time_total} seconds ($ptspam, $ptclam, $ptcustom)");
 830
 831     $msginfo->{test_fh}->close if $opt_testmode;
 832
 833     die $err if $err;
 834 }
 835
 836 my $initial_memory_usage;
 837
 838 sub process_request {
 839   my $self = shift;
 840   my $prop = $self->{server};
 841   my $sock = $prop->{client};
 842
 843   eval {
 844
 845       # make sure the ldap cache is up to date
 846       $self->{ldap}->update (1);
 847
 848       $self->load_config() if $self->{reload_config};
 849
 850       $self->{trusted} = 0;
 851       if ($prop->{sockport} == $opt_int_port && !$opt_untrusted) {
 852           $self->{trusted} = 1;
 853       }
 854
 855       my $smtp = PMG::SMTP->new ($sock);
 856
 857       my $maxcount = $max_requests - $prop->{requests};
 858
 859       my $count = $smtp->loop (\&handle_smtp, $self, $maxcount);
 860       if ($count > 1) {
 861           $prop->{requests} += $count - 1;
 862       }
 863   };
 864
 865   my $err = $@;
 866
 867   $self->log (0, $err) if $err;
 868
 869   kill(15, $prop->{ppid}) if $opt_testmode;
 870
 871   my $mem = PVE::ProcFSTools::read_memory_usage();
 872   if (!defined($initial_memory_usage) || ($prop->{requests} < 10)) {
 873       $initial_memory_usage = $mem->{resident};
 874   }
 875
 876   if ($opt_testmode) {
 877       $self->log (0, "memory usage: $mem->{size} bytes");
 878   } else {
 879       if ($self->{errors}) {
 880           $self->log (0, "fast exit because of errors (free $mem->{size} bytes)");
 881           $self->done (1);
 882       } else {
 883           my $diff = $mem->{resident} - $initial_memory_usage;
 884           if ($diff > 5*1024*1024) {
 885               $self->log (0, "fast exit to reduce server load (free $diff bytes)");
 886               $self->done (1);
 887           }
 888       }
 889   }
 890
 891   $self->done (1) if $err;
 892 }
 893
 894 # test sig_hup with: for ((;;)) ;do kill -HUP  `cat /run/${prog_name}.pid`; done;
 895 # wrapper to avoid multiple calls to sig_hup
 896 sub sig_hup {
 897   my $self = shift;
 898   my $prop = $self->{server};
 899
 900   return if defined ($prop->{_HUP}); # do not call twice
 901
 902   $self->SUPER::sig_hup();
 903 }
 904
 905 sub restart_close_hook {
 906   my $self = shift;
 907
 908   my $sig_set = POSIX::SigSet->new;
 909   $sig_set->addset (&POSIX::SIGHUP);
 910   $sig_set->addset (&POSIX::SIGCHLD); # to avoid zombies
 911   my $old_sig_set = POSIX::SigSet->new();
 912
 913   sigprocmask (SIG_BLOCK, $sig_set, $old_sig_set);
 914 }
 915
 916 sub pre_server_close_hook {
 917     my $self = shift;
 918     my $prop = $self->{server};
 919
 920     if (defined $prop->{_HUP}) {
 921         undef $prop->{pid_file_unlink};
 922     }
 923
 924     if (defined $prop->{children}) {
 925         foreach my $pid (keys %{$prop->{children}}) {
 926             kill (1, $pid); # HUP children
 927         }
 928     }
 929
 930     # nicely shutdown children (give them max 30 seconds to shut down)
 931     my $previous_alarm = alarm(30);
 932     eval {
 933         local $SIG{ALRM} = sub { die "Timed Out!\n" };
 934
 935         my $pid;
 936         1 while ((($pid = waitpid (-1, 0)) > 0) || ($! == EINTR));
 937
 938         alarm(0); # avoid race
 939     };
 940     alarm ($previous_alarm);
 941 }
 942
 943 # initialize mime system before fork
 944 xdg_mime_get_mime_type_for_file ($0);
 945
 946 my $server = bless {
 947     server => $server_attr,
 948 };
 949
 950 if (!$opt_testmode) {
 951     $server->run ();
 952 } else {
 953     if (fork) {
 954         $server->run();
 955     } else {
 956
 957         my $sender ='sender@pmg.example';
 958         my $targets = [
 959             'target1@pmg.example',
 960             'target2@pmg.example',
 961             'target3@pmg.example',
 962         ];
 963
 964         my $smtp;
 965         while (!$smtp) {
 966             $smtp = Net::SMTP->new ('127.0.0.1', Port => 10023);
 967             last if $smtp;
 968             usleep(10);
 969         }
 970
 971         # syslog ('info', "connected to " . $smtp->domain);
 972
 973         $smtp->mail ($sender);
 974         $smtp->to (@$targets);
 975
 976         $smtp->data();
 977
 978         open (TMP, $opt_testmode) ||
 979             die "unable to upen file '$opt_testmode' - $! :ERROR";
 980         while (<TMP>) {
 981             $smtp->datasend ($_);
 982         }
 983         close TMP;
 984
 985         $smtp->datasend ("\n");
 986         $smtp->dataend ();
 987
 988         $smtp->quit;
 989     }
 990 }
 991
 992 exit (0);
 993
 994 __END__
 995
 996 =head1 NAME
 997
 998 pmg-smtp-filter - the Proxmox mail filter
 999
1000 =head1 SYNOPSIS
1001
1002 pmg-smtp-filter [-u] [-t testfile]
1003
1004 =head1 DESCRIPTION
1005
1006 Documentation is available at www.proxmox.com