]>
git.proxmox.com Git - pmg-api.git/blob - bin/pmgpolicy
6 use POSIX
qw(errno_h signal_h);
8 use Net
::Server
::PreForkSimple
;
9 use Net
::DNS
::Resolver
;
14 use Time
::HiRes
qw(gettimeofday);
25 use PMG
::ClusterConfig
;
27 use base
qw(Net::Server::PreForkSimple);
29 my $greylist_delay = 3*60; # greylist window
30 my $greylist_lifetime = 3600*24*2; # retry window
31 my $greylist_awlifetime = 3600*24*36; # expire window
33 my $opt_commandline = [$0, @ARGV];
34 my $opt_policy_port = 10022;
35 my $opt_max_dequeue = 1;
36 my $opt_dequeue_time = 60*2;
42 if (!GetOptions
('pidfile=s' => \
$opt_pidfile,
43 'testmode' => \
$opt_testmode,
44 'database=s' => \
$opt_database)) {
49 $opt_pidfile = "/var/run/pmgpolicy.pid" if !$opt_pidfile;
50 $opt_max_dequeue = 0 if $opt_testmode;
52 initlog
('pmgpolicy', 'mail');
58 my $pmg_cfg = PMG
::Config-
>new ();
59 my $demo = $pmg_cfg->get('admin', 'demo');
60 $max_servers = $pmg_cfg->get('mail', 'max_policy');
63 syslog
('info', 'demo mode detected - not starting server');
69 if (defined ($ENV{BOUND_SOCKETS
})) {
75 port
=> [ $opt_policy_port ],
77 max_servers
=> $max_servers,
78 max_dequeue
=> $opt_max_dequeue,
79 check_for_dequeue
=> $opt_dequeue_time,
81 pid_file
=> $opt_pidfile,
82 commandline
=> $opt_commandline,
83 no_close_by_child
=> 1,
88 if (defined($opt_database)) {
89 $database = $opt_database;
91 $database = "Proxmox_ruledb";
94 $SIG{'__WARN__'} = sub {
98 syslog
('warning', "WARNING: %s", $t);
105 $self->log(2, "starting greylist database maintainance");
107 my ($csec, $usec) = gettimeofday
();
109 my $cinfo = PMG
::ClusterConfig-
>new();
110 my $lcid = $cinfo->{local}->{cid
};
111 my $role = $cinfo->{local}->{type
} // '-';
116 $dbh = PMG
::DBTools
::open_ruledb
($database);
121 $self->log(0, "ERROR: $err");
133 # we do not lock the table here to avoid delays
134 # but that is OK, because we only touch expired records
135 # which do not change nornmally
136 ## $dbh->do ("LOCK TABLE CGreylist IN ROW EXCLUSIVE MODE");
138 # move expired and undelivered records from Greylist to Statistic
142 $rntxt = "AND CID = 0";
144 if ($role eq 'master') {
145 # master is responsible for all non-cluster (deleted) nodes
146 foreach my $rcid (@{$cinfo->{remnodes
}}) {
147 $rntxt .= $rntxt ?
" AND CID != $rcid" : "AND (CID != $rcid";
149 $rntxt .= ")" if $rntxt;
151 $rntxt = "AND (CID = 0 OR CID = $lcid)";
158 my $sth = $dbh->prepare(
159 "SELECT distinct instance, sender FROM CGreylist " .
160 "WHERE passed = 0 AND extime < ? $rntxt");
162 $sth->execute ($now);
165 while (my $ref = $sth->fetchrow_hashref()) {
166 my $sth2 = $dbh->prepare(
167 "SELECT * FROM CGreylist WHERE instance = ? AND sender = ?");
168 $sth2->execute ($ref->{instance
}, $ref->{sender
});
173 while (my $ref2 = $sth2->fetchrow_hashref()) {
174 $rctime = $ref2->{rctime
} if !$rctime;
175 $bc += $ref2->{blocked
};
176 push @rcvrs, $ref2->{receiver
};
181 # hack: sometimes query sth2 does not return anything - maybe a
182 # postgres bug? We simply ignore (when rctime is undefined) it
186 $cmds .= "SELECT nextval ('cstatistic_id_seq');" .
187 "INSERT INTO CStatistic " .
188 "(CID, RID, ID, Time, Bytes, Direction, Spamlevel, VirusInfo, PTime, Sender) VALUES (" .
189 "$lcid, currval ('cstatistic_id_seq'), currval ('cstatistic_id_seq'), ";
191 my $sl = $bc >= 100000 ?
4 : 5;
192 $cmds .= $rctime . ", 0, '1', $sl, NULL, 0, ";
193 $cmds .= $dbh->quote ($ref->{sender
}) . ');';
195 foreach my $r (@rcvrs) {
196 my $tmp = $dbh->quote ($r);
197 $cmds .= "INSERT INTO CReceivers (CStatistic_CID, CStatistic_RID, Receiver, Blocked) ".
198 "VALUES ($lcid, currval ('cstatistic_id_seq'), $tmp, '1'); ";
202 if (length ($cmds) > 100000) {
209 # this produces too much log traffic
210 # my $targets = join (", ", @rcvrs);
211 #my $msg = "expire mail $ref->{instance} from $ref->{sender} to $targets";
212 #$self->log (0, $msg);
215 $dbh->do ($cmds) if $cmds;
220 my $msg = "found $ecount expired mails in greylisting database";
221 $self->log (0, $msg);
224 $dbh->do ("DELETE FROM CGreylist WHERE extime < $now");
231 my ($csec_end, $usec_end) = gettimeofday
();
232 my $ptime = int (($csec_end-$csec)*1000 + ($usec_end - $usec)/1000);
235 $dbh->rollback if $dbh;
238 $self->log(2, "end greylist database maintainance ($ptime ms)");
241 $dbh->disconnect() if $dbh;
247 my $prop = $self->{server
};
249 $prop->{log_level
} = 3;
251 $self->log(0, "Policy daemon (re)started");
254 # reloading server configuration
255 if (defined $prop->{children
}) {
256 foreach my $pid (keys %{$prop->{children
}}) {
257 kill(10, $pid); # SIGUSR1 childs
262 my $sig_set = POSIX
::SigSet-
>new;
263 $sig_set->addset (&POSIX
::SIGHUP
);
264 $sig_set->addset (&POSIX
::SIGCHLD
);
265 my $old_sig_set = POSIX
::SigSet-
>new();
267 sigprocmask
(SIG_UNBLOCK
, $sig_set, $old_sig_set);
273 my $prop = $self->{server
};
275 if ($self->{ruledb
}) {
276 $self->log(0, "reloading configuration $database");
277 $self->{ruledb
}->close();
280 my $pmg_cfg = PMG
::Config-
>new ();
281 $self->{use_spf
} = $pmg_cfg->get('mail', 'spf');
282 $self->{use_greylist
} = $pmg_cfg->get('mail', 'greylist');
285 $self->{use_spf
} = 1;
286 $self->{use_greylist
} = 1;
289 my $nodename = PVE
::INotify
::nodename
();
290 $self->{fqdn
} = PVE
::Tools
::get_fqdn
($nodename);
292 my $cinfo = PMG
::ClusterConfig-
>new();
293 my $lcid = $cinfo->{local}->{cid
};
294 $self->{cinfo
} = $cinfo;
295 $self->{lcid
} = $lcid;
300 $dbh = PMG
::DBTools
::open_ruledb
($database);
301 $self->{ruledb
} = PMG
::RuleDB-
>new($dbh);
302 $self->{rulecache
} = PMG
::RuleCache-
>new($self->{ruledb
});
305 $self->log(0, "ERROR: unable to load database : $err");
308 $self->{reload_config
} = 0;
311 sub child_init_hook
{
314 my $prop = $self->{server
};
316 $0 = 'pmgpolicy child';
318 setup_fork_signal_mask
(0); # unblocking signals for children
321 $self->load_config();
323 $self->{mux
} = IO
::Multiplex-
>new();
324 $self->{mux
}->set_callback_object($self);
336 # $dnsargs{nameservers} = [ qw (213.129.232.1 213.129.226.2) ];
339 $self->{dns_resolver
} = Net
::DNS
::Resolver-
>new(%dnsargs);
341 $self->{spf_server
} = Mail
::SPF
::Server-
>new(
342 hostname
=> $self->{fqdn
}, dns_resolver
=> $self->{dns_resolver
});
346 $self->child_finish_hook;
351 $self->{reload_config
} = 1;
355 sub child_finish_hook
{
358 my $prop = $self->{server
};
360 $self->{ruledb
}->close() if $self->{ruledb
};
364 my ($self, $instance, $ip, $helo, $sender) = @_;
371 # we only use helo tests when we have no sender,
372 # helo is sometimes empty, so we cant use SPF helo tests
373 # in that case - strange
374 if ($helo && !$sender) {
377 if (defined ($self->{cache
}->{$instance}) &&
378 defined ($self->{cache
}->{$instance}->{spf_helo_result
})) {
380 $query = $self->{cache
}->{$instance}->{spf_helo_result
};
383 my $request = Mail
::SPF
::Request-
>new(
384 scope
=> 'helo', identity
=> $helo, ip_address
=> $ip);
386 $query = $self->{cache
}->{$instance}->{spf_helo_result
} =
387 $self->{spf_server
}->process ($request);
390 $result = $query->code;
391 $spf_header = $query->received_spf_header;
392 $local_expl = $query->local_explanation;
393 $auth_expl = $query->authority_explanation if $query->is_code('fail');
395 # return if we get a definitive result
396 if ($result eq 'pass' || $result eq 'fail' || $result eq 'temperror') {
397 return ($result, $spf_header, $local_expl, $auth_expl);
405 if (defined ($self->{cache
}->{$instance}) &&
406 defined ($self->{cache
}->{$instance}->{spf_mfrom_result
})) {
408 $query = $self->{cache
}->{$instance}->{spf_mfrom_result
};
412 my $request = Mail
::SPF
::Request-
>new(
413 scope
=> 'mfrom', identity
=> $sender,
414 ip_address
=> $ip, helo_identity
=> $helo);
416 $query = $self->{cache
}->{$instance}->{spf_mfrom_result
} =
417 $self->{spf_server
}->process($request);
420 $result = $query->code;
421 $spf_header = $query->received_spf_header;
422 $local_expl = $query->local_explanation;
423 $auth_expl = $query->authority_explanation if $query->is_code('fail');
425 return ($result, $spf_header, $local_expl, $auth_expl);
432 my ($self, $ip, $receiver) = @_;
434 my ($rdomain) = $receiver =~ /([^@]+)$/;
436 my $dkey = "BKMX:$rdomain";
438 if (defined ($self->{cache
}->{$dkey}) &&
439 ($self->{cache
}->{$dkey}->{status
} == 1)) {
440 return $self->{cache
}->{$dkey}->{$ip};
443 my $resolver = $self->{dns_resolver
};
445 if (my $mx = $resolver->send($rdomain, 'MX')) {
446 $self->{cache
}->{$dkey}->{status
} = 1;
447 my @mxa = grep { $_->type eq 'MX' } $mx->answer;
448 my @mxl = sort { $a->preference <=> $b->preference } @mxa;
449 # shift @mxl; # optionaly skip primary MX ?
450 foreach my $rr (@mxl) {
451 my $a = $resolver->send ($rr->exchange, 'A');
453 foreach my $rra ($a->answer) {
454 if ($rra->type eq 'A') {
455 $self->{cache
}->{$dkey}->{$rra->address} = 1;
461 $self->{cache
}->{$dkey}->{status
} = 0;
464 return $self->{cache
}->{$dkey}->{$ip};
468 my ($self, $ctime, $helo, $ip, $sender, $rcpt, $instance) = @_;
470 my $rulecache = $self->{rulecache
};
472 my $dbh = $self->{ruledb
}->{dbh
};
474 # try to reconnect if database connection is broken
476 $self->log(0, 'Database connection broken - trying to reconnect');
479 $dbh = PMG
::DBTools
::open_ruledb
($database);
483 $self->log(0, "unable to reconnect to database server: $err");
486 $self->{ruledb
} = PMG
::RuleDB-
>new($dbh);
489 # some sender substitutions
490 my ($user, $domain) = split('@', $sender, 2);
491 if (defined ($user) && defined ($domain)) {
492 # see http://cr.yp.to/proto/verp.txt
493 $user =~ s/\+.*//; # strip extensions (mailing-list VERP)
494 $user =~ s/\b\d+\b/#/g; #replace nubmers in VERP address
495 $sender = "$user\@$domain";
498 if ($self->is_backup_mx($ip, $rcpt)) {
499 $self->log(3, "accept mails from backup MX host - $ip");
503 # greylist exclusion (sender whitelist)
504 if ($rulecache->greylist_match ($sender, $ip)) {
505 $self->log(3, "accept mails from whitelist - $ip");
509 # greylist exclusion (receiver whitelist)
510 if ($rulecache->greylist_match_receiver ($rcpt)) {
511 $self->log(3, "accept mails to whitelist - <$rcpt>");
515 my ($net, $host) = $ip =~ m/(\d+\.\d+\.\d+)\.(\d+)/;
518 if ((!$opt_testmode && $self->{use_spf
}) ||
519 ($opt_testmode && ($rcpt =~ m/^testspf/))) {
528 my ($result, $smtp_comment, $header_comment);
531 $previous_alarm = alarm(10);
532 local $SIG{ALRM
} = sub { die "SPF timeout\n" };
534 ($result, $spf_header, $local_expl, $auth_expl) =
535 $self->get_spf_result($instance, $ip, $helo, $sender);
537 alarm(0); # avoid race condition
541 alarm($previous_alarm) if defined($previous_alarm);
544 $err = $err->text if UNIVERSAL
::isa
($err, 'Mail::SPF::Exception');
545 $self->log (0, $err);
548 if ($result && $result eq 'pass') {
549 $self->log(3, "SPF says $result");
550 $spf_result = $spf_header ?
"prepend $spf_header" : 'dunno';
553 if ($result && $result eq 'fail') {
554 $self->log(3, "SPF says $result");
555 $spf_result = "reject ${auth_expl}";
561 # try to avoid locks everywhere - we use merge instead of insert
562 #$dbh->do ("LOCK TABLE CGreylist IN ROW EXCLUSIVE MODE");
564 # check if there is already a record in the GL database
565 my $sth = $dbh->prepare(
566 "SELECT * FROM CGreylist " .
567 "where IPNet = ? AND Sender = ? AND Receiver = ?");
569 $sth->execute($net, $sender, $rcpt);
570 my $ref = $sth->fetchrow_hashref();
573 # else add an entry to the GL Database with short
574 # expiration time. run_dequeue() moves those entries into the statistic
575 # table later. We set 'blocked' to 100000 to identify those entries.
577 if (!defined($ref->{rctime
})) {
579 $dbh->do($PMG::DBTools
::cgreylist_merge_sql
, undef,
580 $net, $host, $sender, $rcpt, $instance,
581 $ctime, $ctime + 10, 0, 100000, 0, $ctime, $self->{lcid
});
593 return $spf_result if $spf_result;
597 my $res = $spf_header ?
"prepend $spf_header" : 'dunno';
599 return $res if !$self->{use_greylist
};
601 my $defer_res = "defer_if_permit Service is unavailable (try later)";
605 # we dont use alarm here, because it does not work with DBI
609 # try to avoid locks everywhere - we use merge instead of insert
610 #$dbh->do ("LOCK TABLE CGreylist IN ROW EXCLUSIVE MODE");
612 my $sth = $dbh->prepare(
613 "SELECT * FROM CGreylist " .
614 "where IPNet = ? AND Sender = ? AND Receiver = ?");
616 $sth->execute($net, $sender, $rcpt);
618 my $ref = $sth->fetchrow_hashref();
622 if (!defined($ref->{rctime
})) {
624 $dbh->do($PMG::DBTools
::cgreylist_merge_sql
, undef,
625 $net, $host, $sender, $rcpt, $instance,
626 $ctime, $ctime + $greylist_lifetime, 0, 1, 0, $ctime, $self->{lcid
});
629 $self->log(3, "defer greylisted mail");
631 my $age = $ctime - $ref->{rctime
};
633 if ($age < $greylist_delay) {
634 # defer (resent within greylist_delay window)
636 $self->log(3, "defer greylisted mail");
637 $dbh->do("UPDATE CGreylist " .
638 "SET Blocked = Blocked + 1, Host = ?, MTime = ? " .
639 "WHERE IPNet = ? AND Sender = ? AND Receiver = ?", undef,
640 $host, $ctime, $net, $sender, $rcpt);
642 if ($ctime < $ref->{extime
}) {
643 # accept (not expired)
644 my $lifetime = $sender eq "" ?
0 : $greylist_awlifetime;
645 my $delay = $ref->{passed
} ?
"" : "Delay = $age, ";
646 $dbh->do("UPDATE CGreylist " .
647 "SET Passed = Passed + 1, $delay Host = ?, ExTime = ?, MTime = ? " .
648 "WHERE IPNet = ? AND Sender = ? AND Receiver = ?", undef,
649 $host, $ctime + $lifetime, $ctime, $net, $sender, $rcpt);
651 # defer (record is expired)
653 $dbh->do("UPDATE CGreylist " .
654 "SET Host = ?, RCTime = ?, ExTime = ?, MTime = ?, Instance = ?, " .
655 "Blocked = 1, Passed = 0 " .
656 "WHERE IPNet = ? AND Sender = ? AND Receiver = ?", undef,
657 $host, $ctime, $ctime + $greylist_lifetime, $ctime, $instance,
658 $net, $sender, $rcpt);
667 $self->log (0, $err);
673 # shutdown connections: we need this - else file handles are
674 # not closed and we run out of handles
676 my ($self, $mux, $fh) = @_;
678 $mux->shutdown($fh, 1);
682 my ($self, $mux, $fh, $dataref) = @_;
683 my $prop = $self->{server
};
688 $self->load_config() if $self->{reload_config
};
690 while ($$dataref =~ s/^([^\r\n]*)\r?\n//) {
692 next if !defined ($line);
694 if ($line =~ m/([^=]+)=(.*)/) {
695 $attribute->{substr($1, 0, 255)} = substr($2, 0, 255);
696 } elsif ($line eq '') {
701 die "undefined test time :ERROR" if !defined $attribute->{testtime
};
702 $ctime = $attribute->{testtime
};
705 if ($attribute->{instance
} && $attribute->{recipient
} &&
706 $attribute->{client_address
} && $attribute->{request
} &&
707 $attribute->{request
} eq 'smtpd_access_policy') {
711 $res = $self->greylist_value(
713 lc ($attribute->{helo_name
}),
714 lc ($attribute->{client_address
}),
715 lc ($attribute->{sender
}),
716 lc ($attribute->{recipient
}),
717 lc ($attribute->{instance
}));
724 print $fh "action=$res\n\n";
728 $self->log(0, "greylist policy protocol error - got '%s'", $line);
734 # remove remaining data, if any
735 if ($$dataref ne '') {
736 $self->log(0, "greylist policy protocol error - unused data '%s'", $$dataref);
740 $self->log(0, $err) if $err;
743 sub restart_close_hook
{
746 my $sig_set = POSIX
::SigSet-
>new;
747 $sig_set->addset(&POSIX
::SIGHUP
);
748 $sig_set->addset(&POSIX
::SIGCHLD
); # to avoid zombies
749 my $old_sig_set = POSIX
::SigSet-
>new();
751 sigprocmask
(SIG_BLOCK
, $sig_set, $old_sig_set);
754 sub pre_server_close_hook
{
757 my $prop = $self->{server
};
759 if (defined $prop->{_HUP
}) {
760 undef $prop->{pid_file_unlink
};
763 if (defined $prop->{children
}) {
764 foreach my $pid (keys %{$prop->{children
}}) {
765 kill(1, $pid); # HUP childs
769 # nicely shutdown childs (give them max 30 seconds to shut down)
770 my $previous_alarm = alarm(30);
772 local $SIG{ALRM
} = sub { die "Timed Out!\n" };
775 1 while ((($pid = waitpid(-1, 0)) > 0) || ($! == EINTR
));
777 alarm(0); # avoid race
779 alarm ($previous_alarm);
782 sub setup_fork_signal_mask
{
785 my $sig_set = POSIX
::SigSet-
>new;
786 $sig_set->addset(&POSIX
::SIGINT
);
787 $sig_set->addset(&POSIX
::SIGTERM
);
788 $sig_set->addset(&POSIX
::SIGQUIT
);
789 $sig_set->addset(&POSIX
::SIGHUP
);
790 my $old_sig_set = POSIX
::SigSet-
>new();
793 sigprocmask
(SIG_BLOCK
, $sig_set, $old_sig_set);
795 sigprocmask
(SIG_UNBLOCK
, $sig_set, $old_sig_set);
799 # subroutine to start up a specified number of children.
800 # We need to block signals until handlers are set up correctly.
801 # Else its possible that HUP occurs after fork, which triggers
802 # singal TERM at childs and calls server_close() instead of
803 # simply exit the child.
804 # Note: on server startup signals are setup to trigger
805 # asynchronously for a short period of time (in PreForkSimple]::loop,
806 # run_n_children is called before run_parent)
807 # Net::Server::PreFork does not have this problem, because it is using
808 # signal HUP stop children
812 my $prop = $self->{server
};
814 setup_fork_signal_mask
(1); # block signals
816 $self->SUPER::run_n_children
($n);
818 setup_fork_signal_mask
(0); # unblocking signals for parent
821 # test sig_hup with: for ((;;)) ;do kill -HUP `cat /var/run/pmgpolicy.pid`; done;
822 # wrapper to avoid multiple calls to sig_hup
826 my $prop = $self->{server
};
828 return if defined($prop->{_HUP
}); # do not call twice
830 $self->SUPER::sig_hup
();
833 ### child process which will accept on the port
837 my $prop = $self->{server
};
839 $self->log(4, "Child Preforked ($$)\n");
841 # set correct signal handlers before enabling signals again
842 $SIG{INT
} = $SIG{TERM
} = $SIG{QUIT
} = $SIG{HUP
} = sub {
843 $self->child_finish_hook;
847 delete $prop->{children
};
849 $self->child_init_hook;
853 my $sock = $prop->{sock
}->[0];
855 # make sure we got a good sock
856 if (!defined ($sock)){
857 $self->log(0, "ERROR: Received a bad socket");
861 # sometimes the socket is not usable, don't know why
862 my $flags = fcntl($sock, F_GETFL
, 0);
864 $self->log(0, "socket not ready - $!");
868 # cache is limited, because postfix does max. 100 queries
872 my $mux = $self->{mux
};
873 $mux->listen ($sock);
877 $self->log(0, "ERROR: $err");
880 $self->child_finish_hook;
894 my ($self, $level, $msg, @therest) = @_;
896 my $prop = $self->{server
};
898 return if $level =~ /^\d+$/ && $level > $prop->{log_level
};
900 $level = $syslog_map->{$level} || $level;
902 syslog
($level, $msg, @therest);
904 syslog
($level, $msg);
909 server
=> $server_attr,
912 $server->sig_chld(); # avoid zombies after restart
922 pmgpolicy - The Proxmox policy daemon
930 Documentation is available at www.proxmox.com