X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=PMG%2FUtils.pm;h=473a88fc7fae67dc22979c4e50dda66c9a239a6a;hb=89d4d1555a454be5cdb00122cb980b2a01a6a3d7;hp=020141a9504f873de007872a15e2c3170f1e3131;hpb=5d5af7c56b95dd6dcf2857e175989de62ad8833b;p=pmg-api.git diff --git a/PMG/Utils.pm b/PMG/Utils.pm index 020141a..473a88f 100644 --- a/PMG/Utils.pm +++ b/PMG/Utils.pm @@ -23,6 +23,9 @@ use Socket; use RRDs; use Filesys::Df; use Encode; +use utf8; +no utf8; + use HTML::Entities; use PVE::ProcFSTools; @@ -34,22 +37,12 @@ use PMG::AtomicFile; use PMG::MailQueue; use PMG::SMTPPrinter; -my $realm_regex = qr/[A-Za-z][A-Za-z0-9\.\-_]+/; - -PVE::JSONSchema::register_format('pmg-realm', \&verify_realm); -sub verify_realm { - my ($realm, $noerr) = @_; - - if ($realm !~ m/^${realm_regex}$/) { - return undef if $noerr; - die "value does not look like a valid realm\n"; - } - return $realm; -} +my $valid_pmg_realms = ['pam', 'pmg', 'quarantine']; PVE::JSONSchema::register_standard_option('realm', { description => "Authentication domain ID", - type => 'string', format => 'pmg-realm', + type => 'string', + enum => $valid_pmg_realms, maxLength => 32, }); @@ -87,7 +80,8 @@ sub verify_username { # colon separated lists)! # slash is not allowed because it is used as pve API delimiter # also see "man useradd" - if ($username =~ m!^([^\s:/]+)\@(${realm_regex})$!) { + my $realm_list = join('|', @$valid_pmg_realms); + if ($username =~ m!^([^\s:/]+)\@(${realm_list})$!) { return wantarray ? ($username, $1, $2) : $username; } @@ -114,11 +108,18 @@ PVE::JSONSchema::register_standard_option('username', { PVE::JSONSchema::register_standard_option('pmg-email-address', { description => "Email Address (allow most characters).", type => 'string', - pattern => '(?:|[^\s\/\@]+\@[^\s\/\@]+)', + pattern => '(?:[^\s\/\\\@]+\@[^\s\/\\\@]+)', maxLength => 512, minLength => 3, }); +PVE::JSONSchema::register_standard_option('pmg-whiteblacklist-entry-list', { + description => "White/Blacklist entry list (allow most characters). Can contain globs", + type => 'string', + pattern => '(?:[^\s\/\\\;\,]+)(?:\,[^\s\/\\\;\,]+)*', + minLength => 3, +}); + sub lastid { my ($dbh, $seq) = @_; @@ -217,16 +218,42 @@ sub reinject_mail { } } - if (!$smtp->mail($sender)) { + my $has_utf8_targets = 0; + foreach my $target (@$targets) { + if (utf8::is_utf8($target)) { + $has_utf8_targets = 1; + last; + } + } + + my $mail_opts = " BODY=8BITMIME"; + my $sender_addr; + if (utf8::is_utf8($sender)) { + $sender_addr = encode('UTF-8', $smtp->_addr($sender)); + $mail_opts .= " SMTPUTF8"; + } else { + $sender_addr = $smtp->_addr($sender); + $mail_opts .= " SMTPUTF8" if $has_utf8_targets; + } + + if (!$smtp->_MAIL("FROM:" . $sender_addr . $mail_opts)) { syslog('err', "smtp error - got: %s %s", $smtp->code, scalar ($smtp->message)); die "smtp from: ERROR"; } - my $dsnopts = $nodsn ? {Notify => ['NEVER']} : {}; + my $rcpt_opts = $nodsn ? " NOTIFY=NEVER" : ""; - if (!$smtp->to (@$targets, $dsnopts)) { - syslog ('err', "smtp error - got: %s %s", $smtp->code, scalar($smtp->message)); - die "smtp to: ERROR"; + foreach my $target (@$targets) { + my $rcpt_addr; + if (utf8::is_utf8($target)) { + $rcpt_addr = encode('UTF-8', $smtp->_addr($target)); + } else { + $rcpt_addr = $smtp->_addr($target); + } + if (!$smtp->_RCPT("TO:" . $rcpt_addr . $rcpt_opts)) { + syslog ('err', "smtp error - got: %s %s", $smtp->code, scalar($smtp->message)); + die "smtp to: ERROR"; + } } # Output the head: @@ -266,6 +293,83 @@ sub reinject_mail { return wantarray ? ($resid, $rescode, $resmess) : $resid; } +sub analyze_custom_check { + my ($queue, $dname, $pmg_cfg) = @_; + + my $enable_custom_check = $pmg_cfg->get('admin', 'custom_check'); + return undef if !$enable_custom_check; + + my $timeout = 60*5; + my $customcheck_exe = $pmg_cfg->get('admin', 'custom_check_path'); + my $customcheck_apiver = 'v1'; + my ($csec, $usec) = gettimeofday(); + + my $vinfo; + my $spam_score; + + eval { + + my $log_err = sub { + my ($errmsg) = @_; + $errmsg =~ s/%/%%/; + syslog('err', $errmsg); + }; + + my $customcheck_output_apiver; + my $have_result; + my $parser = sub { + my ($line) = @_; + + my $result_flag; + if ($line =~ /^v\d$/) { + die "api version already defined!\n" if defined($customcheck_output_apiver); + $customcheck_output_apiver = $line; + die "api version mismatch - expected $customcheck_apiver, got $customcheck_output_apiver !\n" + if ($customcheck_output_apiver ne $customcheck_apiver); + } elsif ($line =~ /^SCORE: (-?[0-9]+|.[0-9]+|[0-9]+.[0-9]+)$/) { + $spam_score = $1; + $result_flag = 1; + } elsif ($line =~ /^VIRUS: (.+)$/) { + $vinfo = $1; + $result_flag = 1; + } elsif ($line =~ /^OK$/) { + $result_flag = 1; + } else { + die "got unexpected output!\n"; + } + die "got more than 1 result outputs\n" if ( $have_result && $result_flag); + $have_result = $result_flag; + }; + + PVE::Tools::run_command([$customcheck_exe, $customcheck_apiver, $dname], + errmsg => "$queue->{logid} custom check error", + errfunc => $log_err, outfunc => $parser, timeout => $timeout); + + die "no api version returned\n" if !defined($customcheck_output_apiver); + die "no result output!\n" if !$have_result; + }; + my $err = $@; + + if ($vinfo) { + syslog('info', "$queue->{logid}: virus detected: $vinfo (custom)"); + } + + my ($csec_end, $usec_end) = gettimeofday(); + $queue->{ptime_custom} = + int (($csec_end-$csec)*1000 + ($usec_end - $usec)/1000); + + if ($err) { + syslog ('err', $err); + $vinfo = undef; + $queue->{errors} = 1; + } + + $queue->{vinfo_custom} = $vinfo; + $queue->{spam_custom} = $spam_score; + + return ($vinfo, $spam_score); +} + sub analyze_virus_clam { my ($queue, $dname, $pmg_cfg) = @_; @@ -336,13 +440,96 @@ sub analyze_virus_clam { return $vinfo ? "$vinfo (clamav)" : undef; } +sub analyze_virus_avast { + my ($queue, $dname, $pmg_cfg) = @_; + + my $timeout = 60*5; + my $vinfo; + + my ($csec, $usec) = gettimeofday(); + + my $previous_alarm; + + eval { + + $previous_alarm = alarm($timeout); + + $SIG{ALRM} = sub { + die "$queue->{logid}: Maximum time ($timeout sec) exceeded. " . + "virus analyze (avast) failed: ERROR"; + }; + + open(my $cmd, '-|', '/bin/scan', $dname) || + die "$queue->{logid}: can't exec avast scan: $! : ERROR"; + + my $response = ''; + while (defined(my $line = <$cmd>)) { + if ($line =~ m/^$dname\s+(.*\S)\s*$/) { + # we just use the first detected virus name + $vinfo = $1 if !$vinfo; + } + + $response .= $line; + } + + close($cmd); + + alarm(0); # avoid race conditions + + if ($vinfo) { + syslog('info', "$queue->{logid}: virus detected: $vinfo (avast)"); + } + }; + my $err = $@; + + alarm($previous_alarm); + + my ($csec_end, $usec_end) = gettimeofday(); + $queue->{ptime_clam} = + int (($csec_end-$csec)*1000 + ($usec_end - $usec)/1000); + + if ($err) { + syslog ('err', $err); + $vinfo = undef; + $queue->{errors} = 1; + } + + return undef if !$vinfo; + + $queue->{vinfo_avast} = $vinfo; + + return "$vinfo (avast)"; +} + sub analyze_virus { my ($queue, $filename, $pmg_cfg, $testmode) = @_; # TODO: support other virus scanners? - # always scan with clamav - return analyze_virus_clam($queue, $filename, $pmg_cfg); + if ($testmode) { + my $vinfo_clam = analyze_virus_clam($queue, $filename, $pmg_cfg); + my $vinfo_avast = analyze_virus_avast($queue, $filename, $pmg_cfg); + + return $vinfo_avast || $vinfo_clam; + } + + my $enable_avast = $pmg_cfg->get('admin', 'avast'); + + if ($enable_avast) { + if (my $vinfo = analyze_virus_avast($queue, $filename, $pmg_cfg)) { + return $vinfo; + } + } + + my $enable_clamav = $pmg_cfg->get('admin', 'clamav'); + + if ($enable_clamav) { + if (my $vinfo = analyze_virus_clam($queue, $filename, $pmg_cfg)) { + return $vinfo; + } + } + + return undef; } sub magic_mime_type_for_file { @@ -480,7 +667,7 @@ __EOD__ } sub find_local_network_for_ip { - my ($ip) = @_; + my ($ip, $noerr) = @_; my $testip = Net::IP->new($ip); @@ -508,6 +695,8 @@ sub find_local_network_for_ip { } } + return undef if $noerr; + die "unable to detect local network for ip '$ip'\n"; } @@ -540,6 +729,9 @@ sub get_full_service_state { return $res; } +our $db_service_list = [ + 'pmgpolicy', 'pmgmirror', 'pmgtunnel', 'pmg-smtp-filter' ]; + sub service_wait_stopped { my ($timeout, $service_list) = @_; @@ -576,14 +768,14 @@ sub service_cmd { my ($service, $cmd) = @_; die "unknown service command '$cmd'\n" - if $cmd !~ m/^(start|stop|restart|reload)$/; + if $cmd !~ m/^(start|stop|restart|reload|reload-or-restart)$/; if ($service eq 'pmgdaemon' || $service eq 'pmgproxy') { - if ($cmd eq 'restart') { - # OK - } else { - die "invalid service cmd '$service $cmd': ERROR"; - } + die "invalid service cmd '$service $cmd': ERROR" if $cmd eq 'stop'; + } elsif ($service eq 'fetchmail') { + # use restart instead of start - else it does not start 'exited' unit + # after setting START_DAEMON=yes in /etc/default/fetchmail + $cmd = 'restart' if $cmd eq 'start'; } $service = $service_aliases->{$service} // $service; @@ -622,11 +814,13 @@ sub run_postmap { # make sure the file exists (else postmap fails) IO::File->new($filename, 'a', 0644); - my $age_src = -M $filename // 0; - my $age_dst = -M "$filename.db" // 10000000000; + my $mtime_src = (CORE::stat($filename))[9] // + die "unbale to read mtime of $filename\n"; + + my $mtime_dst = (CORE::stat("$filename.db"))[9] // 0; # if not changed, do nothing - return if $age_src > $age_dst; + return if $mtime_src <= $mtime_dst; eval { PVE::Tools::run_command( @@ -796,7 +990,7 @@ sub update_node_status_rrd { my $netin = 0; my $netout = 0; foreach my $dev (keys %$netdev) { - next if $dev !~ m/^eth\d+$/; + next if $dev !~ m/^$PVE::Network::PHYSICAL_NIC_RE$/; $netin += $netdev->{$dev}->{receive}; $netout += $netdev->{$dev}->{transmit}; } @@ -972,10 +1166,13 @@ sub bencode_header { } sub load_sa_descriptions { + my ($additional_dirs) = @_; my @dirs = ('/usr/share/spamassassin', '/usr/share/spamassassin-extra'); + push @dirs, @$additional_dirs if @$additional_dirs; + my $res = {}; my $parse_sa_file = sub { @@ -1003,6 +1200,8 @@ sub load_sa_descriptions { } } + $res->{'ClamAVHeuristics'}->{desc} = "ClamAV heuristic tests"; + return $res; } @@ -1092,9 +1291,13 @@ sub scan_journal_for_rbl_rejects { # example postscreen log entry for RBL rejects # Aug 29 08:00:36 proxmox postfix/postscreen[11266]: NOQUEUE: reject: RCPT from [x.x.x.x]:1234: 550 5.7.1 Service unavailable; client [x.x.x.x] blocked using zen.spamhaus.org; from=, to=, proto=ESMTP, helo= + # example for PREGREET reject + # Dec 7 06:57:11 proxmox postfix/postscreen[32084]: PREGREET 14 after 0.23 from [x.x.x.x]:63492: EHLO yyyyy\r\n + my $identifier = 'postfix/postscreen'; - my $count = 0; + my $rbl_count = 0; + my $pregreet_count = 0; my $parser = sub { my $line = shift; @@ -1104,8 +1307,11 @@ sub scan_journal_for_rbl_rejects { return; } - return if $line !~ m/\s$identifier\[\d+\]:\sNOQUEUE:\sreject:.*550 5.7.1 Service unavailable;/; - $count++; + if ($line =~ m/\s$identifier\[\d+\]:\sNOQUEUE:\sreject:.*550 5.7.1 Service unavailable;/) { + $rbl_count++; + } elsif ($line =~ m/\s$identifier\[\d+\]:\sPREGREET\s\d+\safter\s/) { + $pregreet_count++; + } }; # limit to last 5000 lines to avoid long delays @@ -1120,7 +1326,7 @@ sub scan_journal_for_rbl_rejects { PVE::Tools::run_command($cmd, outfunc => $parser); - return $count; + return ($rbl_count, $pregreet_count); } my $hwaddress; @@ -1136,4 +1342,40 @@ sub get_hwaddress { return $hwaddress; } +my $default_locale = "en_US.UTF-8 UTF-8"; + +sub cond_add_default_locale { + + my $filename = "/etc/locale.gen"; + + open(my $infh, "<", $filename) || return; + + while (defined(my $line = <$infh>)) { + if ($line =~ m/^\Q${default_locale}\E/) { + # already configured + return; + } + } + + seek($infh, 0, 0) // return; # seek failed + + open(my $outfh, ">", "$filename.tmp") || return; + + my $done; + while (defined(my $line = <$infh>)) { + if ($line =~ m/^#\s*\Q${default_locale}\E.*/) { + print $outfh "${default_locale}\n" if !$done; + $done = 1; + } else { + print $outfh $line; + } + } + + print STDERR "generation pmg default locale\n"; + + rename("$filename.tmp", $filename) || return; # rename failed + + system("dpkg-reconfigure locales -f noninteractive"); +} + 1;