PMG/CLI/pmgqm.pm

   1 package PMG::CLI::pmgqm;
   2
   3 use strict;
   4 use Data::Dumper;
   5 use Template;
   6 use MIME::Entity;
   7 use HTML::Entities;
   8 use Time::Local;
   9 use Clone 'clone';
  10 use Mail::Header;
  11 use POSIX qw(strftime);
  12 use File::Find;
  13 use File::stat;
  14 use URI::Escape;
  15
  16 use PVE::SafeSyslog;
  17 use PVE::Tools;
  18 use PVE::INotify;
  19 use PVE::CLIHandler;
  20
  21 use PMG::RESTEnvironment;
  22 use PMG::Utils;
  23 use PMG::Ticket;
  24 use PMG::DBTools;
  25 use PMG::RuleDB;
  26 use PMG::Config;
  27 use PMG::ClusterConfig;
  28 use PMG::API2::Quarantine;
  29
  30 use base qw(PVE::CLIHandler);
  31
  32 sub setup_environment {
  33     PMG::RESTEnvironment->setup_default_cli_env();
  34 }
  35
  36 sub domain_regex {
  37     my ($domains) = @_;
  38
  39     my @ra;
  40     foreach my $d (@$domains) {
  41         # skip domains with non-DNS name characters
  42         next if $d =~ m/[^A-Za-z0-9\-\.]/;
  43         if ($d =~ m/^\.(.*)$/) {
  44             my $dom = $1;
  45             $dom =~ s/\./\\\./g;
  46             push @ra, $dom;
  47             push @ra, "\.\*\\.$dom";
  48         } else {
  49             $d =~ s/\./\\\./g;
  50             push @ra, $d;
  51         }
  52     }
  53
  54     my $re = join ('|', @ra);
  55
  56     my $regex = qr/\@($re)$/i;
  57
  58     return $regex;
  59 }
  60
  61 sub get_item_data {
  62     my ($data, $ref) = @_;
  63
  64     my @lines = split ('\n', $ref->{header});
  65     my $head = new Mail::Header(\@lines);
  66
  67     my $item = {};
  68
  69     $item->{id} = sprintf("C%dR%d", $ref->{cid}, $ref->{rid});
  70
  71     $item->{subject} = PMG::Utils::rfc1522_to_html(
  72         PVE::Tools::trim($head->get('subject')) || 'No Subject');
  73
  74     my @fromarray = split('\s*,\s*', $head->get('from') || $ref->{sender});
  75     my $from = PMG::Utils::rfc1522_to_html(PVE::Tools::trim($fromarray[0]));
  76     my $sender = PMG::Utils::rfc1522_to_html(PVE::Tools::trim($head->get('sender')));
  77
  78     if ($sender) {
  79         $item->{sender} = $sender;
  80         $item->{from} = sprintf ("%s on behalf of %s", $sender, $from);
  81     } else {
  82         $item->{from} = $from;
  83     }
  84
  85     $item->{pmail} = $ref->{pmail};
  86     $item->{receiver} = $ref->{receiver} || $ref->{pmail};
  87
  88     $item->{date} = strftime("%F", localtime($ref->{time}));
  89     $item->{time} = strftime("%H:%M:%S", localtime($ref->{time}));
  90
  91     $item->{bytes} = $ref->{bytes};
  92     $item->{spamlevel} = $ref->{spamlevel};
  93     $item->{spaminfo} = $ref->{info};
  94
  95     my $title = "Received: $item->{date} $item->{time}\n";
  96     $title .= "From: $ref->{sender}\n";
  97     $title .= "To: $ref->{receiver}\n" if $ref->{receiver};
  98     $title .= sprintf("Size: %d KB\n", int (($ref->{bytes} + 1023) / 1024 ));
  99     $title .= sprintf("Spam level: %d\n", $ref->{spamlevel}) if $ref->{qtype} eq 'S';
 100     $title .= sprintf("Virus info: %s\n", encode_entities ($ref->{info})) if $ref->{qtype} eq 'V';
 101     $title .= sprintf("File: %s", encode_entities($ref->{file}));
 102
 103     $item->{title} = $title;
 104
 105     my $basehref = "https://$data->{fqdn}:$data->{port}/quarantine";
 106     my $ticket = uri_escape($data->{ticket});
 107     $item->{wlhref} = "$basehref?ticket=$ticket&cselect=$item->{id}&action=whitelist";
 108     $item->{blhref} = "$basehref?ticket=$ticket&cselect=$item->{id}&action=blacklist";
 109     $item->{deliverhref} = "$basehref?ticket=$ticket&cselect=$item->{id}&action=deliver";
 110     $item->{deletehref} = "$basehref?ticket=$ticket&cselect=$item->{id}&action=delete";
 111
 112     return $item;
 113 }
 114
 115 __PACKAGE__->register_method ({
 116     name => 'send',
 117     path => 'send',
 118     method => 'POST',
 119     description => "Generate and send spam report emails.",
 120     parameters => {
 121         additionalProperties => 0,
 122         properties => {
 123             receiver => {
 124                 description => "Generate report for a single email address. If not specified, generate reports for all users.",
 125                 type => 'string', format => 'email',
 126                 optional => 1,
 127             },
 128             timespan => {
 129                 description => "Select time span.",
 130                 type => 'string',
 131                 enum => ['today', 'yesterday', 'week'],
 132                 default => 'today',
 133                 optional => 1,
 134             },
 135             style => {
 136                 description => "Spam report style. Value 'none' just prints the spam counts and does not send any emails. Default value is read from spam quarantine configuration.",
 137                 type => 'string',
 138                 enum => ['none', 'short', 'verbose', 'custom'],
 139                 optional => 1,
 140             },
 141             redirect => {
 142                 description => "Redirect spam report email to this address.",
 143                 type => 'string', format => 'email',
 144                 optional => 1,
 145             },
 146             debug => {
 147                 description => "Debug mode. Print raw email to stdout instead of sending them.",
 148                 type => 'boolean',
 149                 optional => 1,
 150                 default => 0,
 151             }
 152         },
 153     },
 154     returns => { type => 'null'},
 155     code => sub {
 156         my ($param) = @_;
 157
 158         my $cinfo = PMG::ClusterConfig->new();
 159         my $role = $cinfo->{local}->{type} // '-';
 160
 161         if (!(($role eq '-') || ($role eq 'master'))) {
 162            die "local node is not master - not sending spam report\n";
 163         }
 164
 165         my $cfg = PMG::Config->new();
 166
 167         my $reportstyle = $param->{style} // $cfg->get('spamquar', 'reportstyle');
 168
 169         my $timespan = $param->{timespan} // 'today';
 170
 171         my (undef, undef, undef, $mday, $mon, $year) = localtime(time());
 172         my $daystart = timelocal(0, 0, 0, $mday, $mon, $year);
 173
 174         my $start;
 175         my $end;
 176
 177         if ($timespan eq 'today') {
 178             $start = $daystart;
 179             $end = $start + 86400;
 180         } elsif ($timespan eq 'yesterday') {
 181             $end = $daystart;
 182             $start = $end - 86400;
 183         } elsif ($timespan eq 'week') {
 184             $end = $daystart;
 185             $start = $end - 7*86400;
 186         } else {
 187             die "internal error";
 188         }
 189
 190         my $hostname = PVE::INotify::nodename();
 191
 192         my $fqdn = $cfg->get('spamquar', 'hostname') //
 193             PVE::Tools::get_fqdn($hostname);
 194
 195         my $port = 8006;
 196
 197         my $global_data = {
 198             protocol => 'https',
 199             port => $port,
 200             fqdn => $fqdn,
 201             hostname => $hostname,
 202             date => strftime("%F", localtime($end - 1)),
 203             timespan => $timespan,
 204             items => [],
 205         };
 206
 207         my $mailfrom = $cfg->get ('spamquar', 'mailfrom') //
 208             "Proxmox Mail Gateway <postmaster>";
 209
 210         my $dbh = PMG::DBTools::open_ruledb();
 211
 212         my $target = $param->{receiver};
 213         my $redirect = $param->{redirect};
 214
 215         if (defined($redirect) && !defined($target)) {
 216             die "can't redirect mails for all users\n";
 217         }
 218
 219         my $domains = PVE::INotify::read_file('domains');
 220         my $domainregex = domain_regex([keys %$domains]);
 221
 222         my $template;
 223
 224         if ($reportstyle ne 'none') {
 225
 226             $template = "spamreport-${reportstyle}.tt";
 227             my $found = 0;
 228             foreach my $path (@$PMG::Config::tt_include_path) {
 229                 if (-f "$path/$template") { $found = 1; last; }
 230             }
 231             if (!$found) {
 232                 warn "unable to find template '$template' - using default\n";
 233                 $template = "spamreport-verbose.tt";
 234             }
 235         }
 236
 237         my $sth = $dbh->prepare(
 238             "SELECT * FROM CMailStore, CMSReceivers " .
 239             "WHERE time >= $start AND time < $end AND " .
 240             ($target ? "pmail = ? AND " : '') .
 241             "QType = 'S' AND CID = CMailStore_CID AND RID = CMailStore_RID " .
 242             "AND Status = 'N' " .
 243             "ORDER BY pmail, time, receiver");
 244
 245         if ($target) {
 246             $sth->execute($target);
 247         } else {
 248             $sth->execute();
 249         }
 250
 251         my $lastref;
 252         my $mailcount = 0;
 253         my $creceiver = '';
 254         my $data;
 255
 256         my $tt = PMG::Config::get_template_toolkit();
 257
 258         my $finalize = sub {
 259
 260             my $extern = ($domainregex && $creceiver !~ $domainregex);
 261
 262             if ($template) {
 263                 if (!$extern) {
 264                     $data->{mailcount} = $mailcount;
 265                     my $sendto = $redirect ? $redirect : $creceiver;
 266                     PMG::Utils::finalize_report($tt, $template, $data, $mailfrom, $sendto, $param->{debug});
 267                 }
 268             } else {
 269                 my $hint = $extern ? " (external address)" : "";
 270                 printf ("%-5d %s$hint\n", $mailcount, $creceiver);
 271             }
 272         };
 273
 274         while (my $ref = $sth->fetchrow_hashref()) {
 275             if ($creceiver ne $ref->{pmail}) {
 276
 277                 $finalize->() if $data;
 278
 279                 $data = clone($global_data);
 280
 281                 $creceiver = $ref->{pmail};
 282                 $mailcount = 0;
 283
 284                 $data->{pmail} = $creceiver;
 285                 $data->{ticket} = PMG::Ticket::assemble_quarantine_ticket($data->{pmail});
 286                 my $esc_ticket = uri_escape($data->{ticket});
 287                 $data->{managehref} = "https://$fqdn:$port/quarantine?ticket=${esc_ticket}";
 288             }
 289
 290             if ($template) {
 291                 push @{$data->{items}}, get_item_data($data, $ref);
 292             }
 293
 294             $mailcount++;
 295             $lastref = $ref;
 296         }
 297
 298         $sth->finish();
 299
 300         $finalize->() if $data;
 301
 302         if (defined($target) && !$mailcount) {
 303             print STDERR "no mails for '$target'\n";
 304         }
 305
 306         return undef;
 307     }});
 308
 309 sub find_stale_files {
 310     my ($path, $lifetime, $purge) = @_;
 311
 312     return if ! -d $path;
 313
 314     my (undef, undef, undef, $mday, $mon, $year) = localtime(time());
 315     my $daystart = timelocal(0, 0, 0, $mday, $mon, $year);
 316     my $expire = $daystart - $lifetime*86400;
 317
 318     my $wanted = sub {
 319         my $name = $File::Find::name;
 320         return if $name !~ m|^($path/.*)$|;
 321         $name = $1; # untaint
 322         my $stat = stat($name);
 323         return if ! -f _;
 324         return if $stat->mtime >= $expire;
 325         if ($purge) {
 326             if (unlink($name)) {
 327                 print "removed: $name\n";
 328             }
 329         } else {
 330             print "$name\n";
 331         }
 332     };
 333
 334     find({ wanted => $wanted, no_chdir => 1 }, $path);
 335 }
 336
 337 sub test_quarantine_files {
 338     my ($spamlifetime, $viruslifetime, $purge) = @_;
 339
 340     print STDERR "searching for stale files\n" if !$purge;
 341
 342     my $spooldir = $PMG::MailQueue::spooldir;
 343
 344     find_stale_files ("$spooldir/spam", $spamlifetime, $purge);
 345     foreach my $dir (<"/var/spool/pmg/cluster/*/spam">) {
 346         next if $dir !~ m|^(/var/spool/pmg/cluster/\d+/spam)$|;
 347         $dir = $1; # untaint
 348         find_stale_files ($dir, $spamlifetime, $purge);
 349     }
 350
 351     find_stale_files ("$spooldir/virus", $viruslifetime, $purge);
 352     foreach my $dir (<"/var/spool/pmg/cluster/*/virus">) {
 353         next if $dir !~ m|^(/var/spool/pmg/cluster/\d+/virus)$|;
 354         $dir = $1; # untaint
 355         find_stale_files ($dir, $viruslifetime, $purge);
 356     }
 357 }
 358
 359 __PACKAGE__->register_method ({
 360     name => 'purge',
 361     path => 'purge',
 362     method => 'POST',
 363     description => "Cleanup Quarantine database. Remove entries older than configured quarantine lifetime.",
 364     parameters => {
 365         additionalProperties => 0,
 366         properties => {
 367             check => {
 368                 description => "Only search for quarantine files older than configured quarantine lifetime. Just print found files, but do not remove them.",
 369                 type => 'boolean',
 370                 optional => 1,
 371                 default => 0,
 372             }
 373         }
 374     },
 375     returns => { type => 'null'},
 376     code => sub {
 377         my ($param) = @_;
 378
 379         my $cfg = PMG::Config->new();
 380
 381         my $spamlifetime = $cfg->get('spamquar', 'lifetime');
 382         my $viruslifetime = $cfg->get ('virusquar', 'lifetime');
 383
 384         my $purge = !$param->{check};
 385
 386         if ($purge) {
 387             print STDERR "purging database\n";
 388
 389             my $dbh = PMG::DBTools::open_ruledb();
 390
 391             if (my $count = PMG::DBTools::purge_quarantine_database($dbh, 'S', $spamlifetime)) {
 392                 print STDERR "removed $count spam quarantine files\n";
 393             }
 394
 395             if (my $count = PMG::DBTools::purge_quarantine_database($dbh, 'V', $viruslifetime)) {
 396                 print STDERR "removed $count virus quarantine files\n";
 397             }
 398         }
 399
 400         test_quarantine_files($spamlifetime, $viruslifetime, $purge);
 401
 402         return undef;
 403     }});
 404
 405
 406 our $cmddef = {
 407     'purge' => [ __PACKAGE__, 'purge', []],
 408     'send' => [ __PACKAGE__, 'send', []],
 409     'spam' => [ 'PMG::API2::Quarantine', 'spam', [], undef, sub {
 410         my $res = shift;
 411         print "Day          Count  AVG(Spam)\n";
 412         foreach my $ref (@$res) {
 413             print sprintf("%-12s %5d %10.2f\n",
 414                           strftime("%F", localtime($ref->{day})),
 415                           $ref->{count}, $ref->{spamavg});
 416         }
 417      }],
 418 };
 419
 420 1;