src/PMG/CLI/pmgqm.pm

   1 package PMG::CLI::pmgqm;
   2
   3 use strict;
   4 use Data::Dumper;
   5 use Template;
   6 use MIME::Entity;
   7 use HTML::Entities;
   8 use Time::Local;
   9 use Clone 'clone';
  10 use Mail::Header;
  11 use POSIX qw(strftime);
  12 use File::Find;
  13 use File::stat;
  14 use URI::Escape;
  15
  16 use PVE::SafeSyslog;
  17 use PVE::Tools;
  18 use PVE::INotify;
  19 use PVE::CLIHandler;
  20
  21 use PMG::RESTEnvironment;
  22 use PMG::Utils;
  23 use PMG::Ticket;
  24 use PMG::DBTools;
  25 use PMG::RuleDB;
  26 use PMG::Config;
  27 use PMG::ClusterConfig;
  28 use PMG::API2::Quarantine;
  29
  30 use base qw(PVE::CLIHandler);
  31
  32 sub setup_environment {
  33     PMG::RESTEnvironment->setup_default_cli_env();
  34 }
  35
  36 sub domain_regex {
  37     my ($domains) = @_;
  38
  39     my @ra;
  40     foreach my $d (@$domains) {
  41         # skip domains with non-DNS name characters
  42         next if $d =~ m/[^A-Za-z0-9\-\.]/;
  43         if ($d =~ m/^\.(.*)$/) {
  44             my $dom = $1;
  45             $dom =~ s/\./\\\./g;
  46             push @ra, $dom;
  47             push @ra, "\.\*\\.$dom";
  48         } else {
  49             $d =~ s/\./\\\./g;
  50             push @ra, $d;
  51         }
  52     }
  53
  54     my $re = join ('|', @ra);
  55
  56     my $regex = qr/\@($re)$/i;
  57
  58     return $regex;
  59 }
  60
  61 sub get_item_data {
  62     my ($data, $ref) = @_;
  63
  64     my @lines = split ('\n', $ref->{header});
  65     my $head = new Mail::Header(\@lines);
  66
  67     my $item = {};
  68
  69     $item->{id} = sprintf("C%dR%dT%d", $ref->{cid}, $ref->{rid}, $ref->{ticketid});
  70
  71     $item->{subject} = PMG::Utils::rfc1522_to_html(
  72         PVE::Tools::trim($head->get('subject')) || 'No Subject');
  73
  74     my @fromarray = split('\s*,\s*', $head->get('from') || $ref->{sender});
  75     my $from = PMG::Utils::rfc1522_to_html(PVE::Tools::trim($fromarray[0]));
  76     my $sender = PMG::Utils::rfc1522_to_html(PVE::Tools::trim($head->get('sender')));
  77
  78     if ($sender) {
  79         $item->{sender} = $sender;
  80         $item->{from} = sprintf ("%s on behalf of %s", $sender, $from);
  81     } else {
  82         $item->{from} = $from;
  83     }
  84
  85     $item->{envelope_sender} = $ref->{sender};
  86     $item->{pmail} = $ref->{pmail};
  87     $item->{receiver} = $ref->{receiver} || $ref->{pmail};
  88
  89     $item->{date} = strftime("%F", localtime($ref->{time}));
  90     $item->{time} = strftime("%H:%M:%S", localtime($ref->{time}));
  91
  92     $item->{bytes} = $ref->{bytes};
  93     $item->{spamlevel} = $ref->{spamlevel};
  94     $item->{spaminfo} = $ref->{info};
  95     $item->{file} = $ref->{file};
  96
  97     my $basehref = "$data->{protocol}://$data->{fqdn}:$data->{port}/quarantine";
  98     my $ticket = uri_escape($data->{ticket});
  99     $item->{href} = "$basehref?ticket=$ticket&cselect=$item->{id}&date=$item->{date}";
 100
 101     return $item;
 102 }
 103
 104 __PACKAGE__->register_method ({
 105     name => 'status',
 106     path => 'status',
 107     method => 'POST',
 108     description => "Print quarantine status (mails per user) for specified time span.",
 109     parameters => {
 110         additionalProperties => 0,
 111         properties => {
 112             timespan => {
 113                 description => "Select time span.",
 114                 type => 'string',
 115                 enum => ['today', 'yesterday', 'week'],
 116                 default => 'today',
 117                 optional => 1,
 118             },
 119         },
 120     },
 121     returns => { type => 'null'},
 122     code => sub {
 123         my ($param) = @_;
 124
 125         my $cinfo = PMG::ClusterConfig->new();
 126         my $role = $cinfo->{local}->{type} // '-';
 127
 128         if (!(($role eq '-') || ($role eq 'master'))) {
 129            warn "local node is not master\n";
 130            return;
 131         }
 132
 133         my $cfg = PMG::Config->new();
 134
 135         my $timespan = $param->{timespan} // 'today';
 136
 137         my ($start, $end) = PMG::Utils::lookup_timespan($timespan);
 138
 139         my $hostname = PVE::INotify::nodename();
 140
 141         my $fqdn = $cfg->get('spamquar', 'hostname') //
 142             PVE::Tools::get_fqdn($hostname);
 143
 144
 145         my $dbh = PMG::DBTools::open_ruledb();
 146
 147         my $domains = PVE::INotify::read_file('domains');
 148         my $domainregex = domain_regex([keys %$domains]);
 149
 150         my $sth = $dbh->prepare(
 151             "SELECT pmail, AVG(spamlevel) as spamlevel, count(*)  FROM CMailStore, CMSReceivers " .
 152             "WHERE time >= $start AND time < $end AND " .
 153             "QType = 'S' AND CID = CMailStore_CID AND RID = CMailStore_RID " .
 154             "AND Status = 'N' " .
 155             "GROUP BY pmail " .
 156             "ORDER BY pmail");
 157
 158         $sth->execute();
 159
 160         print "Count  Spamlevel Mail\n";
 161         my $res = [];
 162         while (my $ref = $sth->fetchrow_hashref()) {
 163             push @$res, $ref;
 164             my $extern = ($domainregex && $ref->{pmail} !~ $domainregex);
 165             my $hint = $extern ? " (external address)" : "";
 166             printf ("%-5d %10.2f %s$hint\n", $ref->{count}, $ref->{spamlevel}, $ref->{pmail});
 167         }
 168
 169         $sth->finish();
 170
 171         return undef;
 172     }});
 173
 174 __PACKAGE__->register_method ({
 175     name => 'send',
 176     path => 'send',
 177     method => 'POST',
 178     description => "Generate and send spam report emails.",
 179     parameters => {
 180         additionalProperties => 0,
 181         properties => {
 182             receiver => {
 183                 description => "Generate report for a single email address. If not specified, generate reports for all users.",
 184                 type => 'string', format => 'email',
 185                 optional => 1,
 186             },
 187             timespan => {
 188                 description => "Select time span.",
 189                 type => 'string',
 190                 enum => ['today', 'yesterday', 'week'],
 191                 default => 'today',
 192                 optional => 1,
 193             },
 194             style => {
 195                 description => "Spam report style. Default value is read from spam quarantine configuration.",
 196                 type => 'string',
 197                 enum => ['short', 'verbose', 'custom'],
 198                 optional => 1,
 199             },
 200             redirect => {
 201                 description => "Redirect spam report email to this address.",
 202                 type => 'string', format => 'email',
 203                 optional => 1,
 204             },
 205             debug => {
 206                 description => "Debug mode. Print raw email to stdout instead of sending them.",
 207                 type => 'boolean',
 208                 optional => 1,
 209                 default => 0,
 210             }
 211         },
 212     },
 213     returns => { type => 'null'},
 214     code => sub {
 215         my ($param) = @_;
 216
 217         my $cinfo = PMG::ClusterConfig->new();
 218         my $role = $cinfo->{local}->{type} // '-';
 219
 220         if (!(($role eq '-') || ($role eq 'master'))) {
 221            warn "local node is not master - not sending spam report\n";
 222            return;
 223         }
 224
 225         my $cfg = PMG::Config->new();
 226
 227         my $reportstyle = $param->{style} // $cfg->get('spamquar', 'reportstyle');
 228
 229         # overwrite report style none when:
 230         # - explicit receiver specified
 231         # - when debug flag enabled
 232         if ($reportstyle eq 'none') {
 233             $reportstyle = 'verbose' if $param->{debug} || defined($param->{receiver});
 234         }
 235
 236         return if $reportstyle eq 'none'; # do nothing
 237
 238         my $timespan = $param->{timespan} // 'today';
 239
 240         my ($start, $end) = PMG::Utils::lookup_timespan($timespan);
 241
 242         my $hostname = PVE::INotify::nodename();
 243
 244         my $fqdn = $cfg->get('spamquar', 'hostname') //
 245             PVE::Tools::get_fqdn($hostname);
 246
 247         my $port = $cfg->get('spamquar', 'port') // 8006;
 248
 249         my $protocol = $cfg->get('spamquar', 'protocol') // 'https';
 250
 251         my $global_data = {
 252             protocol => $protocol,
 253             port => $port,
 254             fqdn => $fqdn,
 255             hostname => $hostname,
 256             date => strftime("%F", localtime($end - 1)),
 257             timespan => $timespan,
 258             items => [],
 259         };
 260
 261         my $mailfrom = $cfg->get ('spamquar', 'mailfrom') //
 262             "Proxmox Mail Gateway <postmaster>";
 263
 264         my $dbh = PMG::DBTools::open_ruledb();
 265
 266         my $target = $param->{receiver};
 267         my $redirect = $param->{redirect};
 268
 269         if (defined($redirect) && !defined($target)) {
 270             die "can't redirect mails for all users\n";
 271         }
 272
 273         my $domains = PVE::INotify::read_file('domains');
 274         my $domainregex = domain_regex([keys %$domains]);
 275
 276         my $template = "spamreport-${reportstyle}.tt";
 277         my $found = 0;
 278         foreach my $path (@$PMG::Config::tt_include_path) {
 279             if (-f "$path/$template") { $found = 1; last; }
 280         }
 281         if (!$found) {
 282             warn "unable to find template '$template' - using default\n";
 283             $template = "spamreport-verbose.tt";
 284         }
 285
 286         my $sth = $dbh->prepare(
 287             "SELECT * FROM CMailStore, CMSReceivers " .
 288             "WHERE time >= $start AND time < $end AND " .
 289             ($target ? "pmail = ? AND " : '') .
 290             "QType = 'S' AND CID = CMailStore_CID AND RID = CMailStore_RID " .
 291             "AND Status = 'N' " .
 292             "ORDER BY pmail, time, receiver");
 293
 294         if ($target) {
 295             $sth->execute($target);
 296         } else {
 297             $sth->execute();
 298         }
 299
 300         my $lastref;
 301         my $mailcount = 0;
 302         my $creceiver = '';
 303         my $data;
 304
 305         my $tt = PMG::Config::get_template_toolkit();
 306
 307         my $finalize = sub {
 308
 309             my $extern = ($domainregex && $creceiver !~ $domainregex);
 310             if (!$extern) {
 311                 $data->{mailcount} = $mailcount;
 312                 my $sendto = $redirect ? $redirect : $creceiver;
 313                 PMG::Utils::finalize_report($tt, $template, $data, $mailfrom, $sendto, $param->{debug});
 314             }
 315         };
 316
 317         while (my $ref = $sth->fetchrow_hashref()) {
 318             if ($creceiver ne $ref->{pmail}) {
 319
 320                 $finalize->() if $data;
 321
 322                 $data = clone($global_data);
 323
 324                 $creceiver = $ref->{pmail};
 325                 $mailcount = 0;
 326
 327                 $data->{pmail} = $creceiver;
 328                 $data->{ticket} = PMG::Ticket::assemble_quarantine_ticket($data->{pmail});
 329                 my $esc_ticket = uri_escape($data->{ticket});
 330                 $data->{managehref} = "$protocol://$fqdn:$port/quarantine?ticket=${esc_ticket}";
 331             }
 332
 333             push @{$data->{items}}, get_item_data($data, $ref);
 334
 335             $mailcount++;
 336             $lastref = $ref;
 337         }
 338
 339         $sth->finish();
 340
 341         $finalize->() if $data;
 342
 343         if (defined($target) && !$mailcount) {
 344             print STDERR "no mails for '$target'\n";
 345         }
 346
 347         return undef;
 348     }});
 349
 350 sub find_stale_files {
 351     my ($path, $lifetime, $purge) = @_;
 352
 353     return if ! -d $path;
 354
 355     my (undef, undef, undef, $mday, $mon, $year) = localtime(time());
 356     my $daystart = timelocal(0, 0, 0, $mday, $mon, $year);
 357     my $expire = $daystart - $lifetime*86400;
 358
 359     my $wanted = sub {
 360         my $name = $File::Find::name;
 361         return if $name !~ m|^($path/.*)$|;
 362         $name = $1; # untaint
 363         my $stat = stat($name);
 364         return if ! -f _;
 365         return if $stat->mtime >= $expire;
 366         if ($purge) {
 367             if (unlink($name)) {
 368                 print "removed: $name\n";
 369             }
 370         } else {
 371             print "$name\n";
 372         }
 373     };
 374
 375     find({ wanted => $wanted, no_chdir => 1 }, $path);
 376 }
 377
 378 sub test_quarantine_files {
 379     my ($spamlifetime, $viruslifetime, $purge) = @_;
 380
 381     print STDERR "searching for stale files\n" if !$purge;
 382
 383     my $spooldir = $PMG::MailQueue::spooldir;
 384
 385     find_stale_files ("$spooldir/spam", $spamlifetime, $purge);
 386     foreach my $dir (<"/var/spool/pmg/cluster/*/spam">) {
 387         next if $dir !~ m|^(/var/spool/pmg/cluster/\d+/spam)$|;
 388         $dir = $1; # untaint
 389         find_stale_files ($dir, $spamlifetime, $purge);
 390     }
 391
 392     find_stale_files ("$spooldir/virus", $viruslifetime, $purge);
 393     foreach my $dir (<"/var/spool/pmg/cluster/*/virus">) {
 394         next if $dir !~ m|^(/var/spool/pmg/cluster/\d+/virus)$|;
 395         $dir = $1; # untaint
 396         find_stale_files ($dir, $viruslifetime, $purge);
 397     }
 398 }
 399
 400 __PACKAGE__->register_method ({
 401     name => 'purge',
 402     path => 'purge',
 403     method => 'POST',
 404     description => "Cleanup Quarantine database. Remove entries older than configured quarantine lifetime.",
 405     parameters => {
 406         additionalProperties => 0,
 407         properties => {
 408             check => {
 409                 description => "Only search for quarantine files older than configured quarantine lifetime. Just print found files, but do not remove them.",
 410                 type => 'boolean',
 411                 optional => 1,
 412                 default => 0,
 413             }
 414         }
 415     },
 416     returns => { type => 'null'},
 417     code => sub {
 418         my ($param) = @_;
 419
 420         my $cfg = PMG::Config->new();
 421
 422         my $spamlifetime = $cfg->get('spamquar', 'lifetime');
 423         my $viruslifetime = $cfg->get ('virusquar', 'lifetime');
 424
 425         my $purge = !$param->{check};
 426
 427         if ($purge) {
 428             print STDERR "purging database\n";
 429
 430             my $dbh = PMG::DBTools::open_ruledb();
 431
 432             if (my $count = PMG::DBTools::purge_quarantine_database($dbh, 'S', $spamlifetime)) {
 433                 print STDERR "removed $count spam quarantine files\n";
 434             }
 435
 436             if (my $count = PMG::DBTools::purge_quarantine_database($dbh, 'V', $viruslifetime)) {
 437                 print STDERR "removed $count virus quarantine files\n";
 438             }
 439
 440             if (my $count = PMG::DBTools::purge_quarantine_database($dbh, 'A', $spamlifetime)) {
 441                 print STDERR "removed $count attachment quarantine files\n";
 442             }
 443         }
 444
 445         test_quarantine_files($spamlifetime, $viruslifetime, $purge);
 446
 447         return undef;
 448     }});
 449
 450
 451 our $cmddef = {
 452     'purge' => [ __PACKAGE__, 'purge', []],
 453     'send' => [ __PACKAGE__, 'send', []],
 454     'status' => [ __PACKAGE__, 'status', []],
 455 };
 456
 457 1;