PMG/CLI/pmgqm.pm

   1 package PMG::CLI::pmgqm;
   2
   3 use strict;
   4 use Data::Dumper;
   5 use Template;
   6 use MIME::Entity;
   7 use HTML::Entities;
   8 use Time::Local;
   9 use Clone 'clone';
  10 use Mail::Header;
  11 use POSIX qw(strftime);
  12 use File::Find;
  13 use File::stat;
  14
  15 use PVE::SafeSyslog;
  16 use PVE::Tools;
  17 use PVE::INotify;
  18 use PVE::CLIHandler;
  19
  20 use PMG::RESTEnvironment;
  21 use PMG::Utils;
  22 use PMG::Ticket;
  23 use PMG::DBTools;
  24 use PMG::RuleDB;
  25 use PMG::Config;
  26 use PMG::ClusterConfig;
  27
  28 use base qw(PVE::CLIHandler);
  29
  30 sub setup_environment {
  31     PMG::RESTEnvironment->setup_default_cli_env();
  32 }
  33
  34 sub domain_regex {
  35     my ($domains) = @_;
  36
  37     my @ra;
  38     foreach my $d (@$domains) {
  39         # skip domains with non-DNS name characters
  40         next if $d =~ m/[^A-Za-z0-9\-\.]/;
  41         if ($d =~ m/^\.(.*)$/) {
  42             my $dom = $1;
  43             $dom =~ s/\./\\\./g;
  44             push @ra, $dom;
  45             push @ra, "\.\*\\.$dom";
  46         } else {
  47             $d =~ s/\./\\\./g;
  48             push @ra, $d;
  49         }
  50     }
  51
  52     my $re = join ('|', @ra);
  53
  54     my $regex = qr/\@($re)$/i;
  55
  56     return $regex;
  57 }
  58
  59 sub get_item_data {
  60     my ($data, $ref) = @_;
  61
  62     my @lines = split ('\n', $ref->{header});
  63     my $head = new Mail::Header(\@lines);
  64
  65     my $item = {};
  66
  67     $item->{id} = sprintf("C%dR%d", $ref->{cid}, $ref->{rid});
  68
  69     $item->{subject} = PMG::Utils::rfc1522_to_html(
  70         PVE::Tools::trim($head->get('subject')) || 'No Subject');
  71
  72     my @fromarray = split('\s*,\s*', $head->get('from') || $ref->{sender});
  73     my $from = PMG::Utils::rfc1522_to_html(PVE::Tools::trim($fromarray[0]));
  74     my $sender = PMG::Utils::rfc1522_to_html(PVE::Tools::trim($head->get('sender')));
  75
  76     if ($sender) {
  77         $item->{sender} = $sender;
  78         $item->{from} = sprintf ("%s on behalf of %s", $sender, $from);
  79     } else {
  80         $item->{from} = $from;
  81     }
  82
  83     $item->{pmail} = $ref->{pmail};
  84     $item->{receiver} = $ref->{receiver} || $ref->{pmail};
  85
  86     $item->{date} = strftime("%F", localtime($ref->{time}));
  87     $item->{time} = strftime("%H:%M:%S", localtime($ref->{time}));
  88
  89     $item->{bytes} = $ref->{bytes};
  90     $item->{spamlevel} = $ref->{spamlevel};
  91     $item->{spaminfo} = $ref->{info};
  92
  93     my $title = "Received: $item->{date} $item->{time}\n";
  94     $title .= "From: $ref->{sender}\n";
  95     $title .= "To: $ref->{receiver}\n" if $ref->{receiver};
  96     $title .= sprintf("Size: %d KB\n", int (($ref->{bytes} + 1023) / 1024 ));
  97     $title .= sprintf("Spam level: %d\n", $ref->{spamlevel}) if $ref->{qtype} eq 'S';
  98     $title .= sprintf("Virus info: %s\n", encode_entities ($ref->{info})) if $ref->{qtype} eq 'V';
  99     $title .= sprintf("File: %s", encode_entities($ref->{file}));
 100
 101     $item->{title} = $title;
 102
 103     my $basehref = "https://$data->{fqdn}:$data->{port}/userquar";
 104
 105     $item->{wlhref} = "$basehref?ticket=$data->{ticket}&cselect=$item->{id}&whitelist=1";
 106     $item->{blhref} = "$basehref?ticket=$data->{ticket}&cselect=$item->{id}&blacklist=1";
 107     $item->{deliverhref} = "$basehref?ticket=$data->{ticket}&cselect=$item->{id}&deliver=1";
 108     $item->{deletehref} = "$basehref?ticket=$data->{ticket}&cselect=$item->{id}&delete=1";
 109
 110     return $item;
 111 }
 112
 113 sub finalize_report {
 114     my ($tt, $template, $data, $mailfrom, $receiver, $debug) = @_;
 115
 116     my $html = '';
 117
 118     $tt->process($template, $data, \$html) ||
 119         die $tt->error() . "\n";
 120
 121     my $title;
 122     if ($html =~ m|^\s*<title>(.*)</title>|m) {
 123         $title = $1;
 124     } else {
 125         die "unable to extract template title\n";
 126     }
 127
 128     my $top = MIME::Entity->build(
 129         Type    => "multipart/related",
 130         To      => $data->{pmail},
 131         From    => $mailfrom,
 132         Subject => PMG::Utils::bencode_header(decode_entities($title)));
 133
 134     $top->attach(
 135         Data     => $html,
 136         Type     => "text/html",
 137         Encoding => $debug ? 'binary' : 'quoted-printable');
 138
 139     if ($debug) {
 140         $top->print();
 141         return;
 142     }
 143     # we use an empty envelope sender (we dont want to receive NDRs)
 144     PMG::Utils::reinject_mail ($top, '', [$receiver], undef, $data->{fqdn});
 145 }
 146
 147 __PACKAGE__->register_method ({
 148     name => 'send',
 149     path => 'send',
 150     method => 'POST',
 151     description => "Generate and send spam report emails.",
 152     parameters => {
 153         additionalProperties => 0,
 154         properties => {
 155             receiver => {
 156                 description => "Generate report for a single email address. If not specified, generate reports for all users.",
 157                 type => 'string', format => 'email',
 158                 optional => 1,
 159             },
 160             timespan => {
 161                 description => "Select time span.",
 162                 type => 'string',
 163                 enum => ['today', 'yesterday', 'week'],
 164                 default => 'today',
 165                 optional => 1,
 166             },
 167             style => {
 168                 description => "Spam report style. Value 'none' just prints the spam counts and does not send any emails. Default value is read from spam quarantine configuration.",
 169                 type => 'string',
 170                 enum => ['none', 'short', 'verbose', 'custom'],
 171                 optional => 1,
 172             },
 173             redirect => {
 174                 description => "Redirect spam report email to this address.",
 175                 type => 'string', format => 'email',
 176                 optional => 1,
 177             },
 178             debug => {
 179                 description => "Debug mode. Print raw email to stdout instead of sending them.",
 180                 type => 'boolean',
 181                 optional => 1,
 182                 default => 0,
 183             }
 184         },
 185     },
 186     returns => { type => 'null'},
 187     code => sub {
 188         my ($param) = @_;
 189
 190         my $cinfo = PMG::ClusterConfig->new();
 191         my $role = $cinfo->{local}->{type} // '-';
 192
 193         if (!(($role eq '-') || ($role eq 'master'))) {
 194            die "local node is not master - not sending spam report\n";
 195         }
 196
 197         my $cfg = PMG::Config->new();
 198
 199         my $reportstyle = $param->{style} // $cfg->get('spamquar', 'reportstyle');
 200
 201         my $timespan = $param->{timespan} // 'today';
 202
 203         my (undef, undef, undef, $mday, $mon, $year) = localtime(time());
 204         my $daystart = timelocal(0, 0, 0, $mday, $mon, $year);
 205
 206         my $start;
 207         my $end;
 208
 209         if ($timespan eq 'today') {
 210             $start = $daystart;
 211             $end = $start + 86400;
 212         } elsif ($timespan eq 'yesterday') {
 213             $end = $daystart;
 214             $start = $end - 86400;
 215         } elsif ($timespan eq 'week') {
 216             $end = $daystart;
 217             $start = $end - 7*86400;
 218         } else {
 219             die "internal error";
 220         }
 221
 222         my $hostname = PVE::INotify::nodename();
 223
 224         my $fqdn = $cfg->get('spamquar', 'hostname') //
 225             PVE::Tools::get_fqdn($hostname);
 226
 227         my $port = 8006;
 228
 229         my $global_data = {
 230             protocol => 'https',
 231             port => $port,
 232             fqdn => $fqdn,
 233             hostname => $hostname,
 234             date => strftime("%F", localtime($end - 1)),
 235             timespan => $timespan,
 236             items => [],
 237         };
 238
 239         my $mailfrom = $cfg->get ('spamquar', 'mailfrom') //
 240             "Proxmox Mail Gateway <postmaster>";
 241
 242         my $dbh = PMG::DBTools::open_ruledb();
 243
 244         my $target = $param->{receiver};
 245         my $redirect = $param->{redirect};
 246
 247         if (defined($redirect) && !defined($target)) {
 248             die "can't redirect mails for all users\n";
 249         }
 250
 251         my $domains = PVE::INotify::read_file('domains');
 252         my $domainregex = domain_regex([keys %$domains]);
 253
 254         my $template;
 255
 256         if ($reportstyle ne 'none') {
 257
 258             $template = "spamreport-${reportstyle}.tt";
 259             my $found = 0;
 260             foreach my $path (@$PMG::Config::tt_include_path) {
 261                 if (-f "$path/$template") { $found = 1; last; }
 262             }
 263             if (!$found) {
 264                 warn "unable to find template '$template' - using default\n";
 265                 $template = "spamreport-verbose.tt";
 266             }
 267         }
 268
 269         my $sth = $dbh->prepare(
 270             "SELECT * FROM CMailStore, CMSReceivers " .
 271             "WHERE time >= $start AND time < $end AND " .
 272             ($target ? "pmail = ? AND " : '') .
 273             "QType = 'S' AND CID = CMailStore_CID AND RID = CMailStore_RID " .
 274             "AND Status = 'N' " .
 275             "ORDER BY pmail, time, receiver");
 276
 277         if ($target) {
 278             $sth->execute($target);
 279         } else {
 280             $sth->execute();
 281         }
 282
 283         my $lastref;
 284         my $mailcount = 0;
 285         my $creceiver = '';
 286         my $data;
 287
 288         my $tt = PMG::Config::get_template_toolkit();
 289
 290         my $finalize = sub {
 291
 292             my $extern = ($domainregex && $creceiver !~ $domainregex);
 293
 294             if ($template) {
 295                 if (!$extern) {
 296                     $data->{mailcount} = $mailcount;
 297                     my $sendto = $redirect ? $redirect : $creceiver;
 298                     finalize_report($tt, $template, $data, $mailfrom, $sendto, $param->{debug});
 299                 }
 300             } else {
 301                 my $hint = $extern ? " (external address)" : "";
 302                 printf ("%-5d %s$hint\n", $mailcount, $creceiver);
 303             }
 304         };
 305
 306         while (my $ref = $sth->fetchrow_hashref()) {
 307             if ($creceiver ne $ref->{pmail}) {
 308
 309                 $finalize->() if $data;
 310
 311                 $data = clone($global_data);
 312
 313                 $creceiver = $ref->{pmail};
 314                 $mailcount = 0;
 315
 316                 $data->{pmail} = $creceiver;
 317                 $data->{ticket} = PMG::Ticket::assemble_quarantine_ticket($data->{pmail});
 318                 $data->{managehref} = "https://$fqdn:$port?ticket=$data->{ticket}";
 319             }
 320
 321             if ($template) {
 322                 push @{$data->{items}}, get_item_data($data, $ref);
 323             }
 324
 325             $mailcount++;
 326             $lastref = $ref;
 327         }
 328
 329         $sth->finish();
 330
 331         $finalize->() if $data;
 332
 333         if (defined($target) && !$mailcount) {
 334             print STDERR "no mails for '$target'\n";
 335         }
 336
 337         return undef;
 338     }});
 339
 340 sub find_stale_files {
 341     my ($path, $lifetime, $purge) = @_;
 342
 343     return if ! -d $path;
 344
 345     my (undef, undef, undef, $mday, $mon, $year) = localtime(time());
 346     my $daystart = timelocal(0, 0, 0, $mday, $mon, $year);
 347     my $expire = $daystart - $lifetime*86400;
 348
 349     my $wanted = sub {
 350         my $name = $File::Find::name;
 351         return if $name !~ m|^($path/.*)$|;
 352         $name = $1; # untaint
 353         my $stat = stat($name);
 354         return if ! -f _;
 355         return if $stat->mtime >= $expire;
 356         if ($purge) {
 357             if (unlink($name)) {
 358                 print "removed: $name\n";
 359             }
 360         } else {
 361             print "$name\n";
 362         }
 363     };
 364
 365     find({ wanted => $wanted, no_chdir => 1 }, $path);
 366 }
 367
 368 sub test_quarantine_files {
 369     my ($spamlifetime, $viruslifetime, $purge) = @_;
 370
 371     print STDERR "searching for stale files\n" if !$purge;
 372
 373     my $spooldir = $PMG::MailQueue::spooldir;
 374
 375     find_stale_files ("$spooldir/spam", $spamlifetime, $purge);
 376     foreach my $dir (<"/var/spool/pmg/cluster/*/spam">) {
 377         next if $dir !~ m|^(/var/spool/pmg/cluster/\d+/spam)$|;
 378         $dir = $1; # untaint
 379         find_stale_files ($dir, $spamlifetime, $purge);
 380     }
 381
 382     find_stale_files ("$spooldir/virus", $viruslifetime, $purge);
 383     foreach my $dir (<"/var/spool/pmg/cluster/*/virus">) {
 384         next if $dir !~ m|^(/var/spool/pmg/cluster/\d+/virus)$|;
 385         $dir = $1; # untaint
 386         find_stale_files ($dir, $viruslifetime, $purge);
 387     }
 388 }
 389
 390 __PACKAGE__->register_method ({
 391     name => 'purge',
 392     path => 'purge',
 393     method => 'POST',
 394     description => "Cleanup Quarantine database. Remove entries older than configured quarantine lifetime.",
 395     parameters => {
 396         additionalProperties => 0,
 397         properties => {
 398             check => {
 399                 description => "Only search for quarantine files older than configured quarantine lifetime. Just print found files, but does not remove them.",
 400                 type => 'boolean',
 401                 optional => 1,
 402                 default => 0,
 403             }
 404         }
 405     },
 406     returns => { type => 'null'},
 407     code => sub {
 408         my ($param) = @_;
 409
 410         my $cfg = PMG::Config->new();
 411
 412         my $spamlifetime = $cfg->get('spamquar', 'lifetime');
 413         my $viruslifetime = $cfg->get ('virusquar', 'lifetime');
 414
 415         my $purge = !$param->{check};
 416
 417         if ($purge) {
 418             print STDERR "purging database\n";
 419
 420             my $dbh = PMG::DBTools::open_ruledb();
 421
 422             if (my $count = PMG::DBTools::purge_quarantine_database($dbh, 'S', $spamlifetime)) {
 423                 print STDERR "removed $count spam quarantine files\n";
 424             }
 425
 426             if (my $count = PMG::DBTools::purge_quarantine_database($dbh, 'V', $viruslifetime)) {
 427                 print STDERR "removed $count virus quarantine files\n";
 428             }
 429         }
 430
 431         test_quarantine_files($spamlifetime, $viruslifetime, $purge);
 432
 433         return undef;
 434     }});
 435
 436
 437 our $cmddef = {
 438     'purge' => [ __PACKAGE__, 'purge', []],
 439     'send' => [ __PACKAGE__, 'send', []],
 440 };
 441
 442 1;