src/PMG/CLI/pmgqm.pm

   1 package PMG::CLI::pmgqm;
   2
   3 use strict;
   4 use Data::Dumper;
   5 use Encode qw(encode);
   6 use Template;
   7 use MIME::Entity;
   8 use HTML::Entities;
   9 use Time::Local;
  10 use Clone 'clone';
  11 use Mail::Header;
  12 use POSIX qw(strftime);
  13 use File::Find;
  14 use File::stat;
  15 use URI::Escape;
  16
  17 use PVE::SafeSyslog;
  18 use PVE::Tools;
  19 use PVE::INotify;
  20 use PVE::CLIHandler;
  21 use PVE::JSONSchema qw(get_standard_option);
  22
  23 use PMG::RESTEnvironment;
  24 use PMG::Utils;
  25 use PMG::Ticket;
  26 use PMG::DBTools;
  27 use PMG::RuleDB;
  28 use PMG::Config;
  29 use PMG::ClusterConfig;
  30 use PMG::API2::Quarantine;
  31
  32 use base qw(PVE::CLIHandler);
  33
  34 sub setup_environment {
  35     PMG::RESTEnvironment->setup_default_cli_env();
  36 }
  37
  38 sub get_item_data {
  39     my ($data, $ref) = @_;
  40
  41     my @lines = split ('\n', $ref->{header});
  42     my $head = new Mail::Header(\@lines);
  43
  44     my $item = {};
  45
  46     $item->{id} = sprintf("C%dR%dT%d", $ref->{cid}, $ref->{rid}, $ref->{ticketid});
  47
  48     $item->{subject} = PMG::Utils::rfc1522_to_html(
  49         PVE::Tools::trim($head->get('subject')) || 'No Subject');
  50
  51     my $from = PMG::Utils::rfc1522_to_html(PVE::Tools::trim($head->get('from') // $ref->{sender}));
  52     my $sender = PMG::Utils::rfc1522_to_html(PVE::Tools::trim($head->get('sender')));
  53
  54     if ($sender) {
  55         $item->{sender} = $sender;
  56         $item->{from} = sprintf ("%s on behalf of %s", $sender, $from);
  57     } else {
  58         $item->{from} = $from;
  59     }
  60
  61     $item->{envelope_sender} = $ref->{sender};
  62     $item->{pmail} = encode_entities(PMG::Utils::try_decode_utf8($ref->{pmail}));
  63     $item->{receiver} = $ref->{receiver} || $ref->{pmail};
  64
  65     $item->{date} = strftime("%F", localtime($ref->{time}));
  66     $item->{time} = strftime("%H:%M:%S", localtime($ref->{time}));
  67
  68     $item->{bytes} = $ref->{bytes};
  69     $item->{spamlevel} = $ref->{spamlevel};
  70     $item->{spaminfo} = $ref->{info};
  71     $item->{file} = $ref->{file};
  72
  73     my $basehref = "$data->{protocol_fqdn_port}/quarantine";
  74     if ($data->{authmode} ne 'ldap') {
  75         my $ticket = uri_escape($data->{ticket});
  76         $item->{href} = "$basehref?ticket=$ticket&cselect=$item->{id}&date=$item->{date}";
  77     } else {
  78         $item->{href} = "$basehref?cselect=$item->{id}&date=$item->{date}";
  79     }
  80
  81     return $item;
  82 }
  83
  84 __PACKAGE__->register_method ({
  85     name => 'status',
  86     path => 'status',
  87     method => 'POST',
  88     description => "Print quarantine status (mails per user) for specified time span.",
  89     parameters => {
  90         additionalProperties => 0,
  91         properties => {
  92             timespan => {
  93                 description => "Select time span.",
  94                 type => 'string',
  95                 enum => ['today', 'yesterday', 'week'],
  96                 default => 'today',
  97                 optional => 1,
  98             },
  99         },
 100     },
 101     returns => { type => 'null'},
 102     code => sub {
 103         my ($param) = @_;
 104
 105         my $cinfo = PMG::ClusterConfig->new();
 106         my $role = $cinfo->{local}->{type} // '-';
 107
 108         if (!(($role eq '-') || ($role eq 'master'))) {
 109            warn "local node is not master\n";
 110            return;
 111         }
 112
 113         my $cfg = PMG::Config->new();
 114
 115         my $timespan = $param->{timespan} // 'today';
 116
 117         my ($start, $end) = PMG::Utils::lookup_timespan($timespan);
 118
 119         my $hostname = PVE::INotify::nodename();
 120
 121         my $fqdn = $cfg->get('spamquar', 'hostname') //
 122             PVE::Tools::get_fqdn($hostname);
 123
 124
 125         my $dbh = PMG::DBTools::open_ruledb();
 126
 127         my $domains = PVE::INotify::read_file('domains');
 128         my $domainregex = PMG::Utils::domain_regex([keys %$domains]);
 129
 130         my $sth = $dbh->prepare(
 131             "SELECT pmail, AVG(spamlevel) as spamlevel, count(*)  FROM CMailStore, CMSReceivers " .
 132             "WHERE time >= $start AND time < $end AND " .
 133             "QType = 'S' AND CID = CMailStore_CID AND RID = CMailStore_RID " .
 134             "AND Status = 'N' " .
 135             "GROUP BY pmail " .
 136             "ORDER BY pmail");
 137
 138         $sth->execute();
 139
 140         print "Count  Spamlevel Mail\n";
 141         my $res = [];
 142         while (my $ref = $sth->fetchrow_hashref()) {
 143             push @$res, $ref;
 144             my $extern = ($domainregex && $ref->{pmail} !~ $domainregex);
 145             my $hint = $extern ? " (external address)" : "";
 146             printf ("%-5d %10.2f %s$hint\n", $ref->{count}, $ref->{spamlevel}, $ref->{pmail});
 147         }
 148
 149         $sth->finish();
 150
 151         return undef;
 152     }});
 153
 154 __PACKAGE__->register_method ({
 155     name => 'send',
 156     path => 'send',
 157     method => 'POST',
 158     description => "Generate and send spam report emails.",
 159     parameters => {
 160         additionalProperties => 0,
 161         properties => {
 162             receiver => get_standard_option('pmg-email-address', {
 163                 description => "Generate report for a single email address. If not specified, generate reports for all users.",
 164                 optional => 1,
 165             }),
 166             timespan => {
 167                 description => "Select time span.",
 168                 type => 'string',
 169                 enum => ['today', 'yesterday', 'week'],
 170                 default => 'today',
 171                 optional => 1,
 172             },
 173             style => {
 174                 description => "Spam report style. Default value is read from spam quarantine configuration.",
 175                 type => 'string',
 176                 enum => ['short', 'verbose', 'custom'],
 177                 optional => 1,
 178             },
 179             redirect => get_standard_option('pmg-email-address', {
 180                 description => "Redirect spam report email to this address.",
 181                 optional => 1,
 182             }),
 183             debug => {
 184                 description => "Debug mode. Print raw email to stdout instead of sending them.",
 185                 type => 'boolean',
 186                 optional => 1,
 187                 default => 0,
 188             }
 189         },
 190     },
 191     returns => { type => 'null'},
 192     code => sub {
 193         my ($param) = @_;
 194
 195         my $cinfo = PMG::ClusterConfig->new();
 196         my $role = $cinfo->{local}->{type} // '-';
 197
 198         if (!(($role eq '-') || ($role eq 'master'))) {
 199            warn "local node is not master - not sending spam report\n";
 200            return;
 201         }
 202
 203         my $cfg = PMG::Config->new();
 204
 205         my $reportstyle = $param->{style} // $cfg->get('spamquar', 'reportstyle');
 206
 207         # overwrite report style none when:
 208         # - explicit receiver specified
 209         # - when debug flag enabled
 210         if ($reportstyle eq 'none') {
 211             $reportstyle = 'verbose' if $param->{debug} || defined($param->{receiver});
 212         }
 213
 214         return if $reportstyle eq 'none'; # do nothing
 215
 216         my $timespan = $param->{timespan} // 'today';
 217
 218         my ($start, $end) = PMG::Utils::lookup_timespan($timespan);
 219
 220         my $hostname = PVE::INotify::nodename();
 221
 222         my $fqdn = $cfg->get('spamquar', 'hostname') //
 223             PVE::Tools::get_fqdn($hostname);
 224
 225         my $port = $cfg->get('spamquar', 'port') // 8006;
 226
 227         my $protocol = $cfg->get('spamquar', 'protocol') // 'https';
 228
 229         my $protocol_fqdn_port = "$protocol://$fqdn";
 230         if (($protocol eq 'https' && $port != 443) ||
 231             ($protocol eq 'http' && $port != 80)) {
 232             $protocol_fqdn_port .= ":$port";
 233         }
 234
 235         my $authmode = $cfg->get ('spamquar', 'authmode') // 'ticket';
 236
 237         my $global_data = {
 238             protocol => $protocol,
 239             port => $port,
 240             fqdn => $fqdn,
 241             hostname => $hostname,
 242             date => strftime("%F", localtime($end - 1)),
 243             timespan => $timespan,
 244             items => [],
 245             protocol_fqdn_port => $protocol_fqdn_port,
 246             authmode => $authmode,
 247         };
 248
 249         my $mailfrom = $cfg->get ('spamquar', 'mailfrom') //
 250             "Proxmox Mail Gateway <postmaster>";
 251
 252         my $dbh = PMG::DBTools::open_ruledb();
 253
 254         my $target = $param->{receiver};
 255         my $redirect = $param->{redirect};
 256
 257         if (defined($redirect) && !defined($target)) {
 258             die "can't redirect mails for all users\n";
 259         }
 260
 261         my $domains = PVE::INotify::read_file('domains');
 262         my $domainregex = PMG::Utils::domain_regex([keys %$domains]);
 263
 264         my $template = "spamreport-${reportstyle}.tt";
 265         my $found = 0;
 266         foreach my $path (@$PMG::Config::tt_include_path) {
 267             if (-f "$path/$template") { $found = 1; last; }
 268         }
 269         if (!$found) {
 270             warn "unable to find template '$template' - using default\n";
 271             $template = "spamreport-verbose.tt";
 272         }
 273
 274         my $sth = $dbh->prepare(
 275             "SELECT * FROM CMailStore, CMSReceivers " .
 276             "WHERE time >= $start AND time < $end AND " .
 277             ($target ? "pmail = ? AND " : '') .
 278             "QType = 'S' AND CID = CMailStore_CID AND RID = CMailStore_RID " .
 279             "AND Status = 'N' " .
 280             "ORDER BY pmail, time, receiver");
 281
 282         if ($target) {
 283             $sth->execute(encode('UTF-8', $target));
 284         } else {
 285             $sth->execute();
 286         }
 287
 288         my $mailcount = 0;
 289         my $creceiver = '';
 290         my $data;
 291
 292         my $tt = PMG::Config::get_template_toolkit();
 293
 294         my $finalize = sub {
 295
 296             my $extern = ($domainregex && $creceiver !~ $domainregex);
 297             if (!$extern) {
 298                 $data->{mailcount} = $mailcount;
 299                 my $sendto = $redirect ? $redirect : $creceiver;
 300                 PMG::Utils::finalize_report($tt, $template, $data, $mailfrom, $sendto, $param->{debug});
 301             }
 302         };
 303
 304         while (my $ref = $sth->fetchrow_hashref()) {
 305             my $decoded_pmail = PMG::Utils::try_decode_utf8($ref->{pmail});
 306             if ($creceiver ne $decoded_pmail) {
 307
 308                 $finalize->() if $data;
 309
 310                 $data = clone($global_data);
 311
 312                 $creceiver = $decoded_pmail;
 313                 $mailcount = 0;
 314
 315                 $data->{pmail} = encode_entities($decoded_pmail);
 316                 $data->{pmail_raw} = $ref->{pmail};
 317                 $data->{managehref} = "$protocol_fqdn_port/quarantine";
 318                 if ($data->{authmode} ne 'ldap') {
 319                     $data->{ticket} = PMG::Ticket::assemble_quarantine_ticket($data->{pmail_raw});
 320                     my $esc_ticket = uri_escape($data->{ticket});
 321                     $data->{managehref} .= "?ticket=${esc_ticket}";
 322                 }
 323
 324             }
 325
 326             push @{$data->{items}}, get_item_data($data, $ref);
 327
 328             $mailcount++;
 329         }
 330
 331         $sth->finish();
 332
 333         $finalize->() if $data;
 334
 335         if (defined($target) && !$mailcount) {
 336             print STDERR "no mails for '$target'\n";
 337         }
 338
 339         return undef;
 340     }});
 341
 342 sub find_stale_files {
 343     my ($path, $lifetime, $purge) = @_;
 344
 345     return if ! -d $path;
 346
 347     my (undef, undef, undef, $mday, $mon, $year) = localtime(time());
 348     my $daystart = timelocal(0, 0, 0, $mday, $mon, $year);
 349     my $expire = $daystart - $lifetime*86400;
 350
 351     my $wanted = sub {
 352         my $name = $File::Find::name;
 353         return if $name !~ m|^($path/.*)$|;
 354         $name = $1; # untaint
 355         my $stat = stat($name);
 356         return if ! -f _;
 357         return if $stat->mtime >= $expire;
 358         if ($purge) {
 359             if (unlink($name)) {
 360                 print "removed: $name\n";
 361             }
 362         } else {
 363             print "$name\n";
 364         }
 365     };
 366
 367     find({ wanted => $wanted, no_chdir => 1 }, $path);
 368 }
 369
 370 sub test_quarantine_files {
 371     my ($spamlifetime, $viruslifetime, $purge) = @_;
 372
 373     print STDERR "searching for stale files\n" if !$purge;
 374
 375     my $spooldir = $PMG::MailQueue::spooldir;
 376
 377     find_stale_files ("$spooldir/spam", $spamlifetime, $purge);
 378     foreach my $dir (<"/var/spool/pmg/cluster/*/spam">) {
 379         next if $dir !~ m|^(/var/spool/pmg/cluster/\d+/spam)$|;
 380         $dir = $1; # untaint
 381         find_stale_files ($dir, $spamlifetime, $purge);
 382     }
 383
 384     find_stale_files ("$spooldir/virus", $viruslifetime, $purge);
 385     foreach my $dir (<"/var/spool/pmg/cluster/*/virus">) {
 386         next if $dir !~ m|^(/var/spool/pmg/cluster/\d+/virus)$|;
 387         $dir = $1; # untaint
 388         find_stale_files ($dir, $viruslifetime, $purge);
 389     }
 390 }
 391
 392 __PACKAGE__->register_method ({
 393     name => 'purge',
 394     path => 'purge',
 395     method => 'POST',
 396     description => "Cleanup Quarantine database. Remove entries older than configured quarantine lifetime.",
 397     parameters => {
 398         additionalProperties => 0,
 399         properties => {
 400             check => {
 401                 description => "Only search for quarantine files older than configured quarantine lifetime. Just print found files, but do not remove them.",
 402                 type => 'boolean',
 403                 optional => 1,
 404                 default => 0,
 405             }
 406         }
 407     },
 408     returns => { type => 'null'},
 409     code => sub {
 410         my ($param) = @_;
 411
 412         my $cfg = PMG::Config->new();
 413
 414         my $spamlifetime = $cfg->get('spamquar', 'lifetime');
 415         my $viruslifetime = $cfg->get ('virusquar', 'lifetime');
 416
 417         my $purge = !$param->{check};
 418
 419         if ($purge) {
 420             print STDERR "purging database\n";
 421
 422             my $dbh = PMG::DBTools::open_ruledb();
 423
 424             if (my $count = PMG::DBTools::purge_quarantine_database($dbh, 'S', $spamlifetime)) {
 425                 print STDERR "removed $count spam quarantine files\n";
 426             }
 427
 428             if (my $count = PMG::DBTools::purge_quarantine_database($dbh, 'V', $viruslifetime)) {
 429                 print STDERR "removed $count virus quarantine files\n";
 430             }
 431
 432             if (my $count = PMG::DBTools::purge_quarantine_database($dbh, 'A', $spamlifetime)) {
 433                 print STDERR "removed $count attachment quarantine files\n";
 434             }
 435         }
 436
 437         test_quarantine_files($spamlifetime, $viruslifetime, $purge);
 438
 439         return undef;
 440     }});
 441
 442
 443 our $cmddef = {
 444     'purge' => [ __PACKAGE__, 'purge', []],
 445     'send' => [ __PACKAGE__, 'send', []],
 446     'status' => [ __PACKAGE__, 'status', []],
 447 };
 448
 449 1;