]> git.proxmox.com Git - pmg-api.git/blame - PMG/CLI/pmgspamreport.pm
pmgspamreport.timer: new timer to send daily spam reports
[pmg-api.git] / PMG / CLI / pmgspamreport.pm
CommitLineData
9a56f771
DM
1package PMG::CLI::pmgspamreport;
2
3use strict;
4use Data::Dumper;
5use Template;
6use MIME::Entity;
7use HTML::Entities;
8use Time::Local;
9use Clone 'clone';
10use Mail::Header;
11use POSIX qw(strftime);
12
13use PVE::SafeSyslog;
14use PVE::Tools;
15use PVE::INotify;
16use PVE::CLIHandler;
17
18use PMG::RESTEnvironment;
19use PMG::Utils;
9a728eba 20use PMG::Ticket;
9a56f771
DM
21use PMG::DBTools;
22use PMG::RuleDB;
23use PMG::Config;
24use PMG::ClusterConfig;
25
26use base qw(PVE::CLIHandler);
27
28sub setup_environment {
29 PMG::RESTEnvironment->setup_default_cli_env();
30}
31
32sub domain_regex {
33 my ($domains) = @_;
34
35 my @ra;
36 foreach my $d (@$domains) {
37 # skip domains with non-DNS name characters
38 next if $d =~ m/[^A-Za-z0-9\-\.]/;
39 if ($d =~ m/^\.(.*)$/) {
40 my $dom = $1;
41 $dom =~ s/\./\\\./g;
42 push @ra, $dom;
43 push @ra, "\.\*\\.$dom";
44 } else {
45 $d =~ s/\./\\\./g;
46 push @ra, $d;
47 }
48 }
49
50 my $re = join ('|', @ra);
51
52 my $regex = qr/\@($re)$/i;
53
54 return $regex;
55}
56
57sub get_item_data {
1d20daff 58 my ($data, $ref) = @_;
9a56f771
DM
59
60 my @lines = split ('\n', $ref->{header});
61 my $head = new Mail::Header(\@lines);
62
1d20daff 63 my $item = {};
9a56f771 64
1d20daff 65 $item->{subject} = PMG::Utils::rfc1522_to_html(
9a56f771
DM
66 PVE::Tools::trim($head->get('subject')) || 'No Subject');
67
68 my @fromarray = split('\s*,\s*', $head->get('from') || $ref->{sender});
69 my $from = PMG::Utils::rfc1522_to_html(PVE::Tools::trim($fromarray[0]));
70 my $sender = PMG::Utils::rfc1522_to_html(PVE::Tools::trim($head->get('sender')));
71
72 if ($sender) {
1d20daff
DM
73 $item->{sender} = $sender;
74 $item->{from} = sprintf ("%s on behalf of %s", $sender, $from);
9a56f771 75 } else {
1d20daff 76 $item->{from} = $from;
9a56f771
DM
77 }
78
1d20daff
DM
79 $item->{pmail} = $ref->{pmail};
80 $item->{receiver} = $ref->{receiver} || $ref->{pmail};
9a56f771 81
1d20daff
DM
82 $item->{date} = strftime("%F", localtime($ref->{time}));
83 $item->{time} = strftime("%H:%M:%S", localtime($ref->{time}));
9a56f771 84
1d20daff
DM
85 $item->{bytes} = $ref->{bytes};
86 $item->{spamlevel} = $ref->{spamlevel};
87 $item->{spaminfo} = $ref->{info};
9a56f771 88
1d20daff 89 my $title = "Received: $item->{date} $item->{time}\n";
9a56f771
DM
90 $title .= "From: $ref->{sender}\n";
91 $title .= "To: $ref->{receiver}\n" if $ref->{receiver};
92 $title .= sprintf("Size: %d KB\n", int (($ref->{bytes} + 1023) / 1024 ));
93 $title .= sprintf("Spam level: %d\n", $ref->{spamlevel}) if $ref->{qtype} eq 'S';
94 $title .= sprintf("Virus info: %s\n", encode_entities ($ref->{info})) if $ref->{qtype} eq 'V';
95 $title .= sprintf("File: %s", encode_entities($ref->{file}));
96
1d20daff 97 $item->{title} = $title;
9b0428e2 98
1d20daff 99 $item->{ticket} = PMG::Ticket::assemble_quarantine_ticket($ref);
9a728eba 100
1d20daff
DM
101 my $basehref = "https://$data->{fqdn}:$data->{port}/userquar";
102
103 $item->{wlhref} = "$basehref?ticket=$item->{ticket}&cselect=$item->{ticket}&whitelist=1";
104 $item->{blhref} = "$basehref?ticket=$item->{ticket}&cselect=$item->{ticket}&blacklist=1";
105 $item->{deliverhref} = "$basehref?ticket=$item->{ticket}&cselect=$item->{ticket}&deliver=1";
106 $item->{deletehref} = "$basehref?ticket=$item->{ticket}&cselect=$item->{ticket}&delete=1";
107
108 return $item;
9a56f771
DM
109}
110
111sub finalize_report {
78ee390d 112 my ($tt, $template, $data, $mailfrom, $receiver, $debug) = @_;
9a56f771
DM
113
114 my $html = '';
115
78ee390d 116 $tt->process($template, $data, \$html) ||
60f82a46 117 die $tt->error() . "\n";
9a56f771
DM
118
119 my $title;
120 if ($html =~ m|^\s*<title>(.*)</title>|m) {
121 $title = $1;
122 } else {
123 die "unable to extract template title\n";
124 }
125
126 my $top = MIME::Entity->build(
127 Type => "multipart/related",
128 To => $data->{pmail},
129 From => $mailfrom,
130 Subject => PMG::Utils::bencode_header(decode_entities($title)));
131
132 $top->attach(
133 Data => $html,
134 Type => "text/html",
f73a1872 135 Encoding => $debug ? 'binary' : 'quoted-printable');
9a56f771 136
f73a1872
DM
137 if ($debug) {
138 $top->print();
139 return;
140 }
9a56f771
DM
141 # we use an empty envelope sender (we dont want to receive NDRs)
142 PMG::Utils::reinject_mail ($top, '', [$receiver], undef, $data->{fqdn});
143}
144
145__PACKAGE__->register_method ({
146 name => 'send',
147 path => 'send',
148 method => 'POST',
149 description => "Generate and send spam report emails.",
150 parameters => {
151 additionalProperties => 0,
152 properties => {
153 receiver => {
154 description => "Generate report for a single email address. If not specified, generate reports for all users.",
155 type => 'string', format => 'email',
156 optional => 1,
157 },
158 timespan => {
159 description => "Select time span.",
160 type => 'string',
161 enum => ['today', 'yesterday', 'week'],
162 default => 'today',
163 optional => 1,
164 },
165 style => {
f73a1872 166 description => "Spam report style. Value 'none' just prints the spam counts and does not send any emails. Default value is read from spam quarantine configuration.",
9a56f771 167 type => 'string',
77f73dc3 168 enum => ['none', 'short', 'verbose', 'custom'],
9a56f771 169 optional => 1,
9a56f771
DM
170 },
171 redirect => {
172 description => "Redirect spam report email to this address.",
173 type => 'string', format => 'email',
174 optional => 1,
175 },
f73a1872
DM
176 debug => {
177 description => "Debug mode. Print raw email to stdout instead of sending them.",
178 type => 'boolean',
179 optional => 1,
180 default => 0,
181 }
9a56f771
DM
182 },
183 },
184 returns => { type => 'null'},
185 code => sub {
186 my ($param) = @_;
187
188 my $cinfo = PMG::ClusterConfig->new();
189 my $role = $cinfo->{local}->{type} // '-';
190
191 if (!(($role eq '-') || ($role eq 'master'))) {
192 die "local node is not master - not sending spam report\n";
193 }
194
195 my $cfg = PMG::Config->new();
196
f73a1872 197 my $reportstyle = $param->{style} // $cfg->get('spamquar', 'reportstyle');
9a56f771
DM
198
199 my $timespan = $param->{timespan} // 'today';
200
201 my (undef, undef, undef, $mday, $mon, $year) = localtime(time());
202 my $daystart = timelocal(0, 0, 0, $mday, $mon, $year);
203
204 my $start;
205 my $end;
206
207 if ($timespan eq 'today') {
208 $start = $daystart;
209 $end = $start + 86400;
210 } elsif ($timespan eq 'yesterday') {
211 $end = $daystart;
212 $start = $end - 86400;
213 } elsif ($timespan eq 'week') {
214 $end = $daystart;
215 $start = $end - 7*86400;
216 } else {
217 die "internal error";
218 }
219
220 my $hostname = PVE::INotify::nodename();
221
222 my $fqdn = $cfg->get('spamquar', 'hostname') //
223 PVE::Tools::get_fqdn($hostname);
224
225 my $port = 8006;
226
227 my $global_data = {
228 protocol => 'https',
229 port => $port,
230 fqdn => $fqdn,
231 hostname => $hostname,
9a56f771
DM
232 date => strftime("%F", localtime($end - 1)),
233 timespan => $timespan,
234 items => [],
235 };
236
237 my $mailfrom = $cfg->get ('spamquar', 'mailfrom') //
238 "Proxmox Mail Gateway <postmaster>";
239
240 my $dbh = PMG::DBTools::open_ruledb();
241
242 my $target = $param->{receiver};
243 my $redirect = $param->{redirect};
244
1388c118 245 if (defined($redirect) && !defined($target)) {
9a56f771
DM
246 die "can't redirect mails for all users\n";
247 }
248
249 my $domains = PVE::INotify::read_file('domains');
250 my $domainregex = domain_regex([keys %$domains]);
251
78ee390d
DM
252 my $template;
253
9a56f771
DM
254 if ($reportstyle ne 'none') {
255
dc41e7db 256 $template = "spamreport-${reportstyle}.tt";
78ee390d 257 my $found = 0;
310daf18 258 foreach my $path (@$PMG::Config::tt_include_path) {
78ee390d
DM
259 if (-f "$path/$template") { $found = 1; last; }
260 }
261 if (!$found) {
262 warn "unable to find template '$template' - using default\n";
dc41e7db 263 $template = "spamreport-verbose.tt";
9a56f771 264 }
9a56f771 265 }
78ee390d 266
9a56f771
DM
267 my $sth = $dbh->prepare(
268 "SELECT * FROM CMailStore, CMSReceivers " .
269 "WHERE time >= $start AND time < $end AND " .
270 ($target ? "pmail = ? AND " : '') .
271 "QType = 'S' AND CID = CMailStore_CID AND RID = CMailStore_RID " .
272 "AND Status = 'N' " .
273 "ORDER BY pmail, time, receiver");
274
275 if ($target) {
276 $sth->execute($target);
277 } else {
278 $sth->execute();
279 }
280
281 my $lastref;
282 my $mailcount = 0;
283 my $creceiver = '';
284 my $data;
78ee390d 285
310daf18
DM
286 my $tt = PMG::Config::get_template_toolkit();
287
9a56f771
DM
288 my $finalize = sub {
289
290 my $extern = ($domainregex && $creceiver !~ $domainregex);
291
78ee390d 292 if ($template) {
9a56f771 293 if (!$extern) {
9a728eba 294 my $ticket = PMG::Ticket::assemble_quarantine_ticket($lastref);
9a56f771 295 $data->{ticket} = $ticket;
f003ceb3 296 $data->{managehref} = "https://$fqdn:$port?ticket=$ticket";
9a56f771
DM
297 $data->{mailcount} = $mailcount;
298
299 my $sendto = $redirect ? $redirect : $creceiver;
78ee390d 300 finalize_report($tt, $template, $data, $mailfrom, $sendto, $param->{debug});
9a56f771
DM
301 }
302 } else {
303 my $hint = $extern ? " (external address)" : "";
304 printf ("%-5d %s$hint\n", $mailcount, $creceiver);
305 }
306 };
307
308 while (my $ref = $sth->fetchrow_hashref()) {
309 if ($creceiver ne $ref->{pmail}) {
310
311 $finalize->() if $data;
312
313 $data = clone($global_data);
314
315 $creceiver = $ref->{pmail};
316 $mailcount = 0;
317
318 $data->{pmail} = $creceiver;
319 }
320
78ee390d 321 if ($template) {
1d20daff 322 push @{$data->{items}}, get_item_data($data, $ref);
9a56f771
DM
323 }
324
325 $mailcount++;
326 $lastref = $ref;
327 }
328
329 $sth->finish();
330
331 $finalize->() if $data;
332
333 if (defined($target) && !$mailcount) {
334 print STDERR "no mails for '$target'\n";
335 }
336
337 return undef;
338 }});
339
340sub find_stale_files {
341 my ($path, $lifetime, $purge) = @_;
342
343 my $cmd = ['find', $path, '-daystart', '-mtime', '+$lifetime',
344 '-type', 'f'];
345
346 if ($purge) {
347 push @$cmd, '-exec', 'rm', '-vf', '{}', ';';
348 } else {
349 push @$cmd, '-print';
350 }
351
352 PVE::Tools::run_command($cmd);
353}
354
355sub test_quarantine_files {
356 my ($spamlifetime, $viruslifetime, $purge) = @_;
357
358 print STDERR "searching for stale files\n" if !$purge;
359
360 find_stale_files ("/var/spool/proxmox/spam", $spamlifetime, $purge);
361 find_stale_files ("/var/spool/proxmox/cluster/*/spam", $spamlifetime, $purge);
362
363 find_stale_files ("/var/spool/proxmox/virus", $viruslifetime, $purge);
364 find_stale_files ("/var/spool/proxmox/cluster/*/virus", $viruslifetime, $purge);
365}
366
367__PACKAGE__->register_method ({
368 name => 'purge',
369 path => 'purge',
370 method => 'POST',
371 description => "Cleanup Quarantine database. Remove entries older than configured quarantine lifetime.",
372 parameters => {
373 additionalProperties => 0,
374 },
375 returns => { type => 'null'},
376 code => sub {
377 my ($param) = @_;
378
379 my $cfg = PMG::Config->new();
380
381 my $spamlifetime = $cfg->get('spamquar', 'lifetime');
382 my $viruslifetime = $cfg->get ('virusquar', 'lifetime');
383
384 print STDERR "purging database\n";
385
386 my $dbh = PMG::DBTools::open_ruledb();
387
388 if (my $count = PMG::DBTools::purge_quarantine_database($dbh, 'S', $spamlifetime)) {
389 print STDERR "removed $count spam quarantine files\n";
390 }
391
392 if (my $count = PMG::DBTools::purge_quarantine_database($dbh, 'V', $viruslifetime)) {
393 print STDERR "removed $count virus quarantine files\n";
394 }
395
396 test_quarantine_files($spamlifetime, $viruslifetime, 1);
397
398 return undef;
399 }});
400
401__PACKAGE__->register_method ({
402 name => 'check',
403 path => 'check',
404 method => 'GET',
405 description => "Search Quarantine database for entries older than configured quarantine lifetime.",
406 parameters => {
407 additionalProperties => 0,
408 },
409 returns => { type => 'null'},
410 code => sub {
411 my ($param) = @_;
412
413 my $cfg = PMG::Config->new();
414
415 my $spamlifetime = $cfg->get('spamquar', 'lifetime');
416 my $viruslifetime = $cfg->get ('virusquar', 'lifetime');
417
418 test_quarantine_files($spamlifetime, $viruslifetime, 0);
419
420 return undef;
421 }});
422
423
424
425our $cmddef = {
426 'check' => [ __PACKAGE__, 'check', []],
427 'purge' => [ __PACKAGE__, 'purge', []],
428 'send' => [ __PACKAGE__, 'send', []],
429};
430
4311;