]> git.proxmox.com Git - pmg-api.git/blob - src/PMG/CLI/pmgqm.pm
dkim: add QID in warnings
[pmg-api.git] / src / PMG / CLI / pmgqm.pm
1 package PMG::CLI::pmgqm;
2
3 use strict;
4 use Data::Dumper;
5 use Template;
6 use MIME::Entity;
7 use HTML::Entities;
8 use Time::Local;
9 use Clone 'clone';
10 use Mail::Header;
11 use POSIX qw(strftime);
12 use File::Find;
13 use File::stat;
14 use URI::Escape;
15
16 use PVE::SafeSyslog;
17 use PVE::Tools;
18 use PVE::INotify;
19 use PVE::CLIHandler;
20
21 use PMG::RESTEnvironment;
22 use PMG::Utils;
23 use PMG::Ticket;
24 use PMG::DBTools;
25 use PMG::RuleDB;
26 use PMG::Config;
27 use PMG::ClusterConfig;
28 use PMG::API2::Quarantine;
29
30 use base qw(PVE::CLIHandler);
31
32 sub setup_environment {
33 PMG::RESTEnvironment->setup_default_cli_env();
34 }
35
36 sub domain_regex {
37 my ($domains) = @_;
38
39 my @ra;
40 foreach my $d (@$domains) {
41 # skip domains with non-DNS name characters
42 next if $d =~ m/[^A-Za-z0-9\-\.]/;
43 if ($d =~ m/^\.(.*)$/) {
44 my $dom = $1;
45 $dom =~ s/\./\\\./g;
46 push @ra, $dom;
47 push @ra, "\.\*\\.$dom";
48 } else {
49 $d =~ s/\./\\\./g;
50 push @ra, $d;
51 }
52 }
53
54 my $re = join ('|', @ra);
55
56 my $regex = qr/\@($re)$/i;
57
58 return $regex;
59 }
60
61 sub get_item_data {
62 my ($data, $ref) = @_;
63
64 my @lines = split ('\n', $ref->{header});
65 my $head = new Mail::Header(\@lines);
66
67 my $item = {};
68
69 $item->{id} = sprintf("C%dR%dT%d", $ref->{cid}, $ref->{rid}, $ref->{ticketid});
70
71 $item->{subject} = PMG::Utils::rfc1522_to_html(
72 PVE::Tools::trim($head->get('subject')) || 'No Subject');
73
74 my @fromarray = split('\s*,\s*', $head->get('from') || $ref->{sender});
75 my $from = PMG::Utils::rfc1522_to_html(PVE::Tools::trim($fromarray[0]));
76 my $sender = PMG::Utils::rfc1522_to_html(PVE::Tools::trim($head->get('sender')));
77
78 if ($sender) {
79 $item->{sender} = $sender;
80 $item->{from} = sprintf ("%s on behalf of %s", $sender, $from);
81 } else {
82 $item->{from} = $from;
83 }
84
85 $item->{envelope_sender} = $ref->{sender};
86 $item->{pmail} = $ref->{pmail};
87 $item->{receiver} = $ref->{receiver} || $ref->{pmail};
88
89 $item->{date} = strftime("%F", localtime($ref->{time}));
90 $item->{time} = strftime("%H:%M:%S", localtime($ref->{time}));
91
92 $item->{bytes} = $ref->{bytes};
93 $item->{spamlevel} = $ref->{spamlevel};
94 $item->{spaminfo} = $ref->{info};
95 $item->{file} = $ref->{file};
96
97 my $basehref = "$data->{protocol}://$data->{fqdn}:$data->{port}/quarantine";
98 my $ticket = uri_escape($data->{ticket});
99 $item->{href} = "$basehref?ticket=$ticket&cselect=$item->{id}&date=$item->{date}";
100
101 return $item;
102 }
103
104 __PACKAGE__->register_method ({
105 name => 'status',
106 path => 'status',
107 method => 'POST',
108 description => "Print quarantine status (mails per user) for specified time span.",
109 parameters => {
110 additionalProperties => 0,
111 properties => {
112 timespan => {
113 description => "Select time span.",
114 type => 'string',
115 enum => ['today', 'yesterday', 'week'],
116 default => 'today',
117 optional => 1,
118 },
119 },
120 },
121 returns => { type => 'null'},
122 code => sub {
123 my ($param) = @_;
124
125 my $cinfo = PMG::ClusterConfig->new();
126 my $role = $cinfo->{local}->{type} // '-';
127
128 if (!(($role eq '-') || ($role eq 'master'))) {
129 die "local node is not mastern";
130 }
131
132 my $cfg = PMG::Config->new();
133
134 my $timespan = $param->{timespan} // 'today';
135
136 my ($start, $end) = PMG::Utils::lookup_timespan($timespan);
137
138 my $hostname = PVE::INotify::nodename();
139
140 my $fqdn = $cfg->get('spamquar', 'hostname') //
141 PVE::Tools::get_fqdn($hostname);
142
143
144 my $dbh = PMG::DBTools::open_ruledb();
145
146 my $domains = PVE::INotify::read_file('domains');
147 my $domainregex = domain_regex([keys %$domains]);
148
149 my $sth = $dbh->prepare(
150 "SELECT pmail, AVG(spamlevel) as spamlevel, count(*) FROM CMailStore, CMSReceivers " .
151 "WHERE time >= $start AND time < $end AND " .
152 "QType = 'S' AND CID = CMailStore_CID AND RID = CMailStore_RID " .
153 "AND Status = 'N' " .
154 "GROUP BY pmail " .
155 "ORDER BY pmail");
156
157 $sth->execute();
158
159 print "Count Spamlevel Mail\n";
160 my $res = [];
161 while (my $ref = $sth->fetchrow_hashref()) {
162 push @$res, $ref;
163 my $extern = ($domainregex && $ref->{pmail} !~ $domainregex);
164 my $hint = $extern ? " (external address)" : "";
165 printf ("%-5d %10.2f %s$hint\n", $ref->{count}, $ref->{spamlevel}, $ref->{pmail});
166 }
167
168 $sth->finish();
169
170 return undef;
171 }});
172
173 __PACKAGE__->register_method ({
174 name => 'send',
175 path => 'send',
176 method => 'POST',
177 description => "Generate and send spam report emails.",
178 parameters => {
179 additionalProperties => 0,
180 properties => {
181 receiver => {
182 description => "Generate report for a single email address. If not specified, generate reports for all users.",
183 type => 'string', format => 'email',
184 optional => 1,
185 },
186 timespan => {
187 description => "Select time span.",
188 type => 'string',
189 enum => ['today', 'yesterday', 'week'],
190 default => 'today',
191 optional => 1,
192 },
193 style => {
194 description => "Spam report style. Default value is read from spam quarantine configuration.",
195 type => 'string',
196 enum => ['short', 'verbose', 'custom'],
197 optional => 1,
198 },
199 redirect => {
200 description => "Redirect spam report email to this address.",
201 type => 'string', format => 'email',
202 optional => 1,
203 },
204 debug => {
205 description => "Debug mode. Print raw email to stdout instead of sending them.",
206 type => 'boolean',
207 optional => 1,
208 default => 0,
209 }
210 },
211 },
212 returns => { type => 'null'},
213 code => sub {
214 my ($param) = @_;
215
216 my $cinfo = PMG::ClusterConfig->new();
217 my $role = $cinfo->{local}->{type} // '-';
218
219 if (!(($role eq '-') || ($role eq 'master'))) {
220 die "local node is not master - not sending spam report\n";
221 }
222
223 my $cfg = PMG::Config->new();
224
225 my $reportstyle = $param->{style} // $cfg->get('spamquar', 'reportstyle');
226
227 # overwrite report style none when:
228 # - explicit receiver specified
229 # - when debug flag enabled
230 if ($reportstyle eq 'none') {
231 $reportstyle = 'verbose' if $param->{debug} || defined($param->{receiver});
232 }
233
234 return if $reportstyle eq 'none'; # do nothing
235
236 my $timespan = $param->{timespan} // 'today';
237
238 my ($start, $end) = PMG::Utils::lookup_timespan($timespan);
239
240 my $hostname = PVE::INotify::nodename();
241
242 my $fqdn = $cfg->get('spamquar', 'hostname') //
243 PVE::Tools::get_fqdn($hostname);
244
245 my $port = $cfg->get('spamquar', 'port') // 8006;
246
247 my $protocol = $cfg->get('spamquar', 'protocol') // 'https';
248
249 my $global_data = {
250 protocol => $protocol,
251 port => $port,
252 fqdn => $fqdn,
253 hostname => $hostname,
254 date => strftime("%F", localtime($end - 1)),
255 timespan => $timespan,
256 items => [],
257 };
258
259 my $mailfrom = $cfg->get ('spamquar', 'mailfrom') //
260 "Proxmox Mail Gateway <postmaster>";
261
262 my $dbh = PMG::DBTools::open_ruledb();
263
264 my $target = $param->{receiver};
265 my $redirect = $param->{redirect};
266
267 if (defined($redirect) && !defined($target)) {
268 die "can't redirect mails for all users\n";
269 }
270
271 my $domains = PVE::INotify::read_file('domains');
272 my $domainregex = domain_regex([keys %$domains]);
273
274 my $template = "spamreport-${reportstyle}.tt";
275 my $found = 0;
276 foreach my $path (@$PMG::Config::tt_include_path) {
277 if (-f "$path/$template") { $found = 1; last; }
278 }
279 if (!$found) {
280 warn "unable to find template '$template' - using default\n";
281 $template = "spamreport-verbose.tt";
282 }
283
284 my $sth = $dbh->prepare(
285 "SELECT * FROM CMailStore, CMSReceivers " .
286 "WHERE time >= $start AND time < $end AND " .
287 ($target ? "pmail = ? AND " : '') .
288 "QType = 'S' AND CID = CMailStore_CID AND RID = CMailStore_RID " .
289 "AND Status = 'N' " .
290 "ORDER BY pmail, time, receiver");
291
292 if ($target) {
293 $sth->execute($target);
294 } else {
295 $sth->execute();
296 }
297
298 my $lastref;
299 my $mailcount = 0;
300 my $creceiver = '';
301 my $data;
302
303 my $tt = PMG::Config::get_template_toolkit();
304
305 my $finalize = sub {
306
307 my $extern = ($domainregex && $creceiver !~ $domainregex);
308 if (!$extern) {
309 $data->{mailcount} = $mailcount;
310 my $sendto = $redirect ? $redirect : $creceiver;
311 PMG::Utils::finalize_report($tt, $template, $data, $mailfrom, $sendto, $param->{debug});
312 }
313 };
314
315 while (my $ref = $sth->fetchrow_hashref()) {
316 if ($creceiver ne $ref->{pmail}) {
317
318 $finalize->() if $data;
319
320 $data = clone($global_data);
321
322 $creceiver = $ref->{pmail};
323 $mailcount = 0;
324
325 $data->{pmail} = $creceiver;
326 $data->{ticket} = PMG::Ticket::assemble_quarantine_ticket($data->{pmail});
327 my $esc_ticket = uri_escape($data->{ticket});
328 $data->{managehref} = "$protocol://$fqdn:$port/quarantine?ticket=${esc_ticket}";
329 }
330
331 push @{$data->{items}}, get_item_data($data, $ref);
332
333 $mailcount++;
334 $lastref = $ref;
335 }
336
337 $sth->finish();
338
339 $finalize->() if $data;
340
341 if (defined($target) && !$mailcount) {
342 print STDERR "no mails for '$target'\n";
343 }
344
345 return undef;
346 }});
347
348 sub find_stale_files {
349 my ($path, $lifetime, $purge) = @_;
350
351 return if ! -d $path;
352
353 my (undef, undef, undef, $mday, $mon, $year) = localtime(time());
354 my $daystart = timelocal(0, 0, 0, $mday, $mon, $year);
355 my $expire = $daystart - $lifetime*86400;
356
357 my $wanted = sub {
358 my $name = $File::Find::name;
359 return if $name !~ m|^($path/.*)$|;
360 $name = $1; # untaint
361 my $stat = stat($name);
362 return if ! -f _;
363 return if $stat->mtime >= $expire;
364 if ($purge) {
365 if (unlink($name)) {
366 print "removed: $name\n";
367 }
368 } else {
369 print "$name\n";
370 }
371 };
372
373 find({ wanted => $wanted, no_chdir => 1 }, $path);
374 }
375
376 sub test_quarantine_files {
377 my ($spamlifetime, $viruslifetime, $purge) = @_;
378
379 print STDERR "searching for stale files\n" if !$purge;
380
381 my $spooldir = $PMG::MailQueue::spooldir;
382
383 find_stale_files ("$spooldir/spam", $spamlifetime, $purge);
384 foreach my $dir (<"/var/spool/pmg/cluster/*/spam">) {
385 next if $dir !~ m|^(/var/spool/pmg/cluster/\d+/spam)$|;
386 $dir = $1; # untaint
387 find_stale_files ($dir, $spamlifetime, $purge);
388 }
389
390 find_stale_files ("$spooldir/virus", $viruslifetime, $purge);
391 foreach my $dir (<"/var/spool/pmg/cluster/*/virus">) {
392 next if $dir !~ m|^(/var/spool/pmg/cluster/\d+/virus)$|;
393 $dir = $1; # untaint
394 find_stale_files ($dir, $viruslifetime, $purge);
395 }
396 }
397
398 __PACKAGE__->register_method ({
399 name => 'purge',
400 path => 'purge',
401 method => 'POST',
402 description => "Cleanup Quarantine database. Remove entries older than configured quarantine lifetime.",
403 parameters => {
404 additionalProperties => 0,
405 properties => {
406 check => {
407 description => "Only search for quarantine files older than configured quarantine lifetime. Just print found files, but do not remove them.",
408 type => 'boolean',
409 optional => 1,
410 default => 0,
411 }
412 }
413 },
414 returns => { type => 'null'},
415 code => sub {
416 my ($param) = @_;
417
418 my $cfg = PMG::Config->new();
419
420 my $spamlifetime = $cfg->get('spamquar', 'lifetime');
421 my $viruslifetime = $cfg->get ('virusquar', 'lifetime');
422
423 my $purge = !$param->{check};
424
425 if ($purge) {
426 print STDERR "purging database\n";
427
428 my $dbh = PMG::DBTools::open_ruledb();
429
430 if (my $count = PMG::DBTools::purge_quarantine_database($dbh, 'S', $spamlifetime)) {
431 print STDERR "removed $count spam quarantine files\n";
432 }
433
434 if (my $count = PMG::DBTools::purge_quarantine_database($dbh, 'V', $viruslifetime)) {
435 print STDERR "removed $count virus quarantine files\n";
436 }
437
438 if (my $count = PMG::DBTools::purge_quarantine_database($dbh, 'A', $spamlifetime)) {
439 print STDERR "removed $count attachment quarantine files\n";
440 }
441 }
442
443 test_quarantine_files($spamlifetime, $viruslifetime, $purge);
444
445 return undef;
446 }});
447
448
449 our $cmddef = {
450 'purge' => [ __PACKAGE__, 'purge', []],
451 'send' => [ __PACKAGE__, 'send', []],
452 'status' => [ __PACKAGE__, 'status', []],
453 };
454
455 1;