]> git.proxmox.com Git - pmg-api.git/blob - PMG/DBTools.pm
depend on: openssh-client, openssh-server, apt
[pmg-api.git] / PMG / DBTools.pm
1 package PMG::DBTools;
2
3 use strict;
4 use warnings;
5
6 use POSIX ":sys_wait_h";
7 use POSIX ':signal_h';
8 use DBI;
9 use Time::Local;
10
11 use PVE::SafeSyslog;
12 use PVE::Tools;
13
14 use PMG::Utils;
15 use PMG::RuleDB;
16 use PMG::MailQueue;
17 use PMG::Config;
18
19 our $default_db_name = "Proxmox_ruledb";
20
21 our $cgreylist_merge_sql =
22 'INSERT INTO CGREYLIST (IPNet,Host,Sender,Receiver,Instance,RCTime,' .
23 'ExTime,Delay,Blocked,Passed,MTime,CID) ' .
24 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ' .
25 'ON CONFLICT (IPNet,Sender,Receiver) DO UPDATE SET ' .
26 'Host = CASE WHEN CGREYLIST.MTime >= excluded.MTime THEN CGREYLIST.Host ELSE excluded.Host END,' .
27 'CID = GREATEST(CGREYLIST.CID, excluded.CID), RCTime = LEAST(CGREYLIST.RCTime, excluded.RCTime),' .
28 'ExTime = GREATEST(CGREYLIST.ExTime, excluded.ExTime),' .
29 'Delay = GREATEST(CGREYLIST.Delay, excluded.Delay),' .
30 'Blocked = GREATEST(CGREYLIST.Blocked, excluded.Blocked),' .
31 'Passed = GREATEST(CGREYLIST.Passed, excluded.Passed)';
32
33 sub open_ruledb {
34 my ($database, $host, $port) = @_;
35
36 $port //= 5432;
37
38 $database //= $default_db_name;
39
40 if ($host) {
41
42 # Note: pmgtunnel uses UDP sockets inside directory '/var/run/pmgtunnel',
43 # and the cluster 'cid' as port number. You can connect to the
44 # socket with: host => /var/run/pmgtunnel, port => $cid
45
46 my $dsn = "dbi:Pg:dbname=$database;host=$host;port=$port;";
47
48 my $timeout = 5;
49 # only low level alarm interface works for DBI->connect
50 my $mask = POSIX::SigSet->new(SIGALRM);
51 my $action = POSIX::SigAction->new(sub { die "connect timeout\n" }, $mask);
52 my $oldaction = POSIX::SigAction->new();
53 sigaction(SIGALRM, $action, $oldaction);
54
55 my $rdb;
56
57 eval {
58 alarm($timeout);
59 $rdb = DBI->connect($dsn, 'root', undef,
60 { PrintError => 0, RaiseError => 1 });
61 alarm(0);
62 };
63 alarm(0);
64 sigaction(SIGALRM, $oldaction); # restore original handler
65
66 die $@ if $@;
67
68 return $rdb;
69 } else {
70 my $dsn = "DBI:Pg:dbname=$database;host=/var/run/postgresql;port=$port";
71
72 my $dbh = DBI->connect($dsn, $> == 0 ? 'root' : 'www-data', undef,
73 { PrintError => 0, RaiseError => 1 });
74
75 return $dbh;
76 }
77 }
78
79 sub postgres_admin_cmd {
80 my ($cmd, $options, @params) = @_;
81
82 $cmd = ref($cmd) ? $cmd : [ $cmd ];
83 my $uid = getpwnam('postgres') || die "getpwnam postgres failed\n";
84
85 local $> = $uid;
86 $! && die "setuid postgres ($uid) failed - $!\n";
87
88 PVE::Tools::run_command([@$cmd, '-U', 'postgres', @params], %$options);
89 }
90
91 sub delete_ruledb {
92 my ($dbname) = @_;
93
94 postgres_admin_cmd('dropdb', undef, $dbname);
95 }
96
97 sub database_list {
98
99 my $database_list = {};
100
101 my $parser = sub {
102 my $line = shift;
103
104 my ($name, $owner) = map { PVE::Tools::trim($_) } split(/\|/, $line);
105 return if !$name || !$owner;
106
107 $database_list->{$name} = { owner => $owner };
108 };
109
110 postgres_admin_cmd('psql', { outfunc => $parser }, '--list', '--quiet', '--tuples-only');
111
112 return $database_list;
113 }
114
115 my $cgreylist_ctablecmd = <<__EOD;
116 CREATE TABLE CGreylist
117 (IPNet VARCHAR(16) NOT NULL,
118 Host INTEGER NOT NULL,
119 Sender VARCHAR(255) NOT NULL,
120 Receiver VARCHAR(255) NOT NULL,
121 Instance VARCHAR(255),
122 RCTime INTEGER NOT NULL,
123 ExTime INTEGER NOT NULL,
124 Delay INTEGER NOT NULL DEFAULT 0,
125 Blocked INTEGER NOT NULL,
126 Passed INTEGER NOT NULL,
127 CID INTEGER NOT NULL,
128 MTime INTEGER NOT NULL,
129 PRIMARY KEY (IPNet, Sender, Receiver));
130
131 CREATE INDEX CGreylist_Instance_Sender_Index ON CGreylist (Instance, Sender);
132
133 CREATE INDEX CGreylist_ExTime_Index ON CGreylist (ExTime);
134
135 CREATE INDEX CGreylist_MTime_Index ON CGreylist (MTime);
136 __EOD
137
138 my $clusterinfo_ctablecmd = <<__EOD;
139 CREATE TABLE ClusterInfo
140 (CID INTEGER NOT NULL,
141 Name VARCHAR NOT NULL,
142 IValue INTEGER,
143 SValue VARCHAR,
144 PRIMARY KEY (CID, Name))
145 __EOD
146
147 my $local_stat_ctablecmd = <<__EOD;
148 CREATE TABLE LocalStat
149 (Time INTEGER NOT NULL UNIQUE,
150 RBLCount INTEGER DEFAULT 0 NOT NULL,
151 CID INTEGER NOT NULL,
152 MTime INTEGER NOT NULL,
153 PRIMARY KEY (Time, CID));
154
155 CREATE INDEX LocalStat_MTime_Index ON LocalStat (MTime);
156 __EOD
157
158
159 my $daily_stat_ctablecmd = <<__EOD;
160 CREATE TABLE DailyStat
161 (Time INTEGER NOT NULL UNIQUE,
162 CountIn INTEGER NOT NULL,
163 CountOut INTEGER NOT NULL,
164 BytesIn REAL NOT NULL,
165 BytesOut REAL NOT NULL,
166 VirusIn INTEGER NOT NULL,
167 VirusOut INTEGER NOT NULL,
168 SpamIn INTEGER NOT NULL,
169 SpamOut INTEGER NOT NULL,
170 BouncesIn INTEGER NOT NULL,
171 BouncesOut INTEGER NOT NULL,
172 GreylistCount INTEGER NOT NULL,
173 SPFCount INTEGER NOT NULL,
174 PTimeSum REAL NOT NULL,
175 MTime INTEGER NOT NULL,
176 RBLCount INTEGER DEFAULT 0 NOT NULL,
177 PRIMARY KEY (Time));
178
179 CREATE INDEX DailyStat_MTime_Index ON DailyStat (MTime);
180
181 __EOD
182
183 my $domain_stat_ctablecmd = <<__EOD;
184 CREATE TABLE DomainStat
185 (Time INTEGER NOT NULL,
186 Domain VARCHAR(255) NOT NULL,
187 CountIn INTEGER NOT NULL,
188 CountOut INTEGER NOT NULL,
189 BytesIn REAL NOT NULL,
190 BytesOut REAL NOT NULL,
191 VirusIn INTEGER NOT NULL,
192 VirusOut INTEGER NOT NULL,
193 SpamIn INTEGER NOT NULL,
194 SpamOut INTEGER NOT NULL,
195 BouncesIn INTEGER NOT NULL,
196 BouncesOut INTEGER NOT NULL,
197 PTimeSum REAL NOT NULL,
198 MTime INTEGER NOT NULL,
199 PRIMARY KEY (Time, Domain));
200
201 CREATE INDEX DomainStat_MTime_Index ON DomainStat (MTime);
202 __EOD
203
204 my $statinfo_ctablecmd = <<__EOD;
205 CREATE TABLE StatInfo
206 (Name VARCHAR(255) NOT NULL UNIQUE,
207 IValue INTEGER,
208 SValue VARCHAR(255),
209 PRIMARY KEY (Name))
210 __EOD
211
212 my $virusinfo_stat_ctablecmd = <<__EOD;
213 CREATE TABLE VirusInfo
214 (Time INTEGER NOT NULL,
215 Name VARCHAR NOT NULL,
216 Count INTEGER NOT NULL,
217 MTime INTEGER NOT NULL,
218 PRIMARY KEY (Time, Name));
219
220 CREATE INDEX VirusInfo_MTime_Index ON VirusInfo (MTime);
221
222 __EOD
223
224 # mail storage stable
225 # QTypes
226 # V - Virus quarantine
227 # S - Spam quarantine
228 # D - Delayed Mails - not implemented
229 # A - Held for Audit - not implemented
230 # Status
231 # N - new
232 # D - deleted
233
234 my $cmailstore_ctablecmd = <<__EOD;
235 CREATE TABLE CMailStore
236 (CID INTEGER DEFAULT 0 NOT NULL,
237 RID INTEGER NOT NULL,
238 ID SERIAL UNIQUE,
239 Time INTEGER NOT NULL,
240 QType "char" NOT NULL,
241 Bytes INTEGER NOT NULL,
242 Spamlevel INTEGER NOT NULL,
243 Info VARCHAR NULL,
244 Sender VARCHAR(255) NOT NULL,
245 Header VARCHAR NOT NULL,
246 File VARCHAR(255) NOT NULL,
247 PRIMARY KEY (CID, RID));
248 CREATE INDEX CMailStore_Time_Index ON CMailStore (Time);
249
250 CREATE TABLE CMSReceivers
251 (CMailStore_CID INTEGER NOT NULL,
252 CMailStore_RID INTEGER NOT NULL,
253 PMail VARCHAR(255) NOT NULL,
254 Receiver VARCHAR(255),
255 Status "char" NOT NULL,
256 MTime INTEGER NOT NULL);
257
258 CREATE INDEX CMailStore_ID_Index ON CMSReceivers (CMailStore_CID, CMailStore_RID);
259
260 CREATE INDEX CMSReceivers_MTime_Index ON CMSReceivers (MTime);
261
262 __EOD
263
264 my $cstatistic_ctablecmd = <<__EOD;
265 CREATE TABLE CStatistic
266 (CID INTEGER DEFAULT 0 NOT NULL,
267 RID INTEGER NOT NULL,
268 ID SERIAL UNIQUE,
269 Time INTEGER NOT NULL,
270 Bytes INTEGER NOT NULL,
271 Direction Boolean NOT NULL,
272 Spamlevel INTEGER NOT NULL,
273 VirusInfo VARCHAR(255) NULL,
274 PTime INTEGER NOT NULL,
275 Sender VARCHAR(255) NOT NULL,
276 PRIMARY KEY (CID, RID));
277
278 CREATE INDEX CStatistic_Time_Index ON CStatistic (Time);
279
280 CREATE TABLE CReceivers
281 (CStatistic_CID INTEGER NOT NULL,
282 CStatistic_RID INTEGER NOT NULL,
283 Receiver VARCHAR(255) NOT NULL,
284 Blocked Boolean NOT NULL);
285
286 CREATE INDEX CStatistic_ID_Index ON CReceivers (CStatistic_CID, CStatistic_RID);
287 __EOD
288
289 # user preferences (black an whitelists, ...)
290 # Name: perference name ('BL' -> blacklist, 'WL' -> whitelist)
291 # Data: arbitrary data
292 my $userprefs_ctablecmd = <<__EOD;
293 CREATE TABLE UserPrefs
294 (PMail VARCHAR,
295 Name VARCHAR(255),
296 Data VARCHAR,
297 MTime INTEGER NOT NULL,
298 PRIMARY KEY (PMail, Name));
299
300 CREATE INDEX UserPrefs_MTime_Index ON UserPrefs (MTime);
301
302 __EOD
303
304 sub cond_create_dbtable {
305 my ($dbh, $name, $ctablecmd) = @_;
306
307 eval {
308 $dbh->begin_work;
309
310 my $cmd = "SELECT tablename FROM pg_tables " .
311 "WHERE tablename = lower ('$name')";
312
313 my $sth = $dbh->prepare($cmd);
314
315 $sth->execute();
316
317 if (!(my $ref = $sth->fetchrow_hashref())) {
318 $dbh->do ($ctablecmd);
319 }
320
321 $sth->finish();
322
323 $dbh->commit;
324 };
325 if (my $err = $@) {
326 $dbh->rollback;
327 die $err;
328 }
329 }
330
331 sub create_ruledb {
332 my ($dbname) = @_;
333
334 $dbname = $default_db_name if !$dbname;
335
336 my $silent_opts = { outfunc => sub {}, errfunc => sub {} };
337 # make sure we have user 'root'
338 eval { postgres_admin_cmd('createuser', $silent_opts, '-D', 'root'); };
339 # also create 'www-data' (and give it read-only access below)
340 eval { postgres_admin_cmd('createuser', $silent_opts, '-I', '-D', 'www-data'); };
341
342 # use sql_ascii to avoid any character set conversions, and be compatible with
343 # older postgres versions (update from 8.1 must be possible)
344
345 postgres_admin_cmd('createdb', undef, '-E', 'sql_ascii', '-T', 'template0',
346 '--lc-collate=C', '--lc-ctype=C', $dbname);
347
348 my $dbh = open_ruledb($dbname);
349
350 # make sure 'www-data' can read all tables
351 $dbh->do("ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO \"www-data\"");
352
353 $dbh->do (
354 <<EOD
355 CREATE TABLE Attribut
356 (Object_ID INTEGER NOT NULL,
357 Name VARCHAR(20) NOT NULL,
358 Value BYTEA NULL,
359 PRIMARY KEY (Object_ID, Name));
360
361 CREATE INDEX Attribut_Object_ID_Index ON Attribut(Object_ID);
362
363 CREATE TABLE Object
364 (ID SERIAL UNIQUE,
365 ObjectType INTEGER NOT NULL,
366 Objectgroup_ID INTEGER NOT NULL,
367 Value BYTEA NULL,
368 PRIMARY KEY (ID));
369
370 CREATE TABLE Objectgroup
371 (ID SERIAL UNIQUE,
372 Name VARCHAR(255) NOT NULL,
373 Info VARCHAR(255) NULL,
374 Class VARCHAR(10) NOT NULL,
375 PRIMARY KEY (ID));
376
377 CREATE TABLE Rule
378 (ID SERIAL UNIQUE,
379 Name VARCHAR(255) NULL,
380 Priority INTEGER NOT NULL,
381 Active INTEGER NOT NULL DEFAULT 0,
382 Direction INTEGER NOT NULL DEFAULT 2,
383 Count INTEGER NOT NULL DEFAULT 0,
384 PRIMARY KEY (ID));
385
386 CREATE TABLE RuleGroup
387 (Objectgroup_ID INTEGER NOT NULL,
388 Rule_ID INTEGER NOT NULL,
389 Grouptype INTEGER NOT NULL,
390 PRIMARY KEY (Objectgroup_ID, Rule_ID, Grouptype));
391
392 $cgreylist_ctablecmd;
393
394 $clusterinfo_ctablecmd;
395
396 $local_stat_ctablecmd;
397
398 $daily_stat_ctablecmd;
399
400 $domain_stat_ctablecmd;
401
402 $statinfo_ctablecmd;
403
404 $cmailstore_ctablecmd;
405
406 $cstatistic_ctablecmd;
407
408 $userprefs_ctablecmd;
409
410 $virusinfo_stat_ctablecmd;
411 EOD
412 );
413
414 return $dbh;
415 }
416
417 sub cond_create_action_quarantine {
418 my ($ruledb) = @_;
419
420 my $dbh = $ruledb->{dbh};
421
422 eval {
423 my $sth = $dbh->prepare(
424 "SELECT * FROM Objectgroup, Object " .
425 "WHERE Object.ObjectType = ? AND Objectgroup.Class = ? " .
426 "AND Object.objectgroup_id = Objectgroup.id");
427
428 my $otype = PMG::RuleDB::Quarantine::otype();
429 if ($sth->execute($otype, 'action') <= 0) {
430 my $obj = PMG::RuleDB::Quarantine->new ();
431 my $txt = decode_entities(PMG::RuleDB::Quarantine->otype_text);
432 my $quarantine = $ruledb->create_group_with_obj
433 ($obj, $txt, 'Move to quarantine.');
434 }
435 };
436 }
437
438 sub cond_create_std_actions {
439 my ($ruledb) = @_;
440
441 cond_create_action_quarantine($ruledb);
442
443 #cond_create_action_report_spam($ruledb);
444 }
445
446
447 sub upgradedb {
448 my ($ruledb) = @_;
449
450 my $dbh = $ruledb->{dbh};
451
452 # make sure we do not use slow sequential scans when upgraing
453 # database (before analyze can gather statistics)
454 $dbh->do("set enable_seqscan = false");
455
456 my $tables = {
457 'LocalStat', $local_stat_ctablecmd,
458 'DailyStat', $daily_stat_ctablecmd,
459 'DomainStat', $domain_stat_ctablecmd,
460 'StatInfo', $statinfo_ctablecmd,
461 'CMailStore', $cmailstore_ctablecmd,
462 'UserPrefs', $userprefs_ctablecmd,
463 'CGreylist', $cgreylist_ctablecmd,
464 'CStatistic', $cstatistic_ctablecmd,
465 'ClusterInfo', $clusterinfo_ctablecmd,
466 'VirusInfo', $virusinfo_stat_ctablecmd,
467 };
468
469 foreach my $table (keys %$tables) {
470 cond_create_dbtable($dbh, $table, $tables->{$table});
471 }
472
473 cond_create_std_actions($ruledb);
474
475 # upgrade tables here if necessary
476
477 # update obsolete content type names
478 eval {
479 $dbh->do("UPDATE Object " .
480 "SET value = 'content-type:application/java-vm' ".
481 "WHERE objecttype = 3003 " .
482 "AND value = 'content-type:application/x-java-vm';");
483 };
484
485 foreach my $table (keys %$tables) {
486 eval { $dbh->do("ANALYZE $table"); };
487 warn $@ if $@;
488 }
489
490 reload_ruledb();
491 }
492
493 sub init_ruledb {
494 my ($ruledb, $reset, $testmode) = @_;
495
496 my $dbh = $ruledb->{dbh};
497
498 if (!$reset) {
499 # Greylist Objectgroup
500 my $greylistgroup = PMG::RuleDB::Group->new
501 ("GreyExclusion", "-", "greylist");
502 $ruledb->save_group ($greylistgroup);
503
504 } else {
505 # we do not touch greylist objects
506 my $glids = "SELECT object.ID FROM Object, Objectgroup WHERE " .
507 "objectgroup_id = objectgroup.id and class = 'greylist'";
508
509 $dbh->do ("DELETE FROM Rule; " .
510 "DELETE FROM RuleGroup; " .
511 "DELETE FROM Attribut WHERE Object_ID NOT IN ($glids); " .
512 "DELETE FROM Object WHERE ID NOT IN ($glids); " .
513 "DELETE FROM Objectgroup WHERE class != 'greylist';");
514 }
515
516 # WHO Objects
517
518 # Blacklist
519 my $obj = PMG::RuleDB::EMail->new ('nomail@fromthisdomain.com');
520 my $blacklist = $ruledb->create_group_with_obj(
521 $obj, 'Blacklist', 'Global blacklist');
522
523 # Whitelist
524 $obj = PMG::RuleDB::EMail->new('mail@fromthisdomain.com');
525 my $whitelist = $ruledb->create_group_with_obj(
526 $obj, 'Whitelist', 'Global whitelist');
527
528 # WHEN Objects
529
530 # Working hours
531 $obj = PMG::RuleDB::TimeFrame->new(8*60, 16*60);
532 my $working_hours =$ruledb->create_group_with_obj($obj, 'Office Hours' ,
533 'Usual office hours');
534
535 # WHAT Objects
536
537 # Images
538 $obj = PMG::RuleDB::ContentTypeFilter->new('image/.*');
539 my $img_content = $ruledb->create_group_with_obj(
540 $obj, 'Images', 'All kinds of graphic files');
541
542 # Multimedia
543 $obj = PMG::RuleDB::ContentTypeFilter->new('audio/.*');
544 my $mm_content = $ruledb->create_group_with_obj(
545 $obj, 'Multimedia', 'Audio and Video');
546
547 $obj = PMG::RuleDB::ContentTypeFilter->new('video/.*');
548 $ruledb->group_add_object($mm_content, $obj);
549
550 # Office Files
551 $obj = PMG::RuleDB::ContentTypeFilter->new('application/vnd\.ms-excel');
552 my $office_content = $ruledb->create_group_with_obj(
553 $obj, 'Office Files', 'Common Office Files');
554
555 $obj = PMG::RuleDB::ContentTypeFilter->new(
556 'application/vnd\.ms-powerpoint');
557
558 $ruledb->group_add_object($office_content, $obj);
559
560 $obj = PMG::RuleDB::ContentTypeFilter->new('application/msword');
561 $ruledb->group_add_object ($office_content, $obj);
562
563 $obj = PMG::RuleDB::ContentTypeFilter->new(
564 'application/vnd\.openxmlformats-officedocument\..*');
565 $ruledb->group_add_object($office_content, $obj);
566
567 $obj = PMG::RuleDB::ContentTypeFilter->new(
568 'application/vnd\.oasis\.opendocument\..*');
569 $ruledb->group_add_object($office_content, $obj);
570
571 $obj = PMG::RuleDB::ContentTypeFilter->new(
572 'application/vnd\.stardivision\..*');
573 $ruledb->group_add_object($office_content, $obj);
574
575 $obj = PMG::RuleDB::ContentTypeFilter->new(
576 'application/vnd\.sun\.xml\..*');
577 $ruledb->group_add_object($office_content, $obj);
578
579 # Dangerous Content
580 $obj = PMG::RuleDB::ContentTypeFilter->new(
581 'application/x-ms-dos-executable');
582 my $exe_content = $ruledb->create_group_with_obj(
583 $obj, 'Dangerous Content', 'executable files and partial messages');
584
585 $obj = PMG::RuleDB::ContentTypeFilter->new('application/x-java');
586 $ruledb->group_add_object($exe_content, $obj);
587 $obj = PMG::RuleDB::ContentTypeFilter->new('application/javascript');
588 $ruledb->group_add_object($exe_content, $obj);
589 $obj = PMG::RuleDB::ContentTypeFilter->new('application/x-executable');
590 $ruledb->group_add_object($exe_content, $obj);
591 $obj = PMG::RuleDB::ContentTypeFilter->new('application/x-ms-dos-executable');
592 $ruledb->group_add_object($exe_content, $obj);
593 $obj = PMG::RuleDB::ContentTypeFilter->new('message/partial');
594 $ruledb->group_add_object($exe_content, $obj);
595 $obj = PMG::RuleDB::MatchFilename->new('.*\.(vbs|pif|lnk|shs|shb)');
596 $ruledb->group_add_object($exe_content, $obj);
597 $obj = PMG::RuleDB::MatchFilename->new('.*\.\{.+\}');
598 $ruledb->group_add_object($exe_content, $obj);
599
600 # Virus
601 $obj = PMG::RuleDB::Virus->new();
602 my $virus = $ruledb->create_group_with_obj(
603 $obj, 'Virus', 'Matches virus infected mail');
604
605 # WHAT Objects
606
607 # Spam
608 $obj = PMG::RuleDB::Spam->new(3);
609 my $spam3 = $ruledb->create_group_with_obj(
610 $obj, 'Spam (Level 3)', 'Matches possible spam mail');
611
612 $obj = PMG::RuleDB::Spam->new(5);
613 my $spam5 = $ruledb->create_group_with_obj(
614 $obj, 'Spam (Level 5)', 'Matches possible spam mail');
615
616 $obj = PMG::RuleDB::Spam->new(10);
617 my $spam10 = $ruledb->create_group_with_obj(
618 $obj, 'Spam (Level 10)', 'Matches possible spam mail');
619
620 # ACTIONS
621
622 # Mark Spam
623 $obj = PMG::RuleDB::ModField->new('X-SPAM-LEVEL', '__SPAM_INFO__');
624 my $mod_spam_level = $ruledb->create_group_with_obj(
625 $obj, 'Modify Spam Level',
626 'Mark mail as spam by adding a header tag.');
627
628 # Mark Spam
629 $obj = PMG::RuleDB::ModField->new('subject', 'SPAM: __SUBJECT__');
630 my $mod_spam_subject = $ruledb->create_group_with_obj(
631 $obj, 'Modify Spam Subject',
632 'Mark mail as spam by modifying the subject.');
633
634 # Remove matching attachments
635 $obj = PMG::RuleDB::Remove->new(0);
636 my $remove = $ruledb->create_group_with_obj(
637 $obj, 'Remove attachments', 'Remove matching attachments');
638
639 # Remove all attachments
640 $obj = PMG::RuleDB::Remove->new(1);
641 my $remove_all = $ruledb->create_group_with_obj(
642 $obj, 'Remove all attachments', 'Remove all attachments');
643
644 # Accept
645 $obj = PMG::RuleDB::Accept->new();
646 my $accept = $ruledb->create_group_with_obj(
647 $obj, 'Accept', 'Accept mail for Delivery');
648
649 # Block
650 $obj = PMG::RuleDB::Block->new ();
651 my $block = $ruledb->create_group_with_obj($obj, 'Block', 'Block mail');
652
653 # Quarantine
654 $obj = PMG::RuleDB::Quarantine->new();
655 my $quarantine = $ruledb->create_group_with_obj(
656 $obj, 'Quarantine', 'Move mail to quarantine');
657
658 # Notify Admin
659 $obj = PMG::RuleDB::Notify->new('__ADMIN__');
660 my $notify_admin = $ruledb->create_group_with_obj(
661 $obj, 'Notify Admin', 'Send notification');
662
663 # Notify Sender
664 $obj = PMG::RuleDB::Notify->new('__SENDER__');
665 my $notify_sender = $ruledb->create_group_with_obj(
666 $obj, 'Notify Sender', 'Send notification');
667
668 # Add Disclaimer
669 $obj = PMG::RuleDB::Disclaimer->new ();
670 my $add_discl = $ruledb->create_group_with_obj(
671 $obj, 'Disclaimer', 'Add Disclaimer');
672
673 # Attach original mail
674 #$obj = Proxmox::RuleDB::Attach->new ();
675 #my $attach_orig = $ruledb->create_group_with_obj ($obj, 'Attach Original Mail',
676 # 'Attach Original Mail');
677
678 ####################### RULES ##################################
679
680 ## Block Dangerous Files
681 my $rule = PMG::RuleDB::Rule->new ('Block Dangerous Files', 93, 1, 0);
682 $ruledb->save_rule ($rule);
683
684 $ruledb->rule_add_what_group ($rule, $exe_content);
685 $ruledb->rule_add_action ($rule, $remove);
686
687 ## Block Viruses
688 $rule = PMG::RuleDB::Rule->new ('Block Viruses', 96, 1, 0);
689 $ruledb->save_rule ($rule);
690
691 $ruledb->rule_add_what_group ($rule, $virus);
692 $ruledb->rule_add_action ($rule, $notify_admin);
693
694 if ($testmode) {
695 $ruledb->rule_add_action ($rule, $block);
696 } else {
697 $ruledb->rule_add_action ($rule, $quarantine);
698 }
699
700 ## Virus Alert
701 $rule = PMG::RuleDB::Rule->new ('Virus Alert', 96, 1, 1);
702 $ruledb->save_rule ($rule);
703
704 $ruledb->rule_add_what_group ($rule, $virus);
705 $ruledb->rule_add_action ($rule, $notify_sender);
706 $ruledb->rule_add_action ($rule, $notify_admin);
707 $ruledb->rule_add_action ($rule, $block);
708
709 ## Blacklist
710 $rule = PMG::RuleDB::Rule->new ('Blacklist', 98, 1, 0);
711 $ruledb->save_rule ($rule);
712
713 $ruledb->rule_add_from_group ($rule, $blacklist);
714 $ruledb->rule_add_action ($rule, $block);
715
716 ## Modify header
717 if (!$testmode) {
718 $rule = PMG::RuleDB::Rule->new ('Modify Header', 90, 1, 0);
719 $ruledb->save_rule ($rule);
720 $ruledb->rule_add_action ($rule, $mod_spam_level);
721 }
722
723 ## Whitelist
724 $rule = PMG::RuleDB::Rule->new ('Whitelist', 85, 1, 0);
725 $ruledb->save_rule ($rule);
726
727 $ruledb->rule_add_from_group ($rule, $whitelist);
728 $ruledb->rule_add_action ($rule, $accept);
729
730 if ($testmode) {
731 $rule = PMG::RuleDB::Rule->new ('Mark Spam', 80, 1, 0);
732 $ruledb->save_rule ($rule);
733
734 $ruledb->rule_add_what_group ($rule, $spam10);
735 $ruledb->rule_add_action ($rule, $mod_spam_level);
736 $ruledb->rule_add_action ($rule, $mod_spam_subject);
737 } else {
738 # Quarantine/Mark Spam (Level 3)
739 $rule = PMG::RuleDB::Rule->new ('Quarantine/Mark Spam (Level 3)', 80, 1, 0);
740 $ruledb->save_rule ($rule);
741
742 $ruledb->rule_add_what_group ($rule, $spam3);
743 $ruledb->rule_add_action ($rule, $mod_spam_subject);
744 $ruledb->rule_add_action ($rule, $quarantine);
745 #$ruledb->rule_add_action ($rule, $count_spam);
746 }
747
748 # Quarantine/Mark Spam (Level 5)
749 $rule = PMG::RuleDB::Rule->new ('Quarantine/Mark Spam (Level 5)', 79, 0, 0);
750 $ruledb->save_rule ($rule);
751
752 $ruledb->rule_add_what_group ($rule, $spam5);
753 $ruledb->rule_add_action ($rule, $mod_spam_subject);
754 $ruledb->rule_add_action ($rule, $quarantine);
755
756 ## Block Spam Level 10
757 $rule = PMG::RuleDB::Rule->new ('Block Spam (Level 10)', 78, 0, 0);
758 $ruledb->save_rule ($rule);
759
760 $ruledb->rule_add_what_group ($rule, $spam10);
761 $ruledb->rule_add_action ($rule, $block);
762
763 ## Block Outgoing Spam
764 $rule = PMG::RuleDB::Rule->new ('Block outgoing Spam', 70, 0, 1);
765 $ruledb->save_rule ($rule);
766
767 $ruledb->rule_add_what_group ($rule, $spam3);
768 $ruledb->rule_add_action ($rule, $notify_admin);
769 $ruledb->rule_add_action ($rule, $notify_sender);
770 $ruledb->rule_add_action ($rule, $block);
771
772 ## Add disclaimer
773 $rule = PMG::RuleDB::Rule->new ('Add Disclaimer', 60, 0, 1);
774 $ruledb->save_rule ($rule);
775 $ruledb->rule_add_action ($rule, $add_discl);
776
777 # Block Multimedia Files
778 $rule = PMG::RuleDB::Rule->new ('Block Multimedia Files', 87, 0, 2);
779 $ruledb->save_rule ($rule);
780
781 $ruledb->rule_add_what_group ($rule, $mm_content);
782 $ruledb->rule_add_action ($rule, $remove);
783
784 #$ruledb->rule_add_from_group ($rule, $anybody);
785 #$ruledb->rule_add_from_group ($rule, $trusted);
786 #$ruledb->rule_add_to_group ($rule, $anybody);
787 #$ruledb->rule_add_what_group ($rule, $ct_filter);
788 #$ruledb->rule_add_action ($rule, $add_discl);
789 #$ruledb->rule_add_action ($rule, $remove);
790 #$ruledb->rule_add_action ($rule, $bcc);
791 #$ruledb->rule_add_action ($rule, $storeq);
792 #$ruledb->rule_add_action ($rule, $accept);
793
794 cond_create_std_actions ($ruledb);
795
796 reload_ruledb();
797 }
798
799 sub get_remote_time {
800 my ($rdb) = @_;
801
802 my $sth = $rdb->prepare("SELECT EXTRACT (EPOCH FROM TIMESTAMP (0) WITH TIME ZONE 'now') as ctime;");
803 $sth->execute();
804 my $ctinfo = $sth->fetchrow_hashref();
805 $sth->finish ();
806
807 return $ctinfo ? $ctinfo->{ctime} : 0;
808 }
809
810 sub init_masterdb {
811 my ($lcid, $database) = @_;
812
813 die "got unexpected cid for new master" if !$lcid;
814
815 my $dbh;
816
817 eval {
818 $dbh = open_ruledb($database);
819
820 $dbh->begin_work;
821
822 print STDERR "update quarantine database\n";
823 $dbh->do ("UPDATE CMailStore SET CID = $lcid WHERE CID = 0;" .
824 "UPDATE CMSReceivers SET CMailStore_CID = $lcid WHERE CMailStore_CID = 0;");
825
826 print STDERR "update statistic database\n";
827 $dbh->do ("UPDATE CStatistic SET CID = $lcid WHERE CID = 0;" .
828 "UPDATE CReceivers SET CStatistic_CID = $lcid WHERE CStatistic_CID = 0;");
829
830 print STDERR "update greylist database\n";
831 $dbh->do ("UPDATE CGreylist SET CID = $lcid WHERE CID = 0;");
832
833 print STDERR "update localstat database\n";
834 $dbh->do ("UPDATE LocalStat SET CID = $lcid WHERE CID = 0;");
835
836 $dbh->commit;
837 };
838 my $err = $@;
839
840 if ($dbh) {
841 $dbh->rollback if $err;
842 $dbh->disconnect();
843 }
844
845 die $err if $err;
846 }
847
848 sub purge_statistic_database {
849 my ($dbh, $statlifetime) = @_;
850
851 return if $statlifetime <= 0;
852
853 my (undef, undef, undef, $mday, $mon, $year) = localtime(time());
854 my $end = timelocal(0, 0, 0, $mday, $mon, $year);
855 my $start = $end - $statlifetime*86400;
856
857 # delete statistics older than $start
858
859 my $rows = 0;
860
861 eval {
862 $dbh->begin_work;
863
864 my $sth = $dbh->prepare("DELETE FROM CStatistic WHERE time < $start");
865 $sth->execute;
866 $rows = $sth->rows;
867 $sth->finish;
868
869 if ($rows > 0) {
870 $sth = $dbh->prepare(
871 "DELETE FROM CReceivers WHERE NOT EXISTS " .
872 "(SELECT * FROM CStatistic WHERE CID = CStatistic_CID AND RID = CStatistic_RID)");
873
874 $sth->execute;
875 }
876 $dbh->commit;
877 };
878 if (my $err = $@) {
879 $dbh->rollback;
880 die $err;
881 }
882
883 return $rows;
884 }
885
886 sub purge_quarantine_database {
887 my ($dbh, $qtype, $lifetime) = @_;
888
889 my $spooldir = $PMG::MailQueue::spooldir;
890
891 my (undef, undef, undef, $mday, $mon, $year) = localtime(time());
892 my $end = timelocal(0, 0, 0, $mday, $mon, $year);
893 my $start = $end - $lifetime*86400;
894
895 my $sth = $dbh->prepare(
896 "SELECT file FROM CMailStore WHERE time < $start AND QType = '$qtype'");
897
898 $sth->execute();
899
900 my $count = 0;
901
902 while (my $ref = $sth->fetchrow_hashref()) {
903 my $filename = "$spooldir/$ref->{file}";
904 $count++ if unlink($filename);
905 }
906
907 $sth->finish();
908
909 $dbh->do(
910 "DELETE FROM CMailStore WHERE time < $start AND QType = '$qtype';" .
911 "DELETE FROM CMSReceivers WHERE NOT EXISTS " .
912 "(SELECT * FROM CMailStore WHERE CID = CMailStore_CID AND RID = CMailStore_RID)");
913
914 return $count;
915 }
916
917 sub get_quarantine_count {
918 my ($dbh, $qtype) = @_;
919
920 # Note;: We try to estimate used disk space - each mail
921 # is stored in an extra file ...
922
923 my $bs = 4096;
924
925 my $sth = $dbh->prepare(
926 "SELECT count(ID) as count, sum (ceil((Bytes+$bs-1)/$bs)*$bs) / (1024*1024) as mbytes, " .
927 "avg(Bytes) as avgbytes, avg(Spamlevel) as avgspam " .
928 "FROM CMailStore WHERE QType = ?");
929
930 $sth->execute($qtype);
931
932 my $ref = $sth->fetchrow_hashref();
933
934 $sth->finish;
935
936 foreach my $k (qw(count mbytes avgbytes avgspam)) {
937 $ref->{$k} //= 0;
938 }
939
940 return $ref;
941 }
942
943 sub copy_table {
944 my ($ldb, $rdb, $table) = @_;
945
946 $table = lc($table);
947
948 my $sth = $ldb->column_info(undef, undef, $table, undef);
949 my $attrs = $sth->fetchall_arrayref({});
950
951 my @col_arr;
952 foreach my $ref (@$attrs) {
953 push @col_arr, $ref->{COLUMN_NAME};
954 }
955
956 $sth->finish();
957
958 my $cols = join(', ', @col_arr);
959 $cols || die "unable to fetch column definitions of table '$table' : ERROR";
960
961 $rdb->do("COPY $table ($cols) TO STDOUT");
962
963 my $data = '';
964
965 eval {
966 $ldb->do("COPY $table ($cols) FROM stdin");
967
968 while ($rdb->pg_getcopydata($data) >= 0) {
969 $ldb->pg_putcopydata($data);
970 }
971
972 $ldb->pg_putcopyend();
973 };
974 if (my $err = $@) {
975 $ldb->pg_putcopyend();
976 die $err;
977 }
978 }
979
980 sub copy_selected_data {
981 my ($dbh, $select_sth, $table, $attrs, $callback) = @_;
982
983 my $count = 0;
984
985 my $insert_sth = $dbh->prepare(
986 "INSERT INTO ${table}(" . join(',', @$attrs) . ') ' .
987 'VALUES (' . join(',', ('?') x scalar(@$attrs)) . ')');
988
989 while (my $ref = $select_sth->fetchrow_hashref()) {
990 $callback->($ref) if $callback;
991 $count++;
992 $insert_sth->execute(map { $ref->{$_} } @$attrs);
993 }
994
995 return $count;
996 }
997
998 sub update_master_clusterinfo {
999 my ($clientcid) = @_;
1000
1001 my $dbh = open_ruledb();
1002
1003 $dbh->do("DELETE FROM ClusterInfo WHERE CID = $clientcid");
1004
1005 my @mt = ('CMSReceivers', 'CGreylist', 'UserPrefs', 'DomainStat', 'DailyStat', 'LocalStat', 'VirusInfo');
1006
1007 foreach my $table (@mt) {
1008 $dbh->do ("INSERT INTO ClusterInfo (cid, name, ivalue) select $clientcid, 'lastmt_$table', " .
1009 "EXTRACT(EPOCH FROM now())");
1010 }
1011 }
1012
1013 sub update_client_clusterinfo {
1014 my ($mastercid) = @_;
1015
1016 my $dbh = open_ruledb();
1017
1018 $dbh->do ("DELETE FROM StatInfo"); # not needed at node
1019
1020 $dbh->do ("DELETE FROM ClusterInfo WHERE CID = $mastercid");
1021
1022 $dbh->do ("INSERT INTO ClusterInfo (cid, name, ivalue) select $mastercid, 'lastid_CMailStore', " .
1023 "COALESCE (max (rid), -1) FROM CMailStore WHERE cid = $mastercid");
1024
1025 $dbh->do ("INSERT INTO ClusterInfo (cid, name, ivalue) select $mastercid, 'lastid_CStatistic', " .
1026 "COALESCE (max (rid), -1) FROM CStatistic WHERE cid = $mastercid");
1027
1028 my @mt = ('CMSReceivers', 'CGreylist', 'UserPrefs', 'DomainStat', 'DailyStat', 'LocalStat', 'VirusInfo');
1029
1030 foreach my $table (@mt) {
1031 $dbh->do ("INSERT INTO ClusterInfo (cid, name, ivalue) select $mastercid, 'lastmt_$table', " .
1032 "COALESCE (max (mtime), 0) FROM $table");
1033 }
1034 }
1035
1036 sub create_clusterinfo_default {
1037 my ($dbh, $rcid, $name, $ivalue, $svalue) = @_;
1038
1039 my $sth = $dbh->prepare("SELECT * FROM ClusterInfo WHERE CID = ? AND Name = ?");
1040 $sth->execute($rcid, $name);
1041 if (!$sth->fetchrow_hashref()) {
1042 $dbh->do("INSERT INTO ClusterInfo (CID, Name, IValue, SValue) " .
1043 "VALUES (?, ?, ?, ?)", undef,
1044 $rcid, $name, $ivalue, $svalue);
1045 }
1046 $sth->finish();
1047 }
1048
1049 sub read_int_clusterinfo {
1050 my ($dbh, $rcid, $name) = @_;
1051
1052 my $sth = $dbh->prepare(
1053 "SELECT ivalue as value FROM ClusterInfo " .
1054 "WHERE cid = ? AND NAME = ?");
1055 $sth->execute($rcid, $name);
1056 my $cinfo = $sth->fetchrow_hashref();
1057 $sth->finish();
1058
1059 return $cinfo->{value};
1060 }
1061
1062 sub write_maxint_clusterinfo {
1063 my ($dbh, $rcid, $name, $value) = @_;
1064
1065 $dbh->do("UPDATE ClusterInfo SET ivalue = GREATEST(ivalue, ?) " .
1066 "WHERE cid = ? AND name = ?", undef,
1067 $value, $rcid, $name);
1068 }
1069
1070 sub init_nodedb {
1071 my ($cinfo) = @_;
1072
1073 my $ni = $cinfo->{master};
1074
1075 die "no master defined - unable to sync data from master\n" if !$ni;
1076
1077 my $master_ip = $ni->{ip};
1078 my $master_cid = $ni->{cid};
1079 my $master_name = $ni->{name};
1080
1081 my $fn = "/tmp/masterdb$$.tar";
1082 unlink $fn;
1083
1084 my $dbname = $default_db_name;
1085
1086 eval {
1087 print STDERR "copying master database from '${master_ip}'\n";
1088
1089 open (my $fh, ">", $fn) || die "open '$fn' failed - $!\n";
1090
1091 postgres_admin_cmd(
1092 ['/usr/bin/ssh', '-o', 'BatchMode=yes',
1093 '-o', "HostKeyAlias=${master_name}",
1094 $master_ip, 'pg_dump'],
1095 { output => '>&' . fileno($fh) },
1096 $dbname, '-F', 'c');
1097
1098 close($fh);
1099
1100 my $size = -s $fn;
1101
1102 print STDERR "copying master database finished (got $size bytes)\n";
1103
1104 print STDERR "delete local database\n";
1105
1106 postgres_admin_cmd('dropdb', undef, $dbname , '--if-exists');
1107
1108 print STDERR "create new local database\n";
1109
1110 postgres_admin_cmd('createdb', undef, $dbname);
1111
1112 print STDERR "insert received data into local database\n";
1113
1114 my $mess;
1115 my $parser = sub {
1116 my $line = shift;
1117
1118 if ($line =~ m/restoring data for table \"(.+)\"/) {
1119 print STDERR "restoring table $1\n";
1120 } elsif (!$mess && ($line =~ m/creating (INDEX|CONSTRAINT)/)) {
1121 $mess = "creating indexes";
1122 print STDERR "$mess\n";
1123 }
1124 };
1125
1126 my $opts = {
1127 outfunc => $parser,
1128 errfunc => $parser,
1129 errmsg => "pg_restore failed"
1130 };
1131
1132 postgres_admin_cmd('pg_restore', $opts, '-d', $dbname, '-v', $fn);
1133
1134 print STDERR "run analyze to speed up database queries\n";
1135
1136 postgres_admin_cmd('psql', { input => 'analyze;' }, $dbname);
1137
1138 update_client_clusterinfo($master_cid);
1139 };
1140
1141 my $err = $@;
1142
1143 unlink $fn;
1144
1145 die $err if $err;
1146 }
1147
1148 sub cluster_sync_status {
1149 my ($cinfo) = @_;
1150
1151 my $dbh;
1152
1153 my $minmtime;
1154
1155 foreach my $ni (values %{$cinfo->{ids}}) {
1156 next if $cinfo->{local}->{cid} == $ni->{cid}; # skip local CID
1157 $minmtime->{$ni->{cid}} = 0;
1158 }
1159
1160 eval {
1161 $dbh = open_ruledb();
1162
1163 my $sth = $dbh->prepare(
1164 "SELECT cid, MIN (ivalue) as minmtime FROM ClusterInfo " .
1165 "WHERE name = 'lastsync' AND ivalue > 0 " .
1166 "GROUP BY cid");
1167
1168 $sth->execute();
1169
1170 while (my $info = $sth->fetchrow_hashref()) {
1171 foreach my $ni (values %{$cinfo->{ids}}) {
1172 next if $cinfo->{local}->{cid} == $ni->{cid}; # skip local CID
1173 if ($ni->{cid} == $info->{cid}) { # node exists
1174 $minmtime->{$ni->{cid}} = $info->{minmtime};
1175 }
1176 }
1177 }
1178
1179 $sth->finish();
1180 };
1181 my $err = $@;
1182
1183 $dbh->disconnect() if $dbh;
1184
1185 syslog('err', $err) if $err;
1186
1187 return $minmtime;
1188 }
1189
1190 sub load_mail_data {
1191 my ($dbh, $cid, $rid, $pmail) = @_;
1192
1193 my $sth = $dbh->prepare(
1194 "SELECT * FROM CMailStore, CMSReceivers WHERE " .
1195 ($pmail ? 'pmail = ' . $dbh->quote($pmail) . " AND " : '') .
1196 "CID = $cid and RID = $rid AND " .
1197 "CID = CMailStore_CID AND RID = CMailStore_RID");
1198 $sth->execute ();
1199
1200 my $res = $sth->fetchrow_hashref();
1201
1202 $sth->finish();
1203
1204 die "no such mail (C${cid}R${rid})\n" if !defined($res);
1205
1206 return $res;
1207 }
1208
1209 sub reload_ruledb {
1210 my ($ruledb) = @_;
1211
1212 # Note: we pass $ruledb when modifying SMTP whitelist
1213 if (defined($ruledb)) {
1214 eval {
1215 my $rulecache = PMG::RuleCache->new($ruledb);
1216 PMG::Config::rewrite_postfix_whitelist($rulecache);
1217 };
1218 if (my $err = $@) {
1219 warn "problems updating SMTP whitelist - $err";
1220 }
1221 }
1222
1223 my $pid_file = '/var/run/pmg-smtp-filter.pid';
1224 my $pid = PVE::Tools::file_read_firstline($pid_file);
1225
1226 return 0 if !$pid;
1227
1228 return 0 if $pid !~ m/^(\d+)$/;
1229 $pid = $1; # untaint
1230
1231 return kill (10, $pid); # send SIGUSR1
1232 }
1233
1234 1;