From: Dietmar Maurer Date: Fri, 3 Feb 2017 07:08:03 +0000 (+0100) Subject: add RuleCache.pm X-Git-Url: https://git.proxmox.com/?a=commitdiff_plain;h=c881fe35aaead9a3ee4cdb1c7549236ba525d782;p=pmg-api.git add RuleCache.pm --- diff --git a/Makefile b/Makefile index de52998..54e006d 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,7 @@ LIBSOURCES = \ PMG/HTTPServer.pm \ PMG/Ticket.pm \ PMG/AccessControl.pm \ + PMG/RuleCache.pm \ PMG/DBTools.pm \ PMG/RuleDB/Group.pm \ PMG/RuleDB/Rule.pm \ diff --git a/PMG/CLI/pmgdb.pm b/PMG/CLI/pmgdb.pm index 905030f..ac23baa 100644 --- a/PMG/CLI/pmgdb.pm +++ b/PMG/CLI/pmgdb.pm @@ -9,10 +9,50 @@ use PVE::Tools qw(extract_param); use PVE::INotify; use PMG::DBTools; +use PMG::RuleCache; use base qw(PVE::CLIHandler); -my $nodename = PVE::INotify::nodename(); +sub print_objects { + my ($ruledb, $og) = @_; + + my $objects = $ruledb->load_group_objects ($og->{id}); + + foreach my $obj (@$objects) { + my $desc = $obj->short_desc (); + print " OBJECT $obj->{id}: $desc\n"; + } +} + +sub print_rule { + my ($ruledb, $rule) = @_; + + print "Found RULE $rule->{id}: $rule->{name}\n"; + + my ($from, $to, $when, $what, $action) = + $ruledb->load_groups($rule); + + foreach my $og (@$from) { + print " FOUND FROM GROUP $og->{id}: $og->{name}\n"; + print_objects($ruledb, $og); + } + foreach my $og (@$to) { + print " FOUND TO GROUP $og->{id}: $og->{name}\n"; + print_objects($ruledb, $og); + } + foreach my $og (@$when) { + print " FOUND WHEN GROUP $og->{id}: $og->{name}\n"; + print_objects($ruledb, $og); + } + foreach my $og (@$what) { + print " FOUND WHAT GROUP $og->{id}: $og->{name}\n"; + print_objects($ruledb, $og); + } + foreach my $og (@$action) { + print " FOUND ACTION GROUP $og->{id}: $og->{name}\n"; + print_objects($ruledb, $og); + } +} __PACKAGE__->register_method ({ name => 'dump', @@ -26,7 +66,18 @@ __PACKAGE__->register_method ({ code => sub { my ($param) = @_; - print "DUMP\n"; + my $dbh = PMG::DBTools::open_ruledb("Proxmox_ruledb"); + my $ruledb = PMG::RuleDB->new($dbh); + + my $rulecache = PMG::RuleCache->new($ruledb); + + my $rules = $ruledb->load_rules(); + + foreach my $rule (@$rules) { + print_rule($ruledb, $rule); + } + + $ruledb->close(); return undef; }}); diff --git a/PMG/RuleCache.pm b/PMG/RuleCache.pm new file mode 100644 index 0000000..d2adfde --- /dev/null +++ b/PMG/RuleCache.pm @@ -0,0 +1,329 @@ +package PMG::RuleCache; + +use strict; +use warnings; +use DBI; +use Carp; + +use PVE::SafeSyslog; + +use PMG::Utils; +use PMG::RuleDB; +use Digest::SHA; + +my $ocache_size = 1023; + +sub new { + my ($type, $ruledb) = @_; + + my $self; + + $self->{ruledb} = $ruledb; + $self->{ocache} = (); + + bless $self, $type; + + my $rules = (); + + my $dbh = $ruledb->{dbh}; + + my $sha1 = Digest::SHA->new; + + eval { + $dbh->begin_work; + + # read a consistent snapshot + $dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE"); + + my $sth = $dbh->prepare( + "SELECT ID, Name, Priority, Active, Direction FROM Rule " . + "where Active > 0 " . + "ORDER BY Priority DESC"); + + $sth->execute(); + + while (my $ref = $sth->fetchrow_hashref()) { + my $ruleid = $ref->{id}; + my $rule = PMG::RuleDB::Rule->new( + $ref->{name}, $ref->{priority}, $ref->{active}, + $ref->{direction}); + + $rule->{id} = $ruleid; + push @$rules, $rule; + + $sha1->add(join (',', values (%$ref)) . "|"); + + my ($from, $to, $when, $what, $action); + + my $sth1 = $dbh->prepare( + "SELECT Objectgroup_ID, Grouptype FROM RuleGroup " . + "where RuleGroup.Rule_ID = '$ruleid' " . + "ORDER BY Grouptype, Objectgroup_ID"); + + $sth1->execute(); + while (my $ref1 = $sth1->fetchrow_hashref()) { + my $gtype = $ref1->{grouptype}; + my $groupid = $ref1->{objectgroup_id}; + + # emtyp groups differ from non-existent groups! + + if ($gtype == 0) { #from + $from = [] if !defined ($from); + } elsif ($gtype == 1) { # to + $to = [] if !defined ($to); + } elsif ($gtype == 2) { # when + $when = [] if !defined ($when); + } elsif ($gtype == 3) { # what + $what = [] if !defined ($what); + } elsif ($gtype == 4) { # action + $action = [] if !defined ($action); + } + + my $sth2 = $dbh->prepare( + "SELECT ID FROM Object where Objectgroup_ID = '$groupid' " . + "ORDER BY ID"); + $sth2->execute(); + while (my $ref2 = $sth2->fetchrow_hashref()) { + my $objid = $ref2->{'id'}; + my $obj = $self->_get_object($objid); + + $sha1->add (join (',', $objid, $gtype, $groupid) . "|"); + $sha1->add ($obj->{digest}, "|"); + + if ($gtype == 0) { #from + push @$from, $obj; + } elsif ($gtype == 1) { # to + push @$to, $obj; + } elsif ($gtype == 2) { # when + push @$when, $obj; + } elsif ($gtype == 3) { # what + push @$what, $obj; + if ($obj->otype == PMG::RuleDB::ArchiveFilter->otype) { + if ($rule->{direction} == 0) { + $self->{archivefilter_in} = 1; + } elsif ($rule->{direction} == 1) { + $self->{archivefilter_out} = 1; + } else { + $self->{archivefilter_in} = 1; + $self->{archivefilter_out} = 1; + } + } + } elsif ($gtype == 4) { # action + push @$action, $obj; + $self->{"$ruleid:final"} = 1 if $obj->final(); + } + } + $sth2->finish(); + } + + $sth1->finish(); + + $self->{"$ruleid:from"} = $from; + $self->{"$ruleid:to"} = $to; + $self->{"$ruleid:when"} = $when; + $self->{"$ruleid:what"} = $what; + $self->{"$ruleid:action"} = $action; + } + + # Cache Greylist Exclusion + $sth = $dbh->prepare( + "SELECT object.id FROM object, objectgroup " . + "WHERE class = 'greylist' AND " . + "objectgroup.id = object.objectgroup_id " . + "ORDER BY object.id"); + + $sth->execute(); + my $grey_excl_sender = (); + my $grey_excl_receiver = (); + while (my $ref2 = $sth->fetchrow_hashref()) { + my $obj = $self->_get_object ($ref2->{'id'}); + + if ($obj->receivertest()) { + push @$grey_excl_receiver, $obj; + } else { + push @$grey_excl_sender, $obj; + } + $sha1->add (join (',', values (%$ref2)) . "|"); + $sha1->add ($obj->{digest}, "|"); + } + + $self->{"greylist:sender"} = $grey_excl_sender; + $self->{"greylist:receiver"} = $grey_excl_receiver; + + $sth->finish(); + }; + my $err = $@; + + $dbh->rollback; # end transaction + + syslog ('err', PMG::Utils::msgquote("unable to load rulecache : $err")) if $err; + + $self->{rules} = $rules; + + $self->{digest} = $sha1->hexdigest; + + return $self; +} + +sub final { + my ($self, $ruleid) = @_; + + defined($ruleid) || croak ("undefined rule id: ERROR"); + + return $self->{"$ruleid:final"}; +} + +sub rules { + my ($self) = @_; + + $self->{rules}; +} + +sub _get_object { + my ($self, $objid) = @_; + + my $cid = $objid % $ocache_size; + + my $obj = $self->{ocache}[$cid]; + + if (!defined ($obj) || $obj->{id} != $objid) { + $obj = $self->{ruledb}->load_object($objid); + $self->{ocache}[$cid] = $obj; + } + + $obj || croak "unable to get object $objid: ERROR"; + + return $obj; +} + +sub get_actions { + my ($self, $ruleid) = @_; + + defined($ruleid) || croak ("undefined rule id: ERROR"); + + return $self->{"$ruleid:action"}; +} + +sub greylist_match { + my ($self, $addr, $ip) = @_; + + my $grey = $self->{"greylist:sender"}; + + foreach my $obj (@$grey) { + if ($obj->who_match ($addr, $ip)) { + return 1; + } + } + + return 0; +} + +sub greylist_match_receiver { + my ($self, $addr) = @_; + + my $grey = $self->{"greylist:receiver"}; + + foreach my $obj (@$grey) { + if ($obj->who_match($addr)) { + return 1; + } + } + + return 0; +} + +sub from_match { + my ($self, $ruleid, $addr, $ip, $ldap) = @_; + + my $from = $self->{"$ruleid:from"}; + + return 1 if !defined ($from); + + foreach my $obj (@$from) { + return 1 if $obj->who_match($addr, $ip, $ldap); + } + + return 0; +} + +sub to_match { + my ($self, $ruleid, $addr, $ldap) = @_; + + my $to = $self->{"$ruleid:to"}; + + return 1 if !defined ($to); + + foreach my $obj (@$to) { + return 1 if $obj->who_match($addr, undef, $ldap); + } + + return 0; +} + +sub when_match { + my ($self, $ruleid, $time) = @_; + + my $when = $self->{"$ruleid:when"}; + + return 1 if !defined ($when); + + foreach my $obj (@$when) { + return 1 if $obj->when_match($time); + } + + return 0; +} + +sub what_match { + my ($self, $ruleid, $queue, $element, $msginfo, $dbh) = @_; + + my $what = $self->{"$ruleid:what"}; + + my $res; + + # $res->{marks} is used by mark specific actions like remove-attachments + # $res->{$target}->{marks} is only used in apply_rules() to exclude some + # targets (spam blacklist and whitelist) + + if (!defined ($what)) { + # match all targets + foreach my $target (@{$msginfo->{targets}}) { + $res->{$target}->{marks} = []; + } + + $res->{marks} = []; + return $res; + } + + my $marks; + + foreach my $obj (@$what) { + if (!$obj->can('what_match_targets')) { + if (my $match = $obj->what_match($queue, $element, $msginfo, $dbh)) { + push @$marks, @$match; + } + } + } + + foreach my $target (@{$msginfo->{targets}}) { + $res->{$target}->{marks} = $marks; + $res->{marks} = $marks; + } + + foreach my $obj (@$what) { + if ($obj->can ("what_match_targets")) { + my $target_info; + if ($target_info = $obj->what_match_targets($queue, $element, $msginfo, $dbh)) { + foreach my $k (keys %$target_info) { + my $cmarks = $target_info->{$k}->{marks}; # make a copy + $res->{$k} = $target_info->{$k}; + push @{$res->{$k}->{marks}}, @$cmarks if $cmarks; + } + } + } + } + + return $res; +} + +1; diff --git a/PMG/RuleDB/Object.pm b/PMG/RuleDB/Object.pm index 95805af..4229a34 100644 --- a/PMG/RuleDB/Object.pm +++ b/PMG/RuleDB/Object.pm @@ -30,12 +30,6 @@ sub load_attr { croak "never call this method: ERROR"; } -sub load { - my ($ruledb, $objid) = @_; - - return $ruledb->load_object($objid); -} - sub who_match { croak "never call this method: ERROR"; } diff --git a/PMG/Utils.pm b/PMG/Utils.pm index 8de5364..4a4066b 100644 --- a/PMG/Utils.pm +++ b/PMG/Utils.pm @@ -12,6 +12,12 @@ use MIME::Parser; use PVE::SafeSyslog; +sub msgquote { + my $msg = shift || ''; + $msg =~ s/%/%%/g; + return $msg; +} + sub lastid { my ($dbh, $seq) = @_;