X-Git-Url: https://git.proxmox.com/?p=pve-firewall.git;a=blobdiff_plain;f=src%2FPVE%2FFirewall.pm;h=02c602fe17c037ebc6840612066ce6268e07b3f2;hp=d18809a2ef8a417c276b87ccb5d27e60bd2894ef;hb=c6f5cc88d85a9e63654defb0bee4b1c5d6a574cd;hpb=bb272dd311e39cd0d76a01d28c600363e95bc731

diff --git a/src/PVE/Firewall.pm b/src/PVE/Firewall.pm
index d18809a..02c602f 100644
--- a/src/PVE/Firewall.pm
+++ b/src/PVE/Firewall.pm
@@ -4,16 +4,38 @@ use warnings;
 use strict;
 use Data::Dumper;
 use Digest::SHA;
+use PVE::INotify;
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::Cluster;
+use PVE::ProcFSTools;
 use PVE::Tools;
-use PVE::QemuServer;
 use File::Basename;
 use File::Path;
 use IO::File;
 use Net::IP;
 use PVE::Tools qw(run_command lock_file);
+use Encode;
+
+my $clusterfw_conf_filename = "/etc/pve/firewall/cluster.fw";
+
+# dynamically include PVE::QemuServer and PVE::OpenVZ 
+# to avoid dependency problems
+my $have_qemu_server;
+eval {
+    require PVE::QemuServer;
+    $have_qemu_server = 1;
+};
+
+my $have_pve_manager;
+eval {
+    require PVE::OpenVZ;
+    $have_pve_manager = 1;
+};
 
 use Data::Dumper;
 
+my $nodename = PVE::INotify::nodename();
+
 my $pve_fw_lock_filename = "/var/lock/pvefw.lck";
 my $pve_fw_status_filename = "/var/lib/pve-firewall/pvefw.status";
 
@@ -491,23 +513,20 @@ my $icmp_type_names = {
     'address-mask-reply' => 1,
 };
 
-sub get_firewall_macros {
-
-    return $pve_fw_parsed_macros if $pve_fw_parsed_macros;
+sub init_firewall_macros {
 
     $pve_fw_parsed_macros = {};
 
     foreach my $k (keys %$pve_fw_macros) {
-	my $name = lc($k);
-
-	my $macro =  $pve_fw_macros->{$k};
-	$pve_fw_preferred_macro_names->{$name} = $k;
-	$pve_fw_parsed_macros->{$name} = $macro;
+	my $lc_name = lc($k);
+	my $macro = $pve_fw_macros->{$k};
+	$pve_fw_preferred_macro_names->{$lc_name} = $k;
+	$pve_fw_parsed_macros->{$k} = $macro;
     }
-
-    return $pve_fw_parsed_macros;
 }
 
+init_firewall_macros();
+
 my $etc_services;
 
 sub get_etc_services {
@@ -617,14 +636,155 @@ sub parse_port_name_number_or_range {
     return ($nbports);
 }
 
+PVE::JSONSchema::register_format('pve-fw-port-spec', \&pve_fw_verify_port_spec);
+sub pve_fw_verify_port_spec {
+   my ($portstr) = @_;
+
+    parse_port_name_number_or_range($portstr);
+
+   return $portstr;
+}
+
+PVE::JSONSchema::register_format('pve-fw-v4addr-spec', \&pve_fw_verify_v4addr_spec);
+sub pve_fw_verify_v4addr_spec {
+   my ($list) = @_;
+
+   parse_address_list($list);
+
+   return $list;
+}
+
+PVE::JSONSchema::register_format('pve-fw-protocol-spec', \&pve_fw_verify_protocol_spec);
+sub pve_fw_verify_protocol_spec {
+   my ($proto) = @_;
+
+   my $protocols = get_etc_protocols();
+
+   die "unknown protocol '$proto'\n" if $proto &&
+       !(defined($protocols->{byname}->{$proto}) ||
+	 defined($protocols->{byid}->{$proto}));
+
+   return $proto;
+}
+
+
+# helper function for API
+
+my $rule_properties = {
+    pos => {
+	description => "Update rule at position <pos>.",
+	type => 'integer',
+	minimum => 0,
+	optional => 1,
+    },
+    digest => {
+	type => 'string',
+	optional => 1,
+	maxLength => 27,
+	minLength => 27,
+    },
+    type => {
+	type => 'string',
+	optional => 1,
+	enum => ['in', 'out', 'group'],
+    },
+    action => {
+	type => 'string',
+	optional => 1,
+	enum => ['ACCEPT', 'DROP', 'REJECT'],
+    },
+    macro => {
+	type => 'string',
+	optional => 1,
+	maxLength => 128,
+    },
+    iface => get_standard_option('pve-iface', { optional => 1 }),
+    source => {
+	type => 'string', format => 'pve-fw-v4addr-spec',
+	optional => 1,
+    },
+    dest => {
+	type => 'string', format => 'pve-fw-v4addr-spec',
+	optional => 1,
+    },
+    proto => {
+	type => 'string', format => 'pve-fw-protocol-spec',
+	optional => 1,
+    },
+    enable => {
+	type => 'boolean',
+	optional => 1,
+    },
+    sport => {
+	type => 'string', format => 'pve-fw-port-spec',
+	optional => 1,
+    },
+    dport => {
+	type => 'string', format => 'pve-fw-port-spec',
+	optional => 1,
+    },
+    comment => {
+	type => 'string',
+	optional => 1,
+    },
+};
+
+sub cleanup_fw_rule {
+    my ($rule, $digest, $pos) = @_;
+
+    my $r = {};
+
+    foreach my $k (keys %$rule) {
+	next if !$rule_properties->{$k};
+	my $v = $rule->{$k};
+	next if !defined($v);
+	$r->{$k} = $v;
+	$r->{digest} = $digest;
+	$r->{pos} = $pos;
+    }
+
+    return $r;
+}
+
+sub add_rule_properties {
+    my ($properties) = @_;
+
+    foreach my $k (keys %$rule_properties) {
+	$properties->{$k} = $rule_properties->{$k};
+    }
+    
+    return $properties;
+}
+
+sub copy_rule_data {
+    my ($rule, $param) = @_;
+
+    foreach my $k (keys %$rule_properties) {
+	if (defined(my $v = $param->{$k})) {
+	    if ($v eq '' || $v eq '-') {
+		delete $rule->{$k};
+	    } else {
+		$rule->{$k} = $v;
+	    }
+	} else {
+	    delete $rule->{$k};
+	}
+    }
+    return $rule;
+}
+
+# core functions
 my $bridge_firewall_enabled = 0;
 
 sub enable_bridge_firewall {
 
     return if $bridge_firewall_enabled; # only once
 
-    system("echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables");
-    system("echo 1 > /proc/sys/net/bridge/bridge-nf-call-ip6tables");
+    PVE::ProcFSTools::write_proc_entry("/proc/sys/net/bridge/bridge-nf-call-iptables", "1");
+    PVE::ProcFSTools::write_proc_entry("/proc/sys/net/bridge/bridge-nf-call-ip6tables", "1");
+
+    # make sure syncookies are enabled (which is default on newer 3.X kernels anyways)
+    PVE::ProcFSTools::write_proc_entry("/proc/sys/net/ipv4/tcp_syncookies", "1");
 
     $bridge_firewall_enabled = 1;
 }
@@ -643,6 +803,12 @@ sub iptables_restore_cmdlist {
     run_command("/sbin/iptables-restore -n", input => $cmdlist);
 }
 
+sub ipset_restore_cmdlist {
+    my ($cmdlist) = @_;
+
+    run_command(" /usr/sbin/ipset restore", input => $cmdlist);
+}
+
 sub iptables_get_chains {
 
     my $res = {};
@@ -654,7 +820,12 @@ sub iptables_get_chains {
 	return 1 if $name =~ m/^PVEFW-\S+$/;
 
 	return 1 if $name =~ m/^tap\d+i\d+-(:?IN|OUT)$/;
-	return 1 if $name =~ m/^vmbr\d+-(:?FW|IN|OUT)$/;
+
+	return 1 if $name =~ m/^veth\d+.\d+-(:?IN|OUT)$/; # fixme: dev name is configurable
+
+	return 1 if $name =~ m/^venet0-\d+-(:?IN|OUT)$/;
+
+	return 1 if $name =~ m/^vmbr\d+-(:?FW|IN|OUT|IPS)$/;
 	return 1 if $name =~ m/^GROUP-(:?[^\s\-]+)-(:?IN|OUT)$/;
 
 	return undef;
@@ -694,6 +865,43 @@ sub iptables_get_chains {
     return $res;
 }
 
+sub ipset_chain_digest {
+    my ($rules) = @_;
+    my $digest = Digest::SHA->new('sha1');
+    foreach my $rule (sort @$rules) { # note: sorted
+	$digest->add($rule);
+    }
+    return $digest->b64digest;
+}
+
+sub ipset_get_chains {
+
+    my $res = {};
+    my $chains = {};
+
+    my $parser = sub {
+	my $line = shift;
+
+	return if $line =~ m/^#/;
+	return if $line =~ m/^\s*$/;
+	if ($line =~ m/^(\S+)\s(\S+)\s(\S+)/) {
+	    push @{$chains->{$2}}, $line;
+	} else {
+	    # simply ignore the rest
+	    return;
+	}
+    };
+
+    run_command("/usr/sbin/ipset save", outfunc => $parser);
+
+    # compute digest for each chain
+    foreach my $chain (keys %$chains) {
+	$res->{$chain} = ipset_chain_digest($chains->{$chain});
+    }
+
+    return $res;
+}
+
 sub iptables_chain_exist {
     my ($chain) = @_;
 
@@ -716,26 +924,36 @@ sub iptables_rule_exist {
     return 1;
 }
 
-sub ruleset_generate_rule {
+sub ruleset_generate_cmdstr {
     my ($ruleset, $chain, $rule, $actions, $goto) = @_;
 
-    return if $rule->{disable};
+    return if defined($rule->{enable}) && !$rule->{enable};
+
+    die "unable to emit macro - internal error" if $rule->{macro}; # should not happen
 
-    my $cmd = '';
+    my $nbdport = defined($rule->{dport}) ? parse_port_name_number_or_range($rule->{dport}) : 0;
+    my $nbsport = defined($rule->{sport}) ? parse_port_name_number_or_range($rule->{sport}) : 0;
+    my $nbsource = $rule->{source} ? parse_address_list( $rule->{source}) : 0;
+    my $nbdest = $rule->{dest} ? parse_address_list($rule->{dest}) : 0;
 
-    $cmd .= " -m iprange --src-range" if $rule->{nbsource} && $rule->{nbsource} > 1;
-    $cmd .= " -s $rule->{source}" if $rule->{source};
-    $cmd .= " -m iprange --dst-range" if $rule->{nbdest} && $rule->{nbdest} > 1;
-    $cmd .= " -d $rule->{dest}" if $rule->{dest};
+    my @cmd = ();
+
+    push @cmd, "-i $rule->{iface_in}" if $rule->{iface_in};
+    push @cmd, "-o $rule->{iface_out}" if $rule->{iface_out};
+
+    push @cmd, "-m iprange --src-range" if $nbsource > 1;
+    push @cmd, "-s $rule->{source}" if $rule->{source};
+    push @cmd, "-m iprange --dst-range" if $nbdest > 1;
+    push @cmd, "-d $rule->{dest}" if $rule->{dest};
 
     if ($rule->{proto}) {
-	$cmd .= " -p $rule->{proto}";
+	push @cmd, "-p $rule->{proto}";
 
 	my $multiport = 0;
-	$multiport++ if $rule->{nbdport} && ($rule->{nbdport} > 1);
-	$multiport++ if $rule->{nbsport} && ($rule->{nbsport} > 1);
+	$multiport++ if $nbdport > 1;
+	$multiport++ if $nbsport > 1;
 
-	$cmd .= " --match multiport" if $multiport;
+	push @cmd, "--match multiport" if $multiport;
 
 	die "multiport: option '--sports' cannot be used together with '--dports'\n" 
 	    if ($multiport == 2) && ($rule->{dport} ne $rule->{sport});
@@ -744,25 +962,25 @@ sub ruleset_generate_rule {
 	    if ($rule->{proto} && $rule->{proto} eq 'icmp') {
 		# Note: we use dport to store --icmp-type
 		die "unknown icmp-type '$rule->{dport}'\n" if !defined($icmp_type_names->{$rule->{dport}});
-		$cmd .= " -m icmp --icmp-type $rule->{dport}";
+		push @cmd, "-m icmp --icmp-type $rule->{dport}";
 	    } else {
-		if ($rule->{nbdport} && $rule->{nbdport} > 1) {
+		if ($nbdport > 1) {
 		    if ($multiport == 2) {
-			$cmd .= " --ports $rule->{dport}";
+			push @cmd,  "--ports $rule->{dport}";
 		    } else {
-			$cmd .= " --dports $rule->{dport}";
+			push @cmd, "--dports $rule->{dport}";
 		    }
 		} else {
-		    $cmd .= " --dport $rule->{dport}";
+		    push @cmd, "--dport $rule->{dport}";
 		}
 	    }
 	}
 
 	if ($rule->{sport}) {
-	    if ($rule->{nbsport} && $rule->{nbsport} > 1) {
-		$cmd .= " --sports $rule->{sport}" if $multiport != 2;
+	    if ($nbsport > 1) {
+		push @cmd, "--sports $rule->{sport}" if $multiport != 2;
 	    } else {
-		$cmd .= " --sport $rule->{sport}";
+		push @cmd, "--sport $rule->{sport}";
 	    }
 	}
     } elsif ($rule->{dport} || $rule->{sport}) {
@@ -770,21 +988,94 @@ sub ruleset_generate_rule {
 	warn "ignoring source port '$rule->{sport}' - no protocol specified\n" if $rule->{sport};
     }
 
-    $cmd .= " -m addrtype --dst-type $rule->{dsttype}" if $rule->{dsttype};
+    push @cmd, "-m addrtype --dst-type $rule->{dsttype}" if $rule->{dsttype};
 
     if (my $action = $rule->{action}) {
 	$action = $actions->{$action} if defined($actions->{$action}); 
 	$goto = 1 if !defined($goto) && $action eq 'PVEFW-SET-ACCEPT-MARK';
-	$cmd .= $goto ? " -g $action" : " -j $action";
+	push @cmd, $goto ? "-g $action" : "-j $action";
     }
 
-    ruleset_addrule($ruleset, $chain, $cmd) if $cmd;
+    return scalar(@cmd) ? join(' ', @cmd) : undef;
+}
+
+my $apply_macro = sub {
+    my ($macro_name, $param) = @_;
+
+    my $macro_rules = $pve_fw_parsed_macros->{$macro_name};
+    die "unknown macro '$macro_name'\n" if !$macro_rules; # should not happen
+
+    my $rules = [];
+
+    foreach my $templ (@$macro_rules) {
+	my $rule = {};
+	my $param_used = {};
+	foreach my $k (keys %$templ) {
+	    my $v = $templ->{$k};
+	    if ($v eq 'PARAM') {
+		$v = $param->{$k};
+		$param_used->{$k} = 1;
+	    } elsif ($v eq 'DEST') {
+		$v = $param->{dest};
+		$param_used->{dest} = 1;
+	    } elsif ($v eq 'SOURCE') {
+		$v = $param->{source};
+		$param_used->{source} = 1;
+	    }
+
+	    die "missing parameter '$k' in macro '$macro_name'\n" if !defined($v);
+	    $rule->{$k} = $v;
+	}
+	foreach my $k (keys %$param) {
+	    next if $k eq 'macro';
+	    next if !defined($param->{$k});
+	    next if $param_used->{$k};
+	    if (defined($rule->{$k})) {
+		die "parameter '$k' already define in macro (value = '$rule->{$k}')\n"
+		    if $rule->{$k} ne $param->{$k};
+	    } else {
+		$rule->{$k} = $param->{$k};
+	    }
+	}
+	push @$rules, $rule;
+    }
+
+    return $rules;
+};
+
+sub ruleset_generate_rule {
+    my ($ruleset, $chain, $rule, $actions, $goto) = @_;
+
+    my $rules;
+
+    if ($rule->{macro}) {
+	$rules = &$apply_macro($rule->{macro}, $rule);
+    } else {
+	$rules = [ $rule ];
+    }
+
+    foreach my $tmp (@$rules) { 
+	if (my $cmdstr = ruleset_generate_cmdstr($ruleset, $chain, $tmp, $actions, $goto)) {
+	    ruleset_addrule($ruleset, $chain, $cmdstr);
+	}
+    }
+}
+
+sub ruleset_generate_rule_insert {
+    my ($ruleset, $chain, $rule, $actions, $goto) = @_;
+
+    die "implement me" if $rule->{macro}; # not implemented, because not needed so far
+
+    if (my $cmdstr = ruleset_generate_cmdstr($ruleset, $chain, $rule, $actions, $goto)) {
+	ruleset_insertrule($ruleset, $chain, $cmdstr);
+    }
 }
 
 sub ruleset_create_chain {
     my ($ruleset, $chain) = @_;
 
     die "Invalid chain name '$chain' (28 char max)\n" if length($chain) > 28;
+    die "chain name may not contain collons\n" if $chain =~ m/:/; # because of log format
 
     die "chain '$chain' already exists\n" if $ruleset->{$chain};
 
@@ -813,146 +1104,305 @@ sub ruleset_insertrule {
    unshift @{$ruleset->{$chain}}, "-A $chain $rule";
 }
 
+sub get_log_rule_base {
+    my ($chain, $vmid, $msg, $loglevel) = @_;
+    
+    die "internal error - no log level" if !defined($loglevel);
+
+    $vmid = 0 if !defined($vmid);
+
+    # Note: we use special format for prefix to pass further 
+    # info to log daemon (VMID, LOGVELEL and CHAIN)
+
+    return "-j NFLOG --nflog-prefix \":$vmid:$loglevel:$chain: $msg\"";
+}
+
+sub ruleset_addlog {
+    my ($ruleset, $chain, $vmid, $msg, $loglevel, $rule) = @_;
+
+    return if !defined($loglevel);
+
+    my $logrule = get_log_rule_base($chain, $vmid, $msg, $loglevel);
+
+    $logrule = "$rule $logrule" if defined($rule);
+
+    ruleset_addrule($ruleset, $chain, $logrule)
+}
+
 sub generate_bridge_chains {
-    my ($ruleset, $bridge) = @_;
+    my ($ruleset, $hostfw_conf, $bridge, $routing_table) = @_;
 
-    if (!ruleset_chain_exist($ruleset, "PVEFW-FORWARD")){
-	ruleset_create_chain($ruleset, "PVEFW-FORWARD");
-	ruleset_addrule($ruleset, "PVEFW-FORWARD", "-m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT");
-    }
+    my $options = $hostfw_conf->{options} || {};
+
+    die "error: detected direct route to bridge '$bridge'\n"
+	if !$options->{allow_bridge_route} && $routing_table->{$bridge};
 
     if (!ruleset_chain_exist($ruleset, "$bridge-FW")) {
 	ruleset_create_chain($ruleset, "$bridge-FW");
-	ruleset_addrule($ruleset, "PVEFW-FORWARD", "-o $bridge -m physdev --physdev-is-bridged -j $bridge-FW");
-	ruleset_addrule($ruleset, "PVEFW-FORWARD", "-i $bridge -m physdev --physdev-is-bridged -j $bridge-FW");
-	ruleset_addrule($ruleset, "PVEFW-FORWARD", "-o $bridge -j DROP");  # disable interbridge routing
-	ruleset_addrule($ruleset, "PVEFW-FORWARD", "-i $bridge -j DROP"); # disable interbridge routing
+	ruleset_addrule($ruleset, "PVEFW-FORWARD", "-o $bridge -m physdev --physdev-is-out -j $bridge-FW");
+	ruleset_addrule($ruleset, "PVEFW-FORWARD", "-i $bridge -m physdev --physdev-is-in -j $bridge-FW");
     }
 
     if (!ruleset_chain_exist($ruleset, "$bridge-OUT")) {
 	ruleset_create_chain($ruleset, "$bridge-OUT");
-	ruleset_addrule($ruleset, "$bridge-FW", "-m physdev --physdev-is-bridged --physdev-is-in -j $bridge-OUT");
+	ruleset_addrule($ruleset, "$bridge-FW", "-m physdev --physdev-is-in -j $bridge-OUT");
+	ruleset_insertrule($ruleset, "PVEFW-INPUT", "-i $bridge -m physdev --physdev-is-in -j $bridge-OUT");
     }
 
     if (!ruleset_chain_exist($ruleset, "$bridge-IN")) {
 	ruleset_create_chain($ruleset, "$bridge-IN");
-	ruleset_addrule($ruleset, "$bridge-FW", "-m physdev --physdev-is-bridged --physdev-is-out -j $bridge-IN");
+	ruleset_addrule($ruleset, "$bridge-FW", "-m physdev --physdev-is-out -j $bridge-IN");
 	ruleset_addrule($ruleset, "$bridge-FW", "-m mark --mark 1 -j ACCEPT");
 	# accept traffic to unmanaged bridge ports
-	ruleset_addrule($ruleset, "$bridge-FW", "-m physdev --physdev-is-bridged --physdev-is-out -j ACCEPT ");
+	ruleset_addrule($ruleset, "$bridge-FW", "-m physdev --physdev-is-out -j ACCEPT ");
     }
 }
 
-sub generate_tap_rules_direction {
-    my ($ruleset, $group_rules, $iface, $netid, $macaddr, $vmfw_conf, $bridge, $direction) = @_;
+sub ruleset_add_chain_policy {
+    my ($ruleset, $chain, $vmid, $policy, $loglevel, $accept_action) = @_;
 
-    my $lc_direction = lc($direction);
-    my $rules = $vmfw_conf->{$lc_direction};
+    if ($policy eq 'ACCEPT') {
 
-    my $options = $vmfw_conf->{options};
-    my $loglevel = get_option_log_level($options, "log_level_${lc_direction}");
+	ruleset_generate_rule($ruleset, $chain, { action => 'ACCEPT' },
+			      { ACCEPT =>  $accept_action});
 
-    my $tapchain = "$iface-$direction";
+    } elsif ($policy eq 'DROP') {
 
-    ruleset_create_chain($ruleset, $tapchain);
+	ruleset_addrule($ruleset, $chain, "-j PVEFW-Drop");
+
+	ruleset_addlog($ruleset, $chain, $vmid, "policy $policy: ", $loglevel);
+
+	ruleset_addrule($ruleset, $chain, "-j DROP");
+    } elsif ($policy eq 'REJECT') {
+	ruleset_addrule($ruleset, $chain, "-j PVEFW-Reject");
+
+	ruleset_addlog($ruleset, $chain, $vmid, "policy $policy: ", $loglevel);
+
+	ruleset_addrule($ruleset, $chain, "-g PVEFW-reject");
+    } else {
+	# should not happen
+	die "internal error: unknown policy '$policy'";
+    }
+}
+
+sub ruleset_create_vm_chain {
+    my ($ruleset, $chain, $options, $macaddr, $direction) = @_;
+
+    ruleset_create_chain($ruleset, $chain);
+    my $accept = generate_nfqueue($options);
 
     if (!(defined($options->{nosmurfs}) && $options->{nosmurfs} == 0)) {
-	ruleset_addrule($ruleset, $tapchain, "-m conntrack --ctstate INVALID,NEW -j PVEFW-smurfs");
+	ruleset_addrule($ruleset, $chain, "-m conntrack --ctstate INVALID,NEW -j PVEFW-smurfs");
     }
 
     if (!(defined($options->{dhcp}) && $options->{dhcp} == 0)) {
-	ruleset_addrule($ruleset, $tapchain, "-p udp -m udp --dport 67:68 -j ACCEPT");
+	if ($direction eq 'OUT') {
+	    ruleset_generate_rule($ruleset, $chain, { action => 'PVEFW-SET-ACCEPT-MARK', 
+						      proto => 'udp', sport => 68, dport => 67 });
+	} else {
+	    ruleset_generate_rule($ruleset, $chain, { action => 'ACCEPT', 
+						      proto => 'udp', sport => 67, dport => 68 });
+	}
     }
 
     if ($options->{tcpflags}) {
-	ruleset_addrule($ruleset, $tapchain, "-p tcp -j PVEFW-tcpflags");
+	ruleset_addrule($ruleset, $chain, "-p tcp -j PVEFW-tcpflags");
     }
 
-    ruleset_addrule($ruleset, $tapchain, "-m conntrack --ctstate INVALID -j DROP");
-    ruleset_addrule($ruleset, $tapchain, "-m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT");
+    ruleset_addrule($ruleset, $chain, "-m conntrack --ctstate INVALID -j DROP");
+    if($direction eq 'OUT'){
+	ruleset_addrule($ruleset, $chain, "-m conntrack --ctstate RELATED,ESTABLISHED -g PVEFW-SET-ACCEPT-MARK");
 
-    if ($direction eq 'OUT' && defined($macaddr) && 
-	!(defined($options->{macfilter}) && $options->{macfilter} == 0)) {
-	ruleset_addrule($ruleset, $tapchain, "-m mac ! --mac-source $macaddr -j DROP");
+    }else{
+	ruleset_addrule($ruleset, $chain, "-m conntrack --ctstate RELATED,ESTABLISHED -j $accept");
+    }
+    if ($direction eq 'OUT') {
+	if (defined($macaddr) && !(defined($options->{macfilter}) && $options->{macfilter} == 0)) {
+	    ruleset_addrule($ruleset, $chain, "-m mac ! --mac-source $macaddr -j DROP");
+	}
+	ruleset_addrule($ruleset, $chain, "-j MARK --set-mark 0"); # clear mark
     }
 
-    if ($rules) {
-        foreach my $rule (@$rules) {
-	    next if $rule->{iface} && $rule->{iface} ne $netid;
-	    # we go to $bridge-IN if accept in out rules
-	    if($rule->{action}  =~ m/^(GROUP-(\S+))$/){
-		$rule->{action} .= "-$direction";
-		# generate empty group rule if don't exist
-		if(!ruleset_chain_exist($ruleset, $rule->{action})){
-		    generate_group_rules($ruleset, $group_rules, $2);
-		}
-		ruleset_generate_rule($ruleset, $tapchain, $rule);
-		ruleset_addrule($ruleset, $tapchain, "-m mark --mark 1 -j RETURN")
-		    if $direction eq 'OUT';
+
+}
+
+sub ruleset_generate_vm_rules {
+    my ($ruleset, $rules, $cluster_conf, $chain, $netid, $direction, $options) = @_;
+
+    my $lc_direction = lc($direction);
+
+    foreach my $rule (@$rules) {
+	next if $rule->{iface} && $rule->{iface} ne $netid;
+	next if !$rule->{enable};
+	if ($rule->{type} eq 'group') {
+	    my $group_chain = "GROUP-$rule->{action}-$direction"; 
+	    if(!ruleset_chain_exist($ruleset, $group_chain)){
+		generate_group_rules($ruleset, $cluster_conf, $rule->{action});
+	    }
+	    ruleset_addrule($ruleset, $chain, "-j $group_chain");
+	    if ($direction eq 'OUT'){
+		ruleset_addrule($ruleset, $chain, "-m mark --mark 1 -j RETURN");
+	    }else{
+		my $accept = generate_nfqueue($options);
+		ruleset_addrule($ruleset, $chain, "-m mark --mark 1 -j $accept");
+	    }
+
+	} else {
+	    next if $rule->{type} ne $lc_direction;
+	    if ($direction eq 'OUT') {
+		ruleset_generate_rule($ruleset, $chain, $rule, 
+				      { ACCEPT => "PVEFW-SET-ACCEPT-MARK", REJECT => "PVEFW-reject" });
 	    } else {
-		if ($direction eq 'OUT') {
-		    ruleset_generate_rule($ruleset, $tapchain, $rule, 
-					  { ACCEPT => "PVEFW-SET-ACCEPT-MARK", REJECT => "PVEFW-reject" });
-		} else {
-		    ruleset_generate_rule($ruleset, $tapchain, $rule, { REJECT => "PVEFW-reject" });
-		}
+		my $accept = generate_nfqueue($options);
+		ruleset_generate_rule($ruleset, $chain, $rule, { ACCEPT => $accept , REJECT => "PVEFW-reject" });
 	    }
 	}
     }
+}
+
+sub generate_nfqueue {
+    my ($options) = @_;
+
+    my $action = "";
+    if($options->{ips}){
+	$action = "NFQUEUE";
+	if($options->{ips_queues} && $options->{ips_queues} =~ m/^(\d+)(:(\d+))?$/) {
+	    if(defined($3) && defined($1)) {
+		$action .= " --queue-balance $1:$3";
+	    }elsif (defined($1)) {
+		$action .= " --queue-num $1";
+	    }
+	}
+	$action .= " --queue-bypass";
+    }else{
+	$action = "ACCEPT";
+    }
+
+    return $action;
+}
+
+sub ruleset_generate_vm_ipsrules {
+    my ($ruleset, $options, $direction, $iface, $bridge) = @_;
+
+    if ($options->{ips} && $direction eq 'IN') {
+	my $nfqueue = generate_nfqueue($options);
+
+	if (!ruleset_chain_exist($ruleset, "$bridge-IPS")) {
+	    ruleset_create_chain($ruleset, "PVEFW-IPS");
+	}
+
+	if (!ruleset_chain_exist($ruleset, "$bridge-IPS")) {
+	    ruleset_create_chain($ruleset, "$bridge-IPS");
+	    ruleset_insertrule($ruleset, "PVEFW-IPS", "-o $bridge -m physdev --physdev-is-out -j $bridge-IPS");
+	}
+
+        ruleset_addrule($ruleset, "$bridge-IPS", "-m physdev --physdev-out $iface --physdev-is-bridged -j $nfqueue");
+    }
+}
+
+sub generate_venet_rules_direction {
+    my ($ruleset, $cluster_conf, $vmfw_conf, $vmid, $ip, $direction) = @_;
+
+    parse_address_list($ip); # make sure we have a valid $ip list
+
+    my $lc_direction = lc($direction);
+
+    my $rules = $vmfw_conf->{rules};
+
+    my $options = $vmfw_conf->{options};
+    my $loglevel = get_option_log_level($options, "log_level_${lc_direction}");
+
+    my $chain = "venet0-$vmid-$direction";
+
+    ruleset_create_vm_chain($ruleset, $chain, $options, undef, $direction);
+
+    ruleset_generate_vm_rules($ruleset, $rules, $cluster_conf, $chain, 'venet', $direction);
 
     # implement policy
     my $policy;
 
     if ($direction eq 'OUT') {
-	$policy = $options->{'policy-out'} || 'ACCEPT'; # allow everything by default
+	$policy = $options->{policy_out} || 'ACCEPT'; # allow everything by default
     } else {
-	$policy = $options->{'policy-in'} || 'DROP'; # allow everything by default
+	$policy = $options->{policy_in} || 'DROP'; # allow nothing by default
     }
 
-    if ($policy eq 'ACCEPT') {
-	if ($direction eq 'OUT') {
-	    ruleset_addrule($ruleset, $tapchain, "-g PVEFW-SET-ACCEPT-MARK");
-	} else {
-	    ruleset_addrule($ruleset, $tapchain, "-j ACCEPT");
-	}
-    } elsif ($policy eq 'DROP') {
+    my $accept = generate_nfqueue($options);
+    my $accept_action = $direction eq 'OUT' ? "PVEFW-SET-ACCEPT-MARK" : $accept;
+    ruleset_add_chain_policy($ruleset, $chain, $vmid, $policy, $loglevel, $accept_action);
 
-	ruleset_addrule($ruleset, $tapchain, "-j PVEFW-Drop");
+    # plug into FORWARD, INPUT and OUTPUT chain
+    if ($direction eq 'OUT') {
+	ruleset_generate_rule_insert($ruleset, "PVEFW-FORWARD", {
+	    action => $chain,
+	    source => $ip,
+	    iface_in => 'venet0'});
+
+	ruleset_generate_rule_insert($ruleset, "PVEFW-INPUT", {
+	    action => $chain,
+	    source => $ip,
+	    iface_in => 'venet0'});
+    } else {
+	ruleset_generate_rule($ruleset, "PVEFW-FORWARD", {
+	    action => $chain,
+	    dest => $ip,
+	    iface_out => 'venet0'});
+
+	ruleset_generate_rule($ruleset, "PVEFW-OUTPUT", {
+	    action => $chain,
+	    dest => $ip,
+	    iface_out => 'venet0'});
+    }
+}
 
-	ruleset_addrule($ruleset, $tapchain, "-j LOG --log-prefix \"$tapchain-dropped: \" --log-level $loglevel")
-	    if defined($loglevel);
+sub generate_tap_rules_direction {
+    my ($ruleset, $cluster_conf, $iface, $netid, $macaddr, $vmfw_conf, $vmid, $bridge, $direction) = @_;
 
-	ruleset_addrule($ruleset, $tapchain, "-j DROP");
-    } elsif ($policy eq 'REJECT') {
-	ruleset_addrule($ruleset, $tapchain, "-j PVEFW-Reject");
+    my $lc_direction = lc($direction);
 
-	ruleset_addrule($ruleset, $tapchain, "-j LOG --log-prefix \"$tapchain-reject: \" --log-level $loglevel")
-	    if defined($loglevel);
+    my $rules = $vmfw_conf->{rules};
 
-	ruleset_addrule($ruleset, $tapchain, "-g PVEFW-reject");
+    my $options = $vmfw_conf->{options};
+    my $loglevel = get_option_log_level($options, "log_level_${lc_direction}");
+
+    my $tapchain = "$iface-$direction";
+
+    ruleset_create_vm_chain($ruleset, $tapchain, $options, $macaddr, $direction);
+
+    ruleset_generate_vm_rules($ruleset, $rules, $cluster_conf, $tapchain, $netid, $direction, $options);
+
+    ruleset_generate_vm_ipsrules($ruleset, $options, $direction, $iface, $bridge);
+
+    # implement policy
+    my $policy;
+
+    if ($direction eq 'OUT') {
+	$policy = $options->{policy_out} || 'ACCEPT'; # allow everything by default
     } else {
-	# should not happen
-	die "internal error: unknown policy '$policy'";
+	$policy = $options->{policy_in} || 'DROP'; # allow nothing by default
     }
 
-    # plug the tap chain to bridge chain
-    my $physdevdirection = $direction eq 'IN' ? "out" : "in";
-    my $rule = "-m physdev --physdev-$physdevdirection $iface --physdev-is-bridged -j $tapchain";
-    ruleset_insertrule($ruleset, "$bridge-$direction", $rule);
+    my $accept = generate_nfqueue($options);
+    my $accept_action = $direction eq 'OUT' ? "PVEFW-SET-ACCEPT-MARK" : $accept;
+    ruleset_add_chain_policy($ruleset, $tapchain, $vmid, $policy, $loglevel, $accept_action);
 
-    if ($direction eq 'OUT'){
-	# add tap->host rules
-	my $rule = "-m physdev --physdev-$physdevdirection $iface -j $tapchain";
-	ruleset_addrule($ruleset, "PVEFW-INPUT", $rule);
+    # plug the tap chain to bridge chain
+    if ($direction eq 'IN') {
+	ruleset_insertrule($ruleset, "$bridge-IN",
+			   "-m physdev --physdev-is-bridged --physdev-out $iface -j $tapchain");
+    } else {
+	ruleset_insertrule($ruleset, "$bridge-OUT",
+			   "-m physdev --physdev-in $iface -j $tapchain");
     }
 }
 
-sub enablehostfw {
-    my ($ruleset, $rules, $group_rules) = @_;
+sub enable_host_firewall {
+    my ($ruleset, $hostfw_conf, $cluster_conf) = @_;
 
     # fixme: allow security groups
 
-    my $options = $rules->{options};
+    my $options = $hostfw_conf->{options};
+    my $rules = $hostfw_conf->{rules};
 
     # host inbound firewall
     my $chain = "PVEFW-HOST-IN";
@@ -960,24 +1410,32 @@ sub enablehostfw {
 
     my $loglevel = get_option_log_level($options, "log_level_in");
 
+    if (!(defined($options->{nosmurfs}) && $options->{nosmurfs} == 0)) {
+	ruleset_addrule($ruleset, $chain, "-m conntrack --ctstate INVALID,NEW -j PVEFW-smurfs");
+    }
+
+    if ($options->{tcpflags}) {
+	ruleset_addrule($ruleset, $chain, "-p tcp -j PVEFW-tcpflags");
+    }
+
     ruleset_addrule($ruleset, $chain, "-m conntrack --ctstate INVALID -j DROP");
     ruleset_addrule($ruleset, $chain, "-m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT");
     ruleset_addrule($ruleset, $chain, "-i lo -j ACCEPT");
     ruleset_addrule($ruleset, $chain, "-m addrtype --dst-type MULTICAST -j ACCEPT");
-    ruleset_addrule($ruleset, $chain, "-p udp -m conntrack --ctstate NEW -m multiport --dports 5404,5405 -j ACCEPT");
+    ruleset_addrule($ruleset, $chain, "-p udp -m conntrack --ctstate NEW --dport 5404:5405 -j ACCEPT");
     ruleset_addrule($ruleset, $chain, "-p udp -m udp --dport 9000 -j ACCEPT");  #corosync
 
-    if ($rules->{in}) {
-        foreach my $rule (@{$rules->{in}}) {
-            # we use RETURN because we need to check also tap rules
-            ruleset_generate_rule($ruleset, $chain, $rule, { ACCEPT => 'RETURN', REJECT => "PVEFW-reject" });
-        }
-    }
+    # we use RETURN because we need to check also tap rules
+    my $accept_action = 'RETURN';
 
-    ruleset_addrule($ruleset, $chain, "-j LOG --log-prefix \"kvmhost-IN dropped: \" --log-level $loglevel")
-	if defined($loglevel);
+    foreach my $rule (@$rules) {
+	next if $rule->{type} ne 'in';
+	ruleset_generate_rule($ruleset, $chain, $rule, { ACCEPT => $accept_action, REJECT => "PVEFW-reject" });
+    }
 
-    ruleset_addrule($ruleset, $chain, "-j DROP");
+    # implement input policy
+    my $policy = $options->{policy_in} || 'DROP'; # allow nothing by default
+    ruleset_add_chain_policy($ruleset, $chain, 0, $policy, $loglevel, $accept_action);
 
     # host outbound firewall
     $chain = "PVEFW-HOST-OUT";
@@ -989,40 +1447,39 @@ sub enablehostfw {
     ruleset_addrule($ruleset, $chain, "-m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT");
     ruleset_addrule($ruleset, $chain, "-o lo -j ACCEPT");
     ruleset_addrule($ruleset, $chain, "-m addrtype --dst-type MULTICAST -j ACCEPT");
-    ruleset_addrule($ruleset, $chain, "-p udp -m conntrack --ctstate NEW -m multiport --dports 5404,5405 -j ACCEPT");
+    ruleset_addrule($ruleset, $chain, "-p udp -m conntrack --ctstate NEW --dport 5404:5405 -j ACCEPT");
     ruleset_addrule($ruleset, $chain, "-p udp -m udp --dport 9000 -j ACCEPT"); #corosync
 
-    if ($rules->{out}) {
-        foreach my $rule (@{$rules->{out}}) {
-            # we use RETURN because we need to check also tap rules
-            ruleset_generate_rule($ruleset, $chain, $rule, { ACCEPT => 'RETURN', REJECT => "PVEFW-reject" });
-        }
-    }
+    # we use RETURN because we may want to check other thigs later
+    $accept_action = 'RETURN';
 
-    ruleset_addrule($ruleset, $chain, "-j LOG --log-prefix \"kvmhost-OUT dropped: \" --log-level $loglevel")
-	if defined($loglevel);
+    foreach my $rule (@$rules) {
+	next if $rule->{type} ne 'out';
+	ruleset_generate_rule($ruleset, $chain, $rule, { ACCEPT => $accept_action, REJECT => "PVEFW-reject" });
+    }
 
-    ruleset_addrule($ruleset, $chain, "-j DROP");
+    # implement output policy
+    $policy = $options->{policy_out} || 'ACCEPT'; # allow everything by default
+    ruleset_add_chain_policy($ruleset, $chain, 0, $policy, $loglevel, $accept_action);
 
     ruleset_addrule($ruleset, "PVEFW-OUTPUT", "-j PVEFW-HOST-OUT");
     ruleset_addrule($ruleset, "PVEFW-INPUT", "-j PVEFW-HOST-IN");
 }
 
 sub generate_group_rules {
-    my ($ruleset, $group_rules, $group) = @_;
-
-    my $rules = $group_rules->{$group};
+    my ($ruleset, $cluster_conf, $group) = @_;
+    die "no such security group '$group'\n" if !$cluster_conf->{groups}->{$group};
 
-    die "no such security group '$group'\n" if !$rules;
+    my $rules = $cluster_conf->{groups}->{$group};
 
     my $chain = "GROUP-${group}-IN";
 
     ruleset_create_chain($ruleset, $chain);
+    ruleset_addrule($ruleset, $chain, "-j MARK --set-mark 0"); # clear mark
 
-    if ($rules->{in}) {
-        foreach my $rule (@{$rules->{in}}) {
- 	    ruleset_generate_rule($ruleset, $chain, $rule, { REJECT => "PVEFW-reject" });
-        }
+    foreach my $rule (@$rules) {
+	next if $rule->{type} ne 'in';
+	ruleset_generate_rule($ruleset, $chain, $rule, { ACCEPT => "PVEFW-SET-ACCEPT-MARK", REJECT => "PVEFW-reject" });
     }
 
     $chain = "GROUP-${group}-OUT";
@@ -1030,13 +1487,12 @@ sub generate_group_rules {
     ruleset_create_chain($ruleset, $chain);
     ruleset_addrule($ruleset, $chain, "-j MARK --set-mark 0"); # clear mark
 
-    if ($rules->{out}) {
-        foreach my $rule (@{$rules->{out}}) {
-            # we use PVEFW-SET-ACCEPT-MARK (Instead of ACCEPT) because we need to
-	    # check also other tap rules later
-            ruleset_generate_rule($ruleset, $chain, $rule, 
-				  { ACCEPT => 'PVEFW-SET-ACCEPT-MARK', REJECT => "PVEFW-reject" });
-        }
+    foreach my $rule (@$rules) {
+	next if $rule->{type} ne 'out';
+	# we use PVEFW-SET-ACCEPT-MARK (Instead of ACCEPT) because we need to
+	# check also other tap rules later
+	ruleset_generate_rule($ruleset, $chain, $rule, 
+			      { ACCEPT => 'PVEFW-SET-ACCEPT-MARK', REJECT => "PVEFW-reject" });
     }
 }
 
@@ -1049,58 +1505,59 @@ for (my $i = 0; $i < $MAX_NETS; $i++)  {
 sub parse_fw_rule {
     my ($line, $need_iface, $allow_groups) = @_;
 
-    my $macros = get_firewall_macros();
-    my $protocols = get_etc_protocols();
-
-    my ($action, $iface, $source, $dest, $proto, $dport, $sport);
+    my ($type, $action, $iface, $source, $dest, $proto, $dport, $sport);
 
     # we can add single line comments to the end of the rule
-    my $comment = $1 if $line =~ s/#\s*(.*?)\s*$//;
+    my $comment = decode('utf8', $1) if $line =~ s/#\s*(.*?)\s*$//;
 
     # we can disable a rule when prefixed with '|'
-    my $disable = 1 if  $line =~ s/^\|//;
+    my $enable = 1;
+
+    $enable = 0 if $line =~ s/^\|//;
 
     my @data = split(/\s+/, $line);
-    my $expected_elements = $need_iface ? 7 : 6;
+    my $expected_elements = $need_iface ? 8 : 7;
 
     die "wrong number of rule elements\n" if scalar(@data) > $expected_elements;
 
     if ($need_iface) {
-	($action, $iface, $source, $dest, $proto, $dport, $sport) = @data
+	($type, $action, $iface, $source, $dest, $proto, $dport, $sport) = @data
     } else {
-	($action, $source, $dest, $proto, $dport, $sport) =  @data;
+	($type, $action, $source, $dest, $proto, $dport, $sport) =  @data;
     }
 
-    die "incomplete rule\n" if !$action;
+    die "incomplete rule\n" if ! ($type && $action);
 
     my $macro;
-    my $macro_name;
-
-    if ($action =~ m/^(ACCEPT|DROP|REJECT)$/) {
-	# OK
-    } elsif ($allow_groups && $action =~ m/^GROUP-(:?\S+)$/) {
-	# OK
-    } elsif ($action =~ m/^(\S+)\((ACCEPT|DROP|REJECT)\)$/) {
-	($macro_name, $action) = ($1, $2);
-	my $lc_macro_name = lc($macro_name);
-	my $preferred_name = $pve_fw_preferred_macro_names->{$lc_macro_name};
-	$macro_name = $preferred_name if $preferred_name;
-	$macro = $macros->{$lc_macro_name};
-	die "unknown macro '$macro_name'\n" if !$macro;
+
+    $type = lc($type);
+
+    if ($type eq  'in' || $type eq 'out') {
+	if ($action =~ m/^(ACCEPT|DROP|REJECT)$/) {
+	    # OK
+	} elsif ($action =~ m/^(\S+)\((ACCEPT|DROP|REJECT)\)$/) {
+	    $action = $2;
+	    my $preferred_name = $pve_fw_preferred_macro_names->{lc($1)};
+	    die "unknown macro '$1'\n" if !$preferred_name;
+	    $macro = $preferred_name;
+	} else {
+	    die "unknown action '$action'\n";
+	}
+    } elsif ($type eq 'group') {
+	die "wrong number of rule elements\n" if scalar(@data) != 3;
+	die "groups disabled\n" if !$allow_groups;
+
+	die "invalid characters in group name\n" if $action !~ m/^[A-Za-z0-9_\-]+$/;	
     } else {
-	die "unknown action '$action'\n";
+	die "unknown rule type '$type'\n";
     }
 
     if ($need_iface) {
 	$iface = undef if $iface && $iface eq '-';
-	die "unknown interface '$iface'\n"
-	    if defined($iface) && !$valid_netdev_names->{$iface};
     }
 
     $proto = undef if $proto && $proto eq '-';
-    die "unknown protokol '$proto'\n" if $proto &&
-	!(defined($protocols->{byname}->{$proto}) ||
-	  defined($protocols->{byid}->{$proto}));
+    pve_fw_verify_protocol_spec($proto) if $proto;
 
     $source = undef if $source && $source eq '-';
     $dest = undef if $dest && $dest eq '-';
@@ -1108,72 +1565,25 @@ sub parse_fw_rule {
     $dport = undef if $dport && $dport eq '-';
     $sport = undef if $sport && $sport eq '-';
 
-    my $nbsource = undef;
-    my $nbdest = undef;
-
-    $nbsource = parse_address_list($source) if $source;
-    $nbdest = parse_address_list($dest) if $dest;
+    parse_port_name_number_or_range($dport) if defined($dport);
+    parse_port_name_number_or_range($sport) if defined($sport);
+ 
+    parse_address_list($source) if $source;
+    parse_address_list($dest) if $dest;
 
-    my $rules = [];
-
-    my $param = {
-	disable => $disable,
+    return {
+	type => $type,
+	enable => $enable,
 	comment => $comment,
 	action => $action,
+	macro => $macro,
 	iface => $iface,
 	source => $source,
 	dest => $dest,
-	nbsource => $nbsource,
-	nbdest => $nbdest,
 	proto => $proto,
 	dport => $dport,
 	sport => $sport,
     };
-
-    if ($macro) {
-	foreach my $templ (@$macro) {
-	    my $rule = {};
-	    my $param_used = {};
-	    foreach my $k (keys %$templ) {
-		my $v = $templ->{$k};
-		if ($v eq 'PARAM') {
-		    $v = $param->{$k};
-		    $param_used->{$k} = 1;
-		} elsif ($v eq 'DEST') {
-		    $v = $param->{dest};
-		    $param_used->{dest} = 1;
-		} elsif ($v eq 'SOURCE') {
-		    $v = $param->{source};
-		    $param_used->{source} = 1;
-		}
-
-		die "missing parameter '$k' in macro '$macro_name'\n" if !defined($v);
-		$rule->{$k} = $v;
-	    }
-	    foreach my $k (keys %$param) {
-		next if !defined($param->{$k});
-		next if $param_used->{$k};
-		if (defined($rule->{$k})) {
-		    die "parameter '$k' already define in macro (value = '$rule->{$k}')\n"
-			if $rule->{$k} ne $param->{$k};
-		} else {
-		    $rule->{$k} = $param->{$k};
-		}
-	    }
-	    push @$rules, $rule;
-	}
-    } else {
-	push @$rules, $param;
-    }
-
-    foreach my $rule (@$rules) {
-	$rule->{nbdport} = parse_port_name_number_or_range($rule->{dport})
-	    if defined($rule->{dport});
-	$rule->{nbsport} = parse_port_name_number_or_range($rule->{sport})
-	    if defined($rule->{sport});
-    }
-
-    return $rules;
 }
 
 sub parse_vmfw_option {
@@ -1183,15 +1593,18 @@ sub parse_vmfw_option {
 
     my $loglevels = "emerg|alert|crit|err|warning|notice|info|debug|nolog";
 
-    if ($line =~ m/^(enable|dhcp|macfilter|nosmurfs|tcpflags):\s*(0|1)\s*$/i) {
+    if ($line =~ m/^(enable|dhcp|macfilter|nosmurfs|tcpflags|ips):\s*(0|1)\s*$/i) {
 	$opt = lc($1);
 	$value = int($2);
     } elsif ($line =~ m/^(log_level_in|log_level_out):\s*(($loglevels)\s*)?$/i) {
 	$opt = lc($1);
 	$value = $2 ? lc($3) : '';
-    } elsif ($line =~ m/^(policy-(in|out)):\s*(ACCEPT|DROP|REJECT)\s*$/i) {
+    } elsif ($line =~ m/^(policy_(in|out)):\s*(ACCEPT|DROP|REJECT)\s*$/i) {
 	$opt = lc($1);
 	$value = uc($3);
+    } elsif ($line =~ m/^(ips_queues):\s*((\d+)(:(\d+))?)\s*$/i) {
+	$opt = lc($1);
+	$value = $2;
     } else {
 	chomp $line;
 	die "can't parse option '$line'\n"
@@ -1207,15 +1620,34 @@ sub parse_hostfw_option {
 
     my $loglevels = "emerg|alert|crit|err|warning|notice|info|debug|nolog";
 
-    if ($line =~ m/^(enable|dhcp|nosmurfs|tcpflags):\s*(0|1)\s*$/i) {
+    if ($line =~ m/^(enable|dhcp|nosmurfs|tcpflags|allow_bridge_route|optimize):\s*(0|1)\s*$/i) {
 	$opt = lc($1);
 	$value = int($2);
     } elsif ($line =~ m/^(log_level_in|log_level_out|tcp_flags_log_level|smurf_log_level):\s*(($loglevels)\s*)?$/i) {
 	$opt = lc($1);
 	$value = $2 ? lc($3) : '';
-    } elsif ($line =~ m/^(policy-(in|out)):\s*(ACCEPT|DROP|REJECT)\s*$/i) {
+    } elsif ($line =~ m/^(policy_(in|out)):\s*(ACCEPT|DROP|REJECT)\s*$/i) {
 	$opt = lc($1);
 	$value = uc($3);
+    } elsif ($line =~ m/^(nf_conntrack_max):\s*(\d+)\s*$/i) {
+	$opt = lc($1);
+	$value = int($2);
+    } else {
+	chomp $line;
+	die "can't parse option '$line'\n"
+    }
+
+    return ($opt, $value);
+}
+
+sub parse_clusterfw_option {
+    my ($line) = @_;
+
+    my ($opt, $value);
+
+    if ($line =~ m/^(enable):\s*(0|1)\s*$/i) {
+	$opt = lc($1);
+	$value = int($2);
     } else {
 	chomp $line;
 	die "can't parse option '$line'\n"
@@ -1227,11 +1659,15 @@ sub parse_hostfw_option {
 sub parse_vm_fw_rules {
     my ($filename, $fh) = @_;
 
-    my $res = { in => [], out => [], options => {}};
+    my $res = { rules => [], options => {}};
 
     my $section;
 
+    my $digest = Digest::SHA->new('sha1');
+
     while (defined(my $line = <$fh>)) {
+	$digest->add($line);
+
 	next if $line =~ m/^#/;
 	next if $line =~ m/^\s*$/;
 
@@ -1259,27 +1695,33 @@ sub parse_vm_fw_rules {
 	    next;
 	}
 
-	my $rules;
-	eval { $rules = parse_fw_rule($line, 1, 1); };
+	my $rule;
+	eval { $rule = parse_fw_rule($line, 1, 1); };
 	if (my $err = $@) {
 	    warn "$prefix: $err";
 	    next;
 	}
 
-	push @{$res->{$section}}, @$rules;
+	push @{$res->{$section}}, $rule;
     }
 
+    $res->{digest} = $digest->b64digest;
+
     return $res;
 }
 
 sub parse_host_fw_rules {
     my ($filename, $fh) = @_;
 
-    my $res = { in => [], out => [], options => {}};
+    my $res = { rules => [], options => {}};
 
     my $section;
 
+    my $digest = Digest::SHA->new('sha1');
+
     while (defined(my $line = <$fh>)) {
+	$digest->add($line);
+
 	next if $line =~ m/^#/;
 	next if $line =~ m/^\s*$/;
 
@@ -1300,7 +1742,6 @@ sub parse_host_fw_rules {
 
 	if ($section eq 'options') {
 	    eval {
-		print "PARSE:$line\n";
 		my ($opt, $value) = parse_hostfw_option($line);
 		$res->{options}->{$opt} = $value;
 	    };
@@ -1308,54 +1749,110 @@ sub parse_host_fw_rules {
 	    next;
 	}
 
-	my $rules;
-	eval { $rules = parse_fw_rule($line, 1, 1); };
+	my $rule;
+	eval { $rule = parse_fw_rule($line, 1, 1); };
 	if (my $err = $@) {
 	    warn "$prefix: $err";
 	    next;
 	}
-
-	push @{$res->{$section}}, @$rules;
+	
+	push @{$res->{$section}}, $rule;
     }
 
+    $res->{digest} = $digest->b64digest;
+
     return $res;
 }
 
-sub parse_group_fw_rules {
+sub parse_cluster_fw_rules {
     my ($filename, $fh) = @_;
 
     my $section;
     my $group;
 
-    my $res = { in => [], out => [] };
+    my $res = { rules => {}, options => {}, groups => {}, ipset => {} };
+
+    my $digest = Digest::SHA->new('sha1');
 
     while (defined(my $line = <$fh>)) {
+	$digest->add($line);
+
 	next if $line =~ m/^#/;
 	next if $line =~ m/^\s*$/;
 
 	my $linenr = $fh->input_line_number();
 	my $prefix = "$filename (line $linenr)";
 
-	if ($line =~ m/^\[(in|out):(\S+)\]\s*$/i) {
-	    $section = lc($1);
-	    $group = lc($2);
+	if ($line =~ m/^\[options\]$/i) {
+	    $section = 'options';
 	    next;
 	}
-	if (!$section || !$group) {
-	    warn "$prefix: skip line - no section";
+
+	if ($line =~ m/^\[group\s+(\S+)\]\s*$/i) {
+	    $section = 'groups';
+	    $group = lc($1);
 	    next;
 	}
 
-	my $rules;
-	eval { $rules = parse_fw_rule($line, 0, 0); };
-	if (my $err = $@) {
-	    warn "$prefix: $err";
+	if ($line =~ m/^\[rules\]$/i) {
+	    $section = 'rules';
+	    next;
+	}
+    
+	if ($line =~ m/^\[ipgroup\s+(\S+)\]\s*$/i) {
+	    $section = 'ipset';
+	    $group = lc($1);
+	    $res->{$section}->{$group}->{type} = 'hash:ip family inet hashsize 1024 maxelem 65536 ';
+	    next;
+	}
+
+	if ($line =~ m/^\[netgroup\s+(\S+)\]\s*$/i) {
+	    $section = 'ipset';
+	    $group = lc($1);
+	    $res->{$section}->{$group}->{type} = 'hash:net family inet hashsize 1024 maxelem 65536 ';
+	    next;
+	}
+
+	if (!$section) {
+	    warn "$prefix: skip line - no section";
 	    next;
 	}
 
-	push @{$res->{$group}->{$section}}, @$rules;
+	if ($section eq 'options') {
+	    eval {
+		my ($opt, $value) = parse_clusterfw_option($line);
+		$res->{options}->{$opt} = $value;
+	    };
+	    warn "$prefix: $@" if $@;
+	} elsif ($section eq 'rules') {
+	    my $rule;
+	    eval { $rule = parse_fw_rule($line, 1, 1); };
+	    if (my $err = $@) {
+		warn "$prefix: $err";
+		next;
+	    }
+	    push @{$res->{$section}}, $rule;
+	} elsif ($section eq 'groups') {
+	    my $rule;
+	    eval { $rule = parse_fw_rule($line, 0, 0); };
+	    if (my $err = $@) {
+		warn "$prefix: $err";
+		next;
+	    }
+	    push @{$res->{$section}->{$group}}, $rule;
+	} elsif ($section eq 'ipset') {
+	    chomp $line;
+	    my $ip;
+	    if (!Net::IP->new($line)) {
+		warn "$prefix: $line is not an valid ip address";
+		next;
+	    }
+	    push @{$res->{$section}->{$group}->{ip}}, $line;
+	}
     }
 
+    $res->{digest} = $digest->b64digest;
+
     return $res;
 }
 
@@ -1374,35 +1871,63 @@ sub run_locked {
 sub read_local_vm_config {
 
     my $openvz = {};
-
     my $qemu = {};
 
-    my $list = PVE::QemuServer::config_list();
+    my $vmdata = { openvz => $openvz, qemu => $qemu };
 
-    foreach my $vmid (keys %$list) {
-	my $cfspath = PVE::QemuServer::cfs_config_path($vmid);
-	if (my $conf = PVE::Cluster::cfs_read_file($cfspath)) {
-	    $qemu->{$vmid} = $conf;
+    my $vmlist = PVE::Cluster::get_vmlist();
+    return $vmdata if !$vmlist || !$vmlist->{ids};
+    my $ids = $vmlist->{ids};
+
+    foreach my $vmid (keys %$ids) {
+	next if !$vmid; # skip VE0
+	my $d = $ids->{$vmid};
+	next if !$d->{node} || $d->{node} ne $nodename;
+	next if !$d->{type};
+	if ($d->{type} eq 'openvz') {
+	    if ($have_pve_manager) {
+		my $cfspath = PVE::OpenVZ::cfs_config_path($vmid);
+		if (my $conf = PVE::Cluster::cfs_read_file($cfspath)) {
+		    $openvz->{$vmid} = $conf;
+		}
+	    }
+	} elsif ($d->{type} eq 'qemu') {
+	    if ($have_qemu_server) {
+		my $cfspath = PVE::QemuServer::cfs_config_path($vmid);
+		if (my $conf = PVE::Cluster::cfs_read_file($cfspath)) {
+		    $qemu->{$vmid} = $conf;
+		}
+	    }
 	}
     }
 
-    my $vmdata = { openvz => $openvz, qemu => $qemu };
-
     return $vmdata;
 };
 
-sub read_vm_firewall_rules {
+sub load_vmfw_conf {
+    my ($vmid) = @_;
+
+    my $vmfw_conf = {};
+
+    my $filename = "/etc/pve/firewall/$vmid.fw";
+    if (my $fh = IO::File->new($filename, O_RDONLY)) {
+	$vmfw_conf = parse_vm_fw_rules($filename, $fh);
+    }
+
+    return $vmfw_conf;
+}
+
+sub read_vm_firewall_configs {
     my ($vmdata) = @_;
-    my $rules = {};
-    foreach my $vmid (keys %{$vmdata->{qemu}}, keys %{$vmdata->{openvz}}) {
-	my $filename = "/etc/pve/firewall/$vmid.fw";
-	my $fh = IO::File->new($filename, O_RDONLY);
-	next if !$fh;
+    my $vmfw_configs = {};
 
-	$rules->{$vmid} = parse_vm_fw_rules($filename, $fh);
+    foreach my $vmid (keys %{$vmdata->{qemu}}, keys %{$vmdata->{openvz}}) {
+	my $vmfw_conf = load_vmfw_conf($vmid);
+	next if !$vmfw_conf->{options}; # skip if file does not exists
+	$vmfw_configs->{$vmid} = $vmfw_conf;
     }
 
-    return $rules;
+    return $vmfw_configs;
 }
 
 sub get_option_log_level {
@@ -1428,25 +1953,17 @@ sub generate_std_chains {
     my $loglevel = get_option_log_level($options, 'smurf_log_level');
 
     # same as shorewall smurflog.
-    if (defined($loglevel)) {
-	$pve_std_chains-> {'PVEFW-smurflog'} = [
-	    "-j LOG --log-prefix \"smurfs-dropped\" --log-level $loglevel",
-	    "-j DROP",
-	    ];
-    } else {
-	$pve_std_chains-> {'PVEFW-smurflog'} = [ "-j DROP" ];
-    }
+    my $chain = 'PVEFW-smurflog';
+
+    push @{$pve_std_chains->{$chain}}, get_log_rule_base($chain, 0, "DROP: ", $loglevel) if $loglevel;
+    push @{$pve_std_chains->{$chain}}, "-j DROP";
 
     # same as shorewall logflags action.
     $loglevel = get_option_log_level($options, 'tcp_flags_log_level');
-    if (defined($loglevel)) {
-	$pve_std_chains-> {'PVEFW-logflags'} = [
-	    "-j LOG --log-prefix \"logflags-dropped:\" --log-level $loglevel --log-ip-options",
-	    "-j DROP",
-	    ];
-    } else {
-	$pve_std_chains-> {'PVEFW-logflags'} = [ "-j DROP" ];
-    }
+    $chain = 'PVEFW-logflags';
+    # fixme: is this correctly logged by pvewf-logger? (ther is no --log-ip-options for NFLOG)
+    push @{$pve_std_chains->{$chain}}, get_log_rule_base($chain, 0, "DROP: ", $loglevel) if $loglevel;
+    push @{$pve_std_chains->{$chain}}, "-j DROP";
 
     foreach my $chain (keys %$pve_std_chains) {
 	ruleset_create_chain($ruleset, $chain);
@@ -1460,6 +1977,24 @@ sub generate_std_chains {
     }
 }
 
+sub generate_ipset_chains {
+    my ($ipset_ruleset, $options) = @_;
+
+    foreach my $ipset (keys %{$options->{ipset}}) {
+	generate_ipset($ipset_ruleset, $ipset, $options->{ipset}->{$ipset});
+    }
+}
+
+sub generate_ipset {
+    my ($ipset_ruleset, $name, $options) = @_;
+
+    push @{$ipset_ruleset->{$name}}, "create $name $options->{type}";
+
+    foreach my $ip (@{$options->{ip}}) {
+	push @{$ipset_ruleset->{$name}}, "add $name $ip";
+    }
+}
+
 sub save_pvefw_status {
     my ($status) = @_;
 
@@ -1484,43 +2019,145 @@ sub read_pvefw_status {
     return $status;
 }
 
-sub compile {
-    my $vmdata = read_local_vm_config();
-    my $rules = read_vm_firewall_rules($vmdata);
+# fixme: move to pve-common PVE::ProcFSTools
+sub read_proc_net_route {
+    my $filename = "/proc/net/route";
 
-    my $group_rules = {};
-    my $filename = "/etc/pve/firewall/groups.fw";
+    my $res = {};
+
+    my $fh = IO::File->new ($filename, "r");
+    return $res if !$fh;
+
+    my $int_to_quad = sub {
+	return join '.' => map { ($_[0] >> 8*(3-$_)) % 256 } (3, 2, 1, 0);
+    };
+
+    while (defined(my $line = <$fh>)) {
+	next if $line =~/^Iface\s+Destination/; # skip head
+	my ($iface, $dest, $gateway, $metric, $mask, $mtu) = (split(/\s+/, $line))[0,1,2,6,7,8];
+	push @{$res->{$iface}}, {
+	    dest => &$int_to_quad(hex($dest)),
+	    gateway => &$int_to_quad(hex($gateway)),
+	    mask => &$int_to_quad(hex($mask)),
+	    metric => $metric,
+	    mtu => $mtu,
+	};
+    }
+
+    return $res;
+}
+
+sub load_clusterfw_conf {
+
+    my $cluster_conf = {};
+     if (my $fh = IO::File->new($clusterfw_conf_filename, O_RDONLY)) {
+	$cluster_conf = parse_cluster_fw_rules($clusterfw_conf_filename, $fh);
+    }
+
+    return $cluster_conf;
+}
+
+my $rules_to_conf = sub {
+    my ($rules, $need_iface) = @_;
+
+    my $raw = '';
+
+    foreach my $rule (@$rules) {
+	if ($rule->{type} eq  'in' || $rule->{type} eq 'out') {
+	    $raw .= '|' if defined($rule->{enable}) && !$rule->{enable};
+	    $raw .= uc($rule->{type});
+	    $raw .= " " . $rule->{action};
+	    $raw .= " " . ($rule->{iface} || '-') if $need_iface;
+	    $raw .= " " . ($rule->{source} || '-');
+	    $raw .= " " . ($rule->{dest} || '-');
+	    $raw .= " " . ($rule->{proto} || '-');
+	    $raw .= " " . ($rule->{dport} || '-');
+	    $raw .= " " . ($rule->{sport} || '-');
+	    $raw .= " # " . encode('utf8', $rule->{comment}) 
+		if $rule->{comment} && $rule->{comment} !~ m/^\s*$/;
+	    $raw .= "\n";
+	} else {
+	    die "implement me '$rule->{type}'";
+	}
+    }
+
+    return $raw;
+};
+
+sub save_clusterfw_conf {
+    my ($cluster_conf) = @_;
+
+    my $raw = '';
+
+    my $options = $cluster_conf->{options};
+    if (scalar(keys %$options)) {
+	$raw .= "[OPTIONS]\n\n";
+	foreach my $opt (keys %$options) {
+	    $raw .= "$opt: $options->{$opt}\n";
+	}
+	$raw .= "\n";
+    }
+
+    # fixme: save ipset
+
+    my $rules = $cluster_conf->{rules};
+    if (scalar(@$rules)) {
+	$raw .= "[RULES]\n\n";
+	$raw .= &$rules_to_conf($rules, 1);
+	$raw .= "\n";
+    }
+
+    foreach my $group (sort keys %{$cluster_conf->{groups}}) {
+	my $rules = $cluster_conf->{groups}->{$group};
+	$raw .= "[group $group]\n\n";
+	$raw .= &$rules_to_conf($rules, 0);
+	$raw .= "\n";
+    }
+
+    PVE::Tools::file_set_contents($clusterfw_conf_filename, $raw);
+}
+
+sub load_hostfw_conf {
+
+    my $hostfw_conf = {};
+    my $filename = "/etc/pve/local/host.fw";
     if (my $fh = IO::File->new($filename, O_RDONLY)) {
-	$group_rules = parse_group_fw_rules($filename, $fh);
+	$hostfw_conf = parse_host_fw_rules($filename, $fh);
     }
+    return $hostfw_conf;
+}
+
+sub compile {
+    my $vmdata = read_local_vm_config();
+    my $vmfw_configs = read_vm_firewall_configs($vmdata);
 
-    #print Dumper($rules);
+    my $routing_table = read_proc_net_route();
+
+    my $cluster_conf = load_clusterfw_conf();
+
+    my $ipset_ruleset = {};
+    generate_ipset_chains($ipset_ruleset, $cluster_conf);
 
     my $ruleset = {};
 
     ruleset_create_chain($ruleset, "PVEFW-INPUT");
     ruleset_create_chain($ruleset, "PVEFW-OUTPUT");
-    ruleset_create_chain($ruleset, "PVEFW-FORWARD");
 
-    my $host_options = {};
-    my $host_rules;
+    ruleset_create_chain($ruleset, "PVEFW-FORWARD");
 
-    $filename = "/etc/pve/local/host.fw";
-    if (my $fh = IO::File->new($filename, O_RDONLY)) {
-	$host_rules = parse_host_fw_rules($filename, $fh);
-	$host_options = $host_rules->{options};
-    }
+    my $hostfw_conf = load_hostfw_conf();
+    my $hostfw_options = $hostfw_conf->{options} || {};
 
-    generate_std_chains($ruleset, $host_options);
+    generate_std_chains($ruleset, $hostfw_options);
 
-    my $hotsfw_enable = $host_rules && !(defined($host_options->{enable}) && ($host_options->{enable} == 0));
+    my $hostfw_enable = !(defined($hostfw_options->{enable}) && ($hostfw_options->{enable} == 0));
 
-    enablehostfw($ruleset, $host_rules, $group_rules) if $hotsfw_enable;
+    enable_host_firewall($ruleset, $hostfw_conf, $cluster_conf) if $hostfw_enable;
 
     # generate firewall rules for QEMU VMs
     foreach my $vmid (keys %{$vmdata->{qemu}}) {
 	my $conf = $vmdata->{qemu}->{$vmid};
-	my $vmfw_conf = $rules->{$vmid};
+	my $vmfw_conf = $vmfw_configs->{$vmid};
 	next if !$vmfw_conf;
 	next if defined($vmfw_conf->{options}->{enable}) && ($vmfw_conf->{options}->{enable} == 0);
 
@@ -1535,36 +2172,92 @@ sub compile {
 
 	    $bridge .= "v$net->{tag}" if $net->{tag};
 
-
-	    generate_bridge_chains($ruleset, $bridge);
+	    generate_bridge_chains($ruleset, $hostfw_conf, $bridge, $routing_table);
 
 	    my $macaddr = $net->{macaddr};
-	    generate_tap_rules_direction($ruleset, $group_rules, $iface, $netid, $macaddr, $vmfw_conf, $bridge, 'IN');
-	    generate_tap_rules_direction($ruleset, $group_rules, $iface, $netid, $macaddr, $vmfw_conf, $bridge, 'OUT');
+	    generate_tap_rules_direction($ruleset, $cluster_conf, $iface, $netid, $macaddr, 
+					 $vmfw_conf, $vmid, $bridge, 'IN');
+	    generate_tap_rules_direction($ruleset, $cluster_conf, $iface, $netid, $macaddr, 
+					 $vmfw_conf, $vmid, $bridge, 'OUT');
 	}
     }
 
-    if ($hotsfw_enable) {
-	# allow traffic from lo (ourself)
-	ruleset_addrule($ruleset, "PVEFW-INPUT", "-i lo -j ACCEPT");
+    # generate firewall rules for OpenVZ containers
+    foreach my $vmid (keys %{$vmdata->{openvz}}) {
+	my $conf = $vmdata->{openvz}->{$vmid};
+
+	my $vmfw_conf = $vmfw_configs->{$vmid};
+	next if !$vmfw_conf;
+	next if defined($vmfw_conf->{options}->{enable}) && ($vmfw_conf->{options}->{enable} == 0);
+
+	if ($conf->{ip_address} && $conf->{ip_address}->{value}) {
+	    my $ip = $conf->{ip_address}->{value};
+	    generate_venet_rules_direction($ruleset, $cluster_conf, $vmfw_conf, $vmid, $ip, 'IN');
+	    generate_venet_rules_direction($ruleset, $cluster_conf, $vmfw_conf, $vmid, $ip, 'OUT');
+	}
+
+	if ($conf->{netif} && $conf->{netif}->{value}) {
+	    my $netif = PVE::OpenVZ::parse_netif($conf->{netif}->{value});
+	    foreach my $netid (keys %$netif) {
+		my $d = $netif->{$netid};
+		my $bridge = $d->{bridge};
+		if (!$bridge) {
+		    warn "no bridge device for CT $vmid iface '$netid'\n";
+		    next; # fixme?
+		}
+		
+		generate_bridge_chains($ruleset, $hostfw_conf, $bridge, $routing_table);
+
+		my $macaddr = $d->{mac};
+		my $iface = $d->{host_ifname};
+		generate_tap_rules_direction($ruleset, $cluster_conf, $iface, $netid, $macaddr, 
+					     $vmfw_conf, $vmid, $bridge, 'IN');
+		generate_tap_rules_direction($ruleset, $cluster_conf, $iface, $netid, $macaddr, 
+					     $vmfw_conf, $vmid, $bridge, 'OUT');
+	    }
+	}
     }
 
-    return $ruleset;
+    if($hostfw_options->{optimize}){
+
+	my $accept = ruleset_chain_exist($ruleset, "PVEFW-IPS") ? "PVEFW-IPS" : "ACCEPT";
+	ruleset_insertrule($ruleset, "PVEFW-FORWARD", "-m conntrack --ctstate RELATED,ESTABLISHED -j $accept");
+	ruleset_insertrule($ruleset, "PVEFW-FORWARD", "-m conntrack --ctstate INVALID -j DROP");
+    }
+
+    # fixme: what log level should we use here?
+    my $loglevel = get_option_log_level($hostfw_options, "log_level_out");
+
+    # fixme: should we really block inter-bridge traffic?
+
+    # always allow traffic from containers?
+    ruleset_addrule($ruleset, "PVEFW-FORWARD", "-i venet0 -j RETURN");
+
+    # disable interbridge routing
+    ruleset_addrule($ruleset, "PVEFW-FORWARD", "-o vmbr+ -j PVEFW-Drop"); 
+    ruleset_addrule($ruleset, "PVEFW-FORWARD", "-i vmbr+ -j PVEFW-Drop");
+    ruleset_addlog($ruleset, "PVEFW-FORWARD", 0, "DROP: ", $loglevel, "-o vmbr+");  
+    ruleset_addlog($ruleset, "PVEFW-FORWARD", 0, "DROP: ", $loglevel, "-i vmbr+");  
+    ruleset_addrule($ruleset, "PVEFW-FORWARD", "-o vmbr+ -j DROP");  
+    ruleset_addrule($ruleset, "PVEFW-FORWARD", "-i vmbr+ -j DROP");
+
+    return wantarray ? ($ruleset, $hostfw_conf, $ipset_ruleset) : $ruleset;
 }
 
 sub get_ruleset_status {
-    my ($ruleset, $verbose) = @_;
+    my ($ruleset, $verbose, $ipset) = @_;
 
-    my $active_chains = iptables_get_chains();
+    my $active_chains = undef;
+    if($ipset){
+	$active_chains = ipset_get_chains();
+    }else{
+	$active_chains = iptables_get_chains();
+    }
 
     my $statushash = {};
 
     foreach my $chain (sort keys %$ruleset) {
-	my $digest = Digest::SHA->new('sha1');
-	foreach my $cmd (@{$ruleset->{$chain}}) {
-	     $digest->add("$cmd\n");
-	}
-	my $sig = $digest->b64digest;
+	my $sig = ipset_chain_digest($ruleset->{$chain});
 	$statushash->{$chain}->{sig} = $sig;
 
 	my $oldsig = $active_chains->{$chain};
@@ -1591,16 +2284,10 @@ sub get_ruleset_status {
 	    print "delete $chain ($sig)\n" if $verbose;
 	}
     }
-
+    
     return $statushash;
 }
 
-sub print_ruleset {
-    my ($ruleset) = @_;
-
-    get_ruleset_status($ruleset, 1);
-}
-
 sub print_sig_rule {
     my ($chain, $sig) = @_;
 
@@ -1669,20 +2356,74 @@ sub get_rulset_cmdlist {
 	$cmdlist .= "-X $chain\n";
     }
 
+    my $changes = $cmdlist ne "*filter\n" ? 1 : 0;
+ 
     $cmdlist .= "COMMIT\n";
 
-    return $cmdlist;
+    return wantarray ? ($cmdlist, $changes) : $cmdlist;
 }
 
-sub apply_ruleset {
+sub get_ipset_cmdlist {
     my ($ruleset, $verbose) = @_;
 
+    my $cmdlist = "";
+
+    my $statushash = get_ruleset_status($ruleset, $verbose, 1);
+
+    foreach my $chain (sort keys %$ruleset) {
+	my $stat = $statushash->{$chain};
+	die "internal error" if !$stat;
+
+	if ($stat->{action} eq 'create') {
+	    foreach my $cmd (@{$ruleset->{$chain}}) {
+		$cmdlist .= "$cmd\n";
+	    }
+        }
+
+	if ($stat->{action} eq 'update') {
+	    my $chain_swap = $chain."_swap";
+
+	    foreach my $cmd (@{$ruleset->{$chain}}) {
+		$cmd =~ s/$chain/$chain_swap/;
+		$cmdlist .= "$cmd\n";
+
+	    }
+	    $cmdlist .= "swap $chain_swap $chain\n";
+	    $cmdlist .= "flush $chain_swap\n";
+	    $cmdlist .= "destroy $chain_swap\n";
+        }
+
+    }
+
+    foreach my $chain (keys %$statushash) {
+	next if $statushash->{$chain}->{action} ne 'delete';
+
+	$cmdlist .= "flush $chain\n";
+	$cmdlist .= "destroy $chain\n";
+    }
+
+    my $changes = $cmdlist ? 1 : 0;
+
+    return wantarray ? ($cmdlist, $changes) : $cmdlist;
+}
+
+sub apply_ruleset {
+    my ($ruleset, $hostfw_conf, $ipset_ruleset, $verbose) = @_;
+
     enable_bridge_firewall();
 
+    update_nf_conntrack_max($hostfw_conf);
+
+    my $ipsetcmdlist = get_ipset_cmdlist($ipset_ruleset, $verbose);
+
     my $cmdlist = get_rulset_cmdlist($ruleset, $verbose);
 
+    print $ipsetcmdlist if $verbose;
+
     print $cmdlist if $verbose;
 
+    ipset_restore_cmdlist($ipsetcmdlist);
+
     iptables_restore_cmdlist($cmdlist);
 
     # test: re-read status and check if everything is up to date
@@ -1700,19 +2441,43 @@ sub apply_ruleset {
     die "unable to apply firewall changes\n" if $errors;
 }
 
+sub update_nf_conntrack_max {
+    my ($hostfw_conf) = @_;
+
+    my $max = 65536; # reasonable default
+
+    my $options = $hostfw_conf->{options} || {};
+
+    if (defined($options->{nf_conntrack_max}) && ($options->{nf_conntrack_max} > $max)) {
+	$max = $options->{nf_conntrack_max};
+	$max = int(($max+ 8191)/8192)*8192; # round to multiples of 8192
+    }
+
+    my $filename_nf_conntrack_max = "/proc/sys/net/nf_conntrack_max";
+    my $filename_hashsize = "/sys/module/nf_conntrack/parameters/hashsize";
+
+    my $current = int(PVE::Tools::file_read_firstline($filename_nf_conntrack_max) || $max);
+
+    if ($current != $max) {
+	my $hashsize = int($max/4);
+	PVE::ProcFSTools::write_proc_entry($filename_hashsize, $hashsize);
+	PVE::ProcFSTools::write_proc_entry($filename_nf_conntrack_max, $max);
+    }
+}
+
 sub update {
     my ($start, $verbose) = @_;
 
     my $code = sub {
 	my $status = read_pvefw_status();
 
-	my $ruleset = PVE::Firewall::compile();
+	my ($ruleset, $hostfw_conf, $ipset_ruleset) = compile();
 
 	if ($start || $status eq 'active') {
 
 	    save_pvefw_status('active')	if ($status ne 'active');
 
-	    PVE::Firewall::apply_ruleset($ruleset, $verbose);
+	    apply_ruleset($ruleset, $hostfw_conf, $ipset_ruleset, $verbose);
 	} else {
 	    print "Firewall not active (status = $status)\n" if $verbose;
 	}