X-Git-Url: https://git.proxmox.com/?p=pve-firewall.git;a=blobdiff_plain;f=src%2FPVE%2FFirewall.pm;h=48e630004bf424c2db578aad3e60eb61db42f8e6;hp=633aa7ac1c79889dbaa92e4049c4134d2e8257ab;hb=cc37e0005d0617e405883fd12b80faf65cedb153;hpb=0e0432e2ebb669fe8c58939940c58028f8fbcc31

diff --git a/src/PVE/Firewall.pm b/src/PVE/Firewall.pm
index 633aa7a..48e6300 100644
--- a/src/PVE/Firewall.pm
+++ b/src/PVE/Firewall.pm
@@ -132,6 +132,8 @@ my $pve_fw_lock_filename = "/var/lock/pvefw.lck";
 
 my $default_log_level = 'nolog'; # avoid logs by default
 
+my $global_log_ratelimit = '--limit 1/sec';
+
 my $log_level_hash = {
     debug => 7,
     info => 6,
@@ -423,7 +425,7 @@ my $pve_fw_macros = {
     ],
     'Razor' => [
 	"Razor Antispam System",
-	{ action => 'ACCEPT', proto => 'tcp', dport => '2703' },
+	{ action => 'PARAM', proto => 'tcp', dport => '2703' },
     ],
     'Rdate' => [
 	"Remote time retrieval (rdate)",
@@ -860,7 +862,7 @@ sub get_etc_services {
 	next if $line =~m/^#/;
 	next if ($line =~m/^\s*$/);
 
-	if ($line =~ m!^(\S+)\s+(\S+)/(tcp|udp).*$!) {
+	if ($line =~ m!^(\S+)\s+(\S+)/(tcp|udp|sctp).*$!) {
 	    $services->{byid}->{$2}->{name} = $1;
 	    $services->{byid}->{$2}->{port} = $2;
 	    $services->{byid}->{$2}->{$3} = 1;
@@ -876,12 +878,8 @@ sub get_etc_services {
     return $etc_services;
 }
 
-my $etc_protocols;
-
-sub get_etc_protocols {
-    return $etc_protocols if $etc_protocols;
-
-    my $filename = "/etc/protocols";
+sub parse_protocol_file {
+    my ($filename) = @_;
 
     my $fh = IO::File->new($filename, O_RDONLY);
     if (!$fh) {
@@ -896,7 +894,7 @@ sub get_etc_protocols {
 	next if $line =~m/^#/;
 	next if ($line =~m/^\s*$/);
 
-	if ($line =~ m!^(\S+)\s+(\d+)\s+.*$!) {
+	if ($line =~ m!^(\S+)\s+(\d+)(?:\s+.*)?$!) {
 	    $protocols->{byid}->{$2}->{name} = $1;
 	    $protocols->{byname}->{$1} = $protocols->{byid}->{$2};
 	}
@@ -904,6 +902,16 @@ sub get_etc_protocols {
 
     close($fh);
 
+    return $protocols;
+}
+
+my $etc_protocols;
+
+sub get_etc_protocols {
+    return $etc_protocols if $etc_protocols;
+
+    my $protocols = parse_protocol_file('/etc/protocols');
+
     # add special case for ICMP v6
     $protocols->{byid}->{icmpv6}->{name} = "icmpv6";
     $protocols->{byname}->{icmpv6} = $protocols->{byid}->{icmpv6};
@@ -913,6 +921,14 @@ sub get_etc_protocols {
     return $etc_protocols;
 }
 
+my $etc_ethertypes;
+
+sub get_etc_ethertypes {
+    $etc_ethertypes = parse_protocol_file('/etc/ethertypes')
+	if !$etc_ethertypes;
+    return $etc_ethertypes;
+}
+
 my $__local_network;
 
 sub local_network {
@@ -1035,12 +1051,14 @@ sub parse_port_name_number_or_range {
     my @elements = split(/,/, $str);
     die "extraneous commas in list\n" if $str ne join(',', @elements);
     foreach my $item (@elements) {
-	$count++;
-	if ($item =~ m/^(\d+):(\d+)$/) {
+	if ($item =~ m/^([0-9]+):([0-9]+)$/) {
+	    $count += 2;
 	    my ($port1, $port2) = ($1, $2);
 	    die "invalid port '$port1'\n" if $port1 > 65535;
 	    die "invalid port '$port2'\n" if $port2 > 65535;
-	} elsif ($item =~ m/^(\d+)$/) {
+	    die "backwards range '$port1:$port2' not allowed, did you mean '$port2:$port1'?\n" if $port1 > $port2;
+	} elsif ($item =~ m/^([0-9]+)$/) {
+	    $count += 1;
 	    my $port = $1;
 	    die "invalid port '$port'\n" if $port > 65535;
 	} else {
@@ -1054,9 +1072,15 @@ sub parse_port_name_number_or_range {
 	}
     }
 
-    die "ICPM ports not allowed in port range\n" if $icmp_port && $count > 1;
+    die "ICPM ports not allowed in port range\n" if $icmp_port && $count > 0;
+
+    # I really don't like to use the word number here, but it's the only thing
+    # that makes sense in a literal way. The range 1:100 counts as 2, not as
+    # one and not as 100...
+    die "too many entries in port list (> 15 numbers)\n"
+	if $count > 15;
 
-    return $count;
+    return (scalar(@elements) > 1);
 }
 
 PVE::JSONSchema::register_format('pve-fw-sport-spec', \&pve_fw_verify_sport_spec);
@@ -1159,6 +1183,12 @@ our $cluster_option_properties = {
 	minimum => 0,
 	optional => 1,
     },
+    ebtables => {
+	description => "Enable ebtables rules cluster wide.",
+	type => 'boolean',
+	default => 1,
+	optional => 1,
+    },
     policy_in => {
 	description => "Input policy.",
 	type => 'string',
@@ -1171,6 +1201,33 @@ our $cluster_option_properties = {
 	optional => 1,
 	enum => ['ACCEPT', 'REJECT', 'DROP'],
     },
+    log_ratelimit => {
+	description => "Log ratelimiting settings",
+	type => 'string', format => {
+	    enable => {
+		default_key => 1,
+		description => 'Enable or disable log rate limiting',
+		type => 'boolean',
+		default => '1',
+	    },
+	    rate => {
+		type => 'string',
+		description => 'Frequency with which the burst bucket gets refilled',
+		optional => 1,
+		pattern => '[1-9][0-9]*\/(second|minute|hour|day)',
+		format_description => 'rate',
+		default => '1/second',
+	    },
+	    burst => {
+		type => 'integer',
+		minimum => 0,
+		optional => 1,
+		description => 'Inital burst of packages which will get logged',
+		default => 5,
+	    },
+	},
+	optional => 1,
+    },
 };
 
 our $host_option_properties = {
@@ -1214,6 +1271,18 @@ our $host_option_properties = {
 	type => 'boolean',
 	optional => 1,
     },
+    nf_conntrack_allow_invalid => {
+	description => "Allow invalid packets on connection tracking.",
+	type => 'boolean',
+	default => 0,
+	optional => 1,
+    },
+    log_nf_conntrack => {
+	description => "Enable logging of conntrack information.",
+	type => 'boolean',
+	default => 0,
+	optional => 1
+    },
 };
 
 our $vm_option_properties = {
@@ -1329,6 +1398,9 @@ my $rule_properties = {
 	minimum => 0,
 	optional => 1,
     },
+    log => get_standard_option('pve-fw-loglevel', {
+	description => "Log level for firewall rule.",
+    }),
     sport => {
 	description => "Restrict TCP/UDP source port. $port_descr",
 	type => 'string', format => 'pve-fw-sport-spec',
@@ -1690,6 +1762,12 @@ sub ipset_restore_cmdlist {
     run_command("/sbin/ipset restore", input => $cmdlist, errmsg => "ipset_restore_cmdlist");
 }
 
+sub ebtables_restore_cmdlist {
+    my ($cmdlist) = @_;
+
+    run_command("/sbin/ebtables-restore", input => $cmdlist, errmsg => "ebtables_restore_cmdlist");
+}
+
 sub iptables_get_chains {
     my ($iptablescmd) = @_;
 
@@ -1800,6 +1878,45 @@ sub ipset_get_chains {
     return $res;
 }
 
+sub ebtables_get_chains {
+
+    my $res = {};
+    my $chains = {};
+    my $parser = sub {
+	my $line = shift;
+	return if $line =~ m/^#/;
+	return if $line =~ m/^\s*$/;
+	if ($line =~ m/^:(\S+)\s\S+$/) {
+	    # Make sure we know chains exist even if they're empty.
+	    $chains->{$1} //= [];
+	} elsif ($line =~ m/^(?:\S+)\s(\S+)\s(?:\S+).*/) {
+	    my $chain = $1;
+	    $line =~ s/\s+$//;
+	    push @{$chains->{$chain}}, $line;
+	} else {
+	    # simply ignore the rest
+	    return;
+	}
+    };
+
+    run_command("/sbin/ebtables-save", outfunc => $parser);
+    # compute digest for each chain and store rules as well
+    foreach my $chain (keys %$chains) {
+	$res->{$chain}->{rules} = $chains->{$chain};
+	$res->{$chain}->{sig} = iptables_chain_digest($chains->{$chain});
+    }
+    return $res;
+}
+
+# substitude action of rule according to action hash
+sub rule_substitude_action {
+    my ($rule, $actions) = @_;
+
+    if (my $action = $rule->{action}) {
+	$rule->{action} = $actions->{$action} if defined($actions->{$action});
+    }
+}
+
 # generate a src or dst match
 # $dir(ection) is either d or s
 sub ipt_gen_src_or_dst_match {
@@ -1869,19 +1986,12 @@ sub ipt_rule_to_cmds {
 	if (my $proto = $rule->{proto}) {
 	    push @match, "-p $proto";
 
-	    my $nbdport = defined($rule->{dport}) ? parse_port_name_number_or_range($rule->{dport}, 1) : 0;
-	    my $nbsport = defined($rule->{sport}) ? parse_port_name_number_or_range($rule->{sport}, 0) : 0;
+	    my $multidport = defined($rule->{dport}) && parse_port_name_number_or_range($rule->{dport}, 1);
+	    my $multisport = defined($rule->{sport}) && parse_port_name_number_or_range($rule->{sport}, 0);
 
-	    my $multiport = 0;
-	    $multiport++ if $nbdport > 1;
-	    $multiport++ if $nbsport > 1;
+	    my $add_dport = sub {
+		return if !$rule->{dport};
 
-	    push @match, "--match multiport" if $multiport;
-
-	    die "multiport: option '--sports' cannot be used together with '--dports'\n"
-		if ($multiport == 2) && ($rule->{dport} ne $rule->{sport});
-
-	    if ($rule->{dport}) {
 		if ($proto eq 'icmp') {
 		    # Note: we use dport to store --icmp-type
 		    die "unknown icmp-type '$rule->{dport}'\n"
@@ -1894,28 +2004,29 @@ sub ipt_rule_to_cmds {
 		    push @match, "-m icmpv6 --icmpv6-type $rule->{dport}";
 		} elsif (!$PROTOCOLS_WITH_PORTS->{$proto}) {
 		    die "protocol $proto does not have ports\n";
+		} elsif ($multidport) {
+		    push @match, "--match multiport", "--dports $rule->{dport}";
 		} else {
-		    if ($nbdport > 1) {
-			if ($multiport == 2) {
-			    push @match,  "--ports $rule->{dport}";
-			} else {
-			    push @match, "--dports $rule->{dport}";
-			}
-		    } else {
-			push @match, "--dport $rule->{dport}";
-		    }
+		    push @match, "--dport $rule->{dport}";
 		}
-	    }
+	    };
+
+	    my $add_sport = sub {
+		return if !$rule->{sport};
 
-	    if ($rule->{sport}) {
 		die "protocol $proto does not have ports\n"
 		    if !$PROTOCOLS_WITH_PORTS->{$proto};
-		if ($nbsport > 1) {
-		    push @match, "--sports $rule->{sport}" if $multiport != 2;
+		if ($multisport) {
+		    push @match, "--match multiport", "--sports $rule->{sport}";
 		} else {
 		    push @match, "--sport $rule->{sport}";
 		}
-	    }
+	    };
+
+	    # order matters - single port before multiport!
+	    $add_dport->() if $multisport;
+	    $add_sport->();
+	    $add_dport->() if !$multisport;
 	} elsif ($rule->{dport} || $rule->{sport}) {
 	    die "destination port '$rule->{dport}', but no protocol specified\n" if $rule->{dport};
 	    die "source port '$rule->{sport}', but no protocol specified\n" if $rule->{sport};
@@ -1935,113 +2046,18 @@ sub ipt_rule_to_cmds {
     }
 
     my @iptcmds;
-    if (defined $rule->{log} && $rule->{log}) {
-	my $logaction = get_log_rule_base($chain, $vmid, $rule->{logmsg}, $rule->{log});
+    my $log = $rule->{log};
+    if (defined($log) && $log ne 'nolog') {
+	my $loglevel = $log_level_hash->{$log};
+	my $logaction = get_log_rule_base($chain, $vmid, $rule->{logmsg}, $loglevel);
 	push @iptcmds, "-A $chain $matchstr $logaction";
     }
     push @iptcmds, "-A $chain $matchstr $targetstr";
     return @iptcmds;
 }
 
-sub ruleset_generate_match {
-    my ($ruleset, $chain, $ipversion, $rule, $actions, $goto, $cluster_conf, $fw_conf) = @_;
-
-    return if defined($rule->{enable}) && !$rule->{enable};
-    return if $rule->{errors};
-
-    return $rule->{match} if defined $rule->{match};
-
-    die "unable to emit macro - internal error" if $rule->{macro}; # should not happen
-
-    my $nbdport = defined($rule->{dport}) ? parse_port_name_number_or_range($rule->{dport}, 1) : 0;
-    my $nbsport = defined($rule->{sport}) ? parse_port_name_number_or_range($rule->{sport}, 0) : 0;
-
-    my @cmd = ();
-
-    push @cmd, "-i $rule->{iface_in}" if $rule->{iface_in};
-    push @cmd, "-o $rule->{iface_out}" if $rule->{iface_out};
-
-    my $source = $rule->{source};
-    my $dest = $rule->{dest};
-
-    push @cmd, ipt_gen_src_or_dst_match($source, 's', $ipversion, $cluster_conf, $fw_conf) if $source;
-    push @cmd, ipt_gen_src_or_dst_match($dest, 'd', $ipversion, $cluster_conf, $fw_conf) if $dest;
-
-    if (my $proto = $rule->{proto}) {
-	push @cmd, "-p $proto";
-
-	my $multiport = 0;
-	$multiport++ if $nbdport > 1;
-	$multiport++ if $nbsport > 1;
-
-	push @cmd, "--match multiport" if $multiport;
-
-	die "multiport: option '--sports' cannot be used together with '--dports'\n"
-	    if ($multiport == 2) && ($rule->{dport} ne $rule->{sport});
-
-	if ($rule->{dport}) {
-	    if ($proto eq 'icmp') {
-		# Note: we use dport to store --icmp-type
-		die "unknown icmp-type '$rule->{dport}'\n"
-		    if $rule->{dport} !~ /^\d+$/ && !defined($icmp_type_names->{$rule->{dport}});
-		push @cmd, "-m icmp --icmp-type $rule->{dport}";
-	    } elsif ($proto eq 'icmpv6') {
-		# Note: we use dport to store --icmpv6-type
-		die "unknown icmpv6-type '$rule->{dport}'\n"
-		    if $rule->{dport} !~ /^\d+$/ && !defined($icmpv6_type_names->{$rule->{dport}});
-		push @cmd, "-m icmpv6 --icmpv6-type $rule->{dport}";
-	    } elsif (!$PROTOCOLS_WITH_PORTS->{$proto}) {
-		die "protocol $proto does not have ports\n";
-	    } else {
-		if ($nbdport > 1) {
-		    if ($multiport == 2) {
-			push @cmd,  "--ports $rule->{dport}";
-		    } else {
-			push @cmd, "--dports $rule->{dport}";
-		    }
-		} else {
-		    push @cmd, "--dport $rule->{dport}";
-		}
-	    }
-	}
-
-	if ($rule->{sport}) {
-	    die "protocol $proto does not have ports\n"
-		 if !$PROTOCOLS_WITH_PORTS->{$proto};
-	    if ($nbsport > 1) {
-		push @cmd, "--sports $rule->{sport}" if $multiport != 2;
-	    } else {
-		push @cmd, "--sport $rule->{sport}";
-	    }
-	}
-    } elsif ($rule->{dport} || $rule->{sport}) {
-	die "destination port '$rule->{dport}', but no protocol specified\n" if $rule->{dport};
-	die "source port '$rule->{sport}', but no protocol specified\n" if $rule->{sport};
-    }
-
-    push @cmd, "-m addrtype --dst-type $rule->{dsttype}" if $rule->{dsttype};
-
-    return scalar(@cmd) ? join(' ', @cmd) : undef;
-}
-
-sub ruleset_generate_action {
-    my ($ruleset, $chain, $ipversion, $rule, $actions, $goto, $cluster_conf, $fw_conf) = @_;
-
-    return $rule->{target} if defined $rule->{target};
-
-    my @cmd = ();
-
-    if (my $action = $rule->{action}) {
-	$action = $actions->{$action} if defined($actions->{$action});
-	$goto = 1 if !defined($goto) && $action eq 'PVEFW-SET-ACCEPT-MARK';
-	push @cmd, $goto ? "-g $action" : "-j $action";
-    }
-
-    return scalar(@cmd) ? join(' ', @cmd) : undef;
-}
-
 sub ruleset_generate_rule {
-    my ($ruleset, $chain, $ipversion, $rule, $actions, $goto, $cluster_conf, $fw_conf) = @_;
+    my ($ruleset, $chain, $ipversion, $rule, $cluster_conf, $fw_conf, $vmid) = @_;
 
     my $rules;
 
@@ -2054,59 +2070,13 @@ sub ruleset_generate_rule {
     # update all or nothing
     my @ipt_rule_cmds;
     foreach my $r (@$rules) {
-	push @ipt_rule_cmds, ipt_rule_to_cmds($r, $chain, $ipversion, $cluster_conf, $fw_conf);
+	push @ipt_rule_cmds, ipt_rule_to_cmds($r, $chain, $ipversion, $cluster_conf, $fw_conf, $vmid);
     }
     foreach my $c (@ipt_rule_cmds) {
 	ruleset_add_ipt_cmd($ruleset, $chain, $c);
     }
 }
 
-sub ruleset_generate_rule_old {
-    my ($ruleset, $chain, $ipversion, $rule, $actions, $goto, $cluster_conf, $fw_conf) = @_;
-
-    my $rules;
-
-    if ($rule->{macro}) {
-	$rules = &$apply_macro($rule->{macro}, $rule, 0, $ipversion);
-    } else {
-	$rules = [ $rule ];
-    }
-
-    # update all or nothing
-
-    # fixme: lots of temporary ugliness
-    my @mstrs = ();
-    my @astrs = ();
-    my @logging = ();
-    my @logmsg = ();
-    foreach my $tmp (@$rules) {
-	my $m = ruleset_generate_match($ruleset, $chain, $ipversion, $tmp, $actions, $goto, $cluster_conf, $fw_conf);
-	my $a = ruleset_generate_action($ruleset, $chain, $ipversion, $tmp, $actions, $goto, $cluster_conf, $fw_conf);
-	if (defined $m or defined $a) {
-	    push @mstrs, defined($m) ? $m : "";
-	    push @astrs, defined($a) ? $a : "";
-	    push @logging, $tmp->{log};
-	    push @logmsg, $tmp->{logmsg};
-	}
-    }
-
-    for my $i (0 .. $#mstrs) {
-	ruleset_addrule($ruleset, $chain, $mstrs[$i], $astrs[$i], $logging[$i], $logmsg[$i]);
-    }
-}
-
-sub ruleset_generate_rule_insert {
-    my ($ruleset, $chain, $ipversion, $rule, $actions, $goto) = @_;
-
-    die "implement me" if $rule->{macro}; # not implemented, because not needed so far
-
-    my $match = ruleset_generate_match($ruleset, $chain, $ipversion, $rule, $actions, $goto);
-    my $action = ruleset_generate_action($ruleset, $chain, $ipversion, $rule, $actions, $goto);
-    if (defined $match && defined $action) {
-	ruleset_insertrule($ruleset, $chain, $match, $action);
-    }
-}
-
 sub ruleset_create_chain {
     my ($ruleset, $chain) = @_;
 
@@ -2134,15 +2104,18 @@ sub ruleset_add_ipt_cmd {
 }
 
 sub ruleset_addrule {
-   my ($ruleset, $chain, $match, $action, $log, $logmsg, $vmid) = @_;
+    my ($ruleset, $chain, $match, $action, $log, $logmsg, $vmid) = @_;
 
-   die "no such chain '$chain'\n" if !$ruleset->{$chain};
+    die "no such chain '$chain'\n" if !$ruleset->{$chain};
 
-   if (defined($log) && $log) {
-	my $logaction = get_log_rule_base($chain, $vmid, $logmsg, $log);
+    if ($log) {
+	my $loglevel = $log_level_hash->{$log};
+	my $logaction = get_log_rule_base($chain, $vmid, $logmsg, $loglevel);
 	push @{$ruleset->{$chain}}, "-A $chain $match $logaction";
-   }
-   push @{$ruleset->{$chain}}, "-A $chain $match $action";
+    }
+    # for stable ebtables digests avoid double-spaces to match ebtables-save output
+    $match .= ' ' if length($match);
+    push @{$ruleset->{$chain}}, "-A $chain ${match}$action";
 }
 
 sub ruleset_insertrule {
@@ -2159,10 +2132,14 @@ sub get_log_rule_base {
     $vmid = 0 if !defined($vmid);
     $msg = "" if !defined($msg);
 
+    my $rlimit = '';
+    if (defined($global_log_ratelimit)) {
+	$rlimit = "-m limit $global_log_ratelimit ";
+    }
+
     # Note: we use special format for prefix to pass further
     # info to log daemon (VMID, LOGLEVEL and CHAIN)
-
-    return "-j NFLOG --nflog-prefix \":$vmid:$loglevel:$chain: $msg\"";
+    return "${rlimit}-j NFLOG --nflog-prefix \":$vmid:$loglevel:$chain: $msg\"";
 }
 
 sub ruleset_add_chain_policy {
@@ -2170,8 +2147,9 @@ sub ruleset_add_chain_policy {
 
     if ($policy eq 'ACCEPT') {
 
-	ruleset_generate_rule_old($ruleset, $chain, $ipversion, { action => 'ACCEPT' },
-			      { ACCEPT =>  $accept_action});
+	my $rule = { action => 'ACCEPT' };
+	rule_substitude_action($rule, { ACCEPT =>  $accept_action});
+	ruleset_generate_rule($ruleset, $chain, $ipversion, $rule);
 
     } elsif ($policy eq 'DROP') {
 
@@ -2181,7 +2159,7 @@ sub ruleset_add_chain_policy {
     } elsif ($policy eq 'REJECT') {
 	ruleset_addrule($ruleset, $chain, "", "-j PVEFW-Reject");
 
-	ruleset_addrule($ruleset, $chain, "", "-g PVEFW-reject", $loglevel, "policy $policy:", $vmid);
+	ruleset_addrule($ruleset, $chain, "", "-g PVEFW-reject", $loglevel, "policy $policy: ", $vmid);
     } else {
 	# should not happen
 	die "internal error: unknown policy '$policy'";
@@ -2201,9 +2179,11 @@ sub ruleset_chain_add_ndp {
 }
 
 sub ruleset_chain_add_conn_filters {
-    my ($ruleset, $chain, $accept) = @_;
+    my ($ruleset, $chain, $allow_invalid, $accept) = @_;
 
-    ruleset_addrule($ruleset, $chain, "-m conntrack --ctstate INVALID", "-j DROP");
+    if (!$allow_invalid) {
+	ruleset_addrule($ruleset, $chain, "-m conntrack --ctstate INVALID", "-j DROP");
+    }
     ruleset_addrule($ruleset, $chain, "-m conntrack --ctstate RELATED,ESTABLISHED", "-j $accept");
 }
 
@@ -2213,7 +2193,7 @@ sub ruleset_chain_add_input_filters {
     if ($cluster_conf->{ipset}->{blacklist}){
 	if (!ruleset_chain_exist($ruleset, "PVEFW-blacklist")) {
 	    ruleset_create_chain($ruleset, "PVEFW-blacklist");
-	    ruleset_addrule($ruleset, "PVEFW-blacklist", "", "-j DROP", $loglevel, "DROP: ");
+	    ruleset_addrule($ruleset, "PVEFW-blacklist", "", "-j DROP", $loglevel, "DROP: ", 0);
 	}
 	my $ipset_chain = compute_ipset_chain_name(0, 'blacklist', $ipversion);
 	ruleset_addrule($ruleset, $chain, "-m set --match-set ${ipset_chain} src", "-j PVEFW-blacklist");
@@ -2299,7 +2279,7 @@ sub ruleset_add_group_rule {
 }
 
 sub ruleset_generate_vm_rules {
-    my ($ruleset, $rules, $cluster_conf, $vmfw_conf, $chain, $netid, $direction, $options, $ipversion) = @_;
+    my ($ruleset, $rules, $cluster_conf, $vmfw_conf, $chain, $netid, $direction, $options, $ipversion, $vmid) = @_;
 
     my $lc_direction = lc($direction);
 
@@ -2316,14 +2296,13 @@ sub ruleset_generate_vm_rules {
 	} else {
 	    next if $rule->{type} ne $lc_direction;
 	    eval {
+		$rule->{logmsg} = "$rule->{action}: ";
 		if ($direction eq 'OUT') {
-		    ruleset_generate_rule_old($ruleset, $chain, $ipversion, $rule,
-					  { ACCEPT => "PVEFW-SET-ACCEPT-MARK", REJECT => "PVEFW-reject" },
-					  undef, $cluster_conf, $vmfw_conf);
+		    rule_substitude_action($rule, { ACCEPT => "PVEFW-SET-ACCEPT-MARK", REJECT => "PVEFW-reject" });
+		    ruleset_generate_rule($ruleset, $chain, $ipversion, $rule, $cluster_conf, $vmfw_conf, $vmid);
 		} else {
-		    ruleset_generate_rule_old($ruleset, $chain, $ipversion, $rule,
-					  { ACCEPT => $in_accept , REJECT => "PVEFW-reject" },
-					  undef, $cluster_conf, $vmfw_conf);
+		    rule_substitude_action($rule, { ACCEPT => $in_accept , REJECT => "PVEFW-reject" });
+		    ruleset_generate_rule($ruleset, $chain, $ipversion, $rule, $cluster_conf, $vmfw_conf, $vmid);
 		}
 	    };
 	    warn $@ if $@;
@@ -2384,7 +2363,7 @@ sub generate_tap_rules_direction {
     ruleset_create_vm_chain($ruleset, $tapchain, $ipversion, $options, $macaddr, $ipfilter_ipset, $direction);
 
     if ($options->{enable}) {
-	ruleset_generate_vm_rules($ruleset, $rules, $cluster_conf, $vmfw_conf, $tapchain, $netid, $direction, $options, $ipversion);
+	ruleset_generate_vm_rules($ruleset, $rules, $cluster_conf, $vmfw_conf, $tapchain, $netid, $direction, $options, $ipversion, $vmid);
 
 	ruleset_generate_vm_ipsrules($ruleset, $options, $direction, $iface);
 
@@ -2394,7 +2373,7 @@ sub generate_tap_rules_direction {
 	if ($direction eq 'OUT') {
 	    $policy = $options->{policy_out} || 'ACCEPT'; # allow everything by default
 	} else {
-	$policy = $options->{policy_in} || 'DROP'; # allow nothing by default
+	    $policy = $options->{policy_in} || 'DROP'; # allow nothing by default
 	}
 
 	my $accept = generate_nfqueue($options);
@@ -2408,10 +2387,10 @@ sub generate_tap_rules_direction {
     # plug the tap chain to bridge chain
     if ($direction eq 'IN') {
 	ruleset_addrule($ruleset, "PVEFW-FWBR-IN",
-			"-m physdev --physdev-is-bridged --physdev-out $iface", "-j $tapchain");
+			"-m physdev --physdev-is-bridged --physdev-out $iface", "-j $tapchain", $loglevel, 'FWBR-IN: ', $vmid);
     } else {
 	ruleset_addrule($ruleset, "PVEFW-FWBR-OUT",
-			"-m physdev --physdev-is-bridged --physdev-in $iface", "-j $tapchain");
+			"-m physdev --physdev-is-bridged --physdev-in $iface", "-j $tapchain", $loglevel, 'FWBR-OUT: ', $vmid);
     }
 }
 
@@ -2431,7 +2410,7 @@ sub enable_host_firewall {
 
     ruleset_addrule($ruleset, $chain, "-i lo", "-j ACCEPT");
 
-    ruleset_chain_add_conn_filters($ruleset, $chain, 'ACCEPT');
+    ruleset_chain_add_conn_filters($ruleset, $chain, 0, 'ACCEPT');
     ruleset_chain_add_ndp($ruleset, $chain, $ipversion, $options, 'IN', '-j RETURN');
     ruleset_chain_add_input_filters($ruleset, $chain, $ipversion, $options, $cluster_conf, $loglevel);
 
@@ -2451,9 +2430,8 @@ sub enable_host_firewall {
 	    if ($rule->{type} eq 'group') {
 		ruleset_add_group_rule($ruleset, $cluster_conf, $chain, $rule, 'IN', $accept_action, $ipversion);
 	    } elsif ($rule->{type} eq 'in') {
-		ruleset_generate_rule_old($ruleset, $chain, $ipversion, $rule,
-				      { ACCEPT => $accept_action, REJECT => "PVEFW-reject" },
-				      undef, $cluster_conf, $hostfw_conf);
+		rule_substitude_action($rule, { ACCEPT => $accept_action, REJECT => "PVEFW-reject" });
+		ruleset_generate_rule($ruleset, $chain, $ipversion, $rule, $cluster_conf, $hostfw_conf, 0);
 	    }
 	};
 	warn $@ if $@;
@@ -2490,7 +2468,7 @@ sub enable_host_firewall {
 
     ruleset_addrule($ruleset, $chain, "-o lo", "-j ACCEPT");
 
-    ruleset_chain_add_conn_filters($ruleset, $chain, 'ACCEPT');
+    ruleset_chain_add_conn_filters($ruleset, $chain, 0, 'ACCEPT');
 
     # we use RETURN because we may want to check other thigs later
     $accept_action = 'RETURN';
@@ -2505,12 +2483,12 @@ sub enable_host_firewall {
 
 	$rule->{iface_out} = $rule->{iface} if $rule->{iface};
 	eval {
+	    $rule->{logmsg} = "$rule->{action}: ";
 	    if ($rule->{type} eq 'group') {
 		ruleset_add_group_rule($ruleset, $cluster_conf, $chain, $rule, 'OUT', $accept_action, $ipversion);
 	    } elsif ($rule->{type} eq 'out') {
-		ruleset_generate_rule_old($ruleset, $chain, $ipversion,
-				      $rule, { ACCEPT => $accept_action, REJECT => "PVEFW-reject" },
-				      undef, $cluster_conf, $hostfw_conf);
+		rule_substitude_action($rule, { ACCEPT => $accept_action, REJECT => "PVEFW-reject" });
+		ruleset_generate_rule($ruleset, $chain, $ipversion, $rule, $cluster_conf, $hostfw_conf, 0);
 	    }
 	};
 	warn $@ if $@;
@@ -2554,10 +2532,10 @@ sub generate_group_rules {
 
     foreach my $rule (@$rules) {
 	next if $rule->{type} ne 'in';
+	next if !$rule->{enable} || $rule->{errors};
 	next if $rule->{ipversion} && $rule->{ipversion} ne $ipversion;
-	ruleset_generate_rule_old($ruleset, $chain, $ipversion, $rule,
-			      { ACCEPT => "PVEFW-SET-ACCEPT-MARK", REJECT => "PVEFW-reject" }, 
-			      undef, $cluster_conf);
+	rule_substitude_action($rule, { ACCEPT => "PVEFW-SET-ACCEPT-MARK", REJECT => "PVEFW-reject" });
+	ruleset_generate_rule($ruleset, $chain, $ipversion, $rule, $cluster_conf);
     }
 
     $chain = "GROUP-${group}-OUT";
@@ -2567,12 +2545,12 @@ sub generate_group_rules {
 
     foreach my $rule (@$rules) {
 	next if $rule->{type} ne 'out';
+	next if !$rule->{enable} || $rule->{errors};
 	next if $rule->{ipversion} && $rule->{ipversion} ne $ipversion;
 	# we use PVEFW-SET-ACCEPT-MARK (Instead of ACCEPT) because we need to
 	# check also other tap rules later
-	ruleset_generate_rule_old($ruleset, $chain, $ipversion, $rule,
-			      { ACCEPT => 'PVEFW-SET-ACCEPT-MARK', REJECT => "PVEFW-reject" }, 
-			      undef, $cluster_conf);
+	rule_substitude_action($rule, { ACCEPT => 'PVEFW-SET-ACCEPT-MARK', REJECT => "PVEFW-reject" });
+	ruleset_generate_rule($ruleset, $chain, $ipversion, $rule, $cluster_conf);
     }
 }
 
@@ -2649,6 +2627,10 @@ sub parse_fw_rule {
 	    $rule->{dest} = $1;
 	    next;
 	}
+	if ($line =~ s/^-log (emerg|alert|crit|err|warning|notice|info|debug|nolog)\s*//) {
+	    $rule->{log} = $1;
+	    next;
+	}
 
 	last;
     }
@@ -2670,6 +2652,14 @@ sub parse_fw_rule {
     return $rule;
 }
 
+sub verify_ethertype {
+    my ($value) = @_;
+    my $types = get_etc_ethertypes();
+    die "unknown ethernet protocol type: $value\n"
+	if !defined($types->{byname}->{$value}) &&
+	   !defined($types->{byid}->{$value});
+}
+
 sub parse_vmfw_option {
     my ($line) = @_;
 
@@ -2689,6 +2679,10 @@ sub parse_vmfw_option {
     } elsif ($line =~ m/^(ips_queues):\s*((\d+)(:(\d+))?)\s*$/i) {
 	$opt = lc($1);
 	$value = $2;
+    } elsif ($line =~ m/^(layer2_protocols):\s*(((\S+)[,]?)+)\s*$/i) {
+	$opt = lc($1);
+	$value = $2;
+	verify_ethertype($_) foreach split(/\s*,\s*/, $value);
     } else {
 	die "can't parse option '$line'\n"
     }
@@ -2703,7 +2697,7 @@ sub parse_hostfw_option {
 
     my $loglevels = "emerg|alert|crit|err|warning|notice|info|debug|nolog";
 
-    if ($line =~ m/^(enable|nosmurfs|tcpflags|ndp):\s*(0|1)\s*$/i) {
+    if ($line =~ m/^(enable|nosmurfs|tcpflags|ndp|log_nf_conntrack|nf_conntrack_allow_invalid):\s*(0|1)\s*$/i) {
 	$opt = lc($1);
 	$value = int($2);
     } elsif ($line =~ m/^(log_level_in|log_level_out|tcp_flags_log_level|smurf_log_level):\s*(($loglevels)\s*)?$/i) {
@@ -2730,9 +2724,15 @@ sub parse_clusterfw_option {
 	if (($value > 1) && ((time() - $value) > 60)) {
 	    $value = 0
 	}
+    } elsif ($line =~ m/^(ebtables):\s*(0|1)\s*$/i) {
+	$opt = lc($1);
+	$value = int($2);
     } elsif ($line =~ m/^(policy_(in|out)):\s*(ACCEPT|DROP|REJECT)\s*$/i) {
 	$opt = lc($1);
 	$value = uc($3);
+    } elsif ($line =~ m/^(log_ratelimit):\s*(\S+)\s*$/) {
+	$opt = lc($1);
+	$value = $2;
     } else {
 	die "can't parse option '$line'\n"
     }
@@ -3086,6 +3086,7 @@ my $format_rules = sub {
 		$raw .= " -p $rule->{proto}" if $rule->{proto};
 		$raw .= " -dport $rule->{dport}" if $rule->{dport};
 		$raw .= " -sport $rule->{sport}" if $rule->{sport};
+		$raw .= " -log $rule->{log}" if $rule->{log};
 	    }
 
 	    $raw .= " # " . encode('utf8', $rule->{comment})
@@ -3244,9 +3245,7 @@ sub get_option_log_level {
 
     return undef if $v eq '' || $v eq 'nolog';
 
-    $v = $log_level_hash->{$v} if defined($log_level_hash->{$v});
-
-    return $v if ($v >= 0) && ($v <= 7);
+    return $v if defined($log_level_hash->{$v});
 
     warn "unknown log level ($k = '$v')\n";
 
@@ -3279,7 +3278,7 @@ sub generate_std_chains {
 	ruleset_create_chain($ruleset, $chain);
 	foreach my $rule (@{$std_chains->{$chain}}) {
 	    if (ref($rule)) {
-		ruleset_generate_rule($ruleset, $chain, $ipversion, $rule);
+		ruleset_generate_rule($ruleset, $chain, $ipversion, $rule, 0);
 	    } else {
 		die "rule $rule as string - should not happen";
 	    }
@@ -3369,6 +3368,27 @@ sub round_powerof2 {
     return ++$int;
 }
 
+my $set_global_log_ratelimit = sub {
+    my $cluster_opts = shift;
+
+    $global_log_ratelimit = '--limit 1/sec';
+    if (defined(my $log_rlimit = $cluster_opts->{log_ratelimit})) {
+	my $ll_format = $cluster_option_properties->{log_ratelimit}->{format};
+	my $limit = PVE::JSONSchema::parse_property_string($ll_format, $log_rlimit);
+
+	if ($limit->{enable}) {
+	    if (my $rate = $limit->{rate}) {
+		$global_log_ratelimit = "--limit $rate";
+	    }
+	    if (my $burst = $limit->{burst}) {
+		$global_log_ratelimit .= " --limit-burst $burst";
+	    }
+	} else {
+	    $global_log_ratelimit = undef;
+	}
+    }
+};
+
 sub load_clusterfw_conf {
     my ($filename, $verbose) = @_;
 
@@ -3377,6 +3397,8 @@ sub load_clusterfw_conf {
     my $cluster_conf = {};
     if (my $fh = IO::File->new($filename, O_RDONLY)) {
 	$cluster_conf = parse_clusterfw_config($filename, $fh, $verbose);
+
+	$set_global_log_ratelimit->($cluster_conf->{options});
     }
 
     return $cluster_conf;
@@ -3485,7 +3507,7 @@ sub compile {
 	$vmfw_configs = read_vm_firewall_configs($cluster_conf, $vmdata, undef, $verbose);
     }
 
-    return ({},{},{}) if !$cluster_conf->{options}->{enable};
+    return ({},{},{},{}) if !$cluster_conf->{options}->{enable};
 
     my $localnet;
     if ($cluster_conf->{aliases}->{local_network}) {
@@ -3494,7 +3516,7 @@ sub compile {
 	my $localnet_ver;
 	($localnet, $localnet_ver) = parse_ip_or_cidr(local_network() || '127.0.0.0/8');
 
-	$cluster_conf->{aliases}->{local_network} = { 
+	$cluster_conf->{aliases}->{local_network} = {
 	    name => 'local_network', cidr => $localnet, ipversion => $localnet_ver };
     }
 
@@ -3502,9 +3524,10 @@ sub compile {
 
     my $ruleset = compile_iptables_filter($cluster_conf, $hostfw_conf, $vmfw_configs, $vmdata, 4, $verbose);
     my $rulesetv6 = compile_iptables_filter($cluster_conf, $hostfw_conf, $vmfw_configs, $vmdata, 6, $verbose);
+    my $ebtables_ruleset = compile_ebtables_filter($cluster_conf, $hostfw_conf, $vmfw_configs, $vmdata, $verbose);
     my $ipset_ruleset = compile_ipsets($cluster_conf, $vmfw_configs, $vmdata);
 
-    return ($ruleset, $ipset_ruleset, $rulesetv6);
+    return ($ruleset, $ipset_ruleset, $rulesetv6, $ebtables_ruleset);
 }
 
 sub compile_iptables_filter {
@@ -3522,7 +3545,8 @@ sub compile_iptables_filter {
     # fixme: what log level should we use here?
     my $loglevel = get_option_log_level($hostfw_options, "log_level_out");
 
-    ruleset_chain_add_conn_filters($ruleset, "PVEFW-FORWARD", "ACCEPT");
+    my $conn_allow_invalid = $hostfw_options->{nf_conntrack_allow_invalid} // 0;
+    ruleset_chain_add_conn_filters($ruleset, "PVEFW-FORWARD", $conn_allow_invalid, "ACCEPT");
 
     ruleset_create_chain($ruleset, "PVEFW-FWBR-IN");
     ruleset_chain_add_input_filters($ruleset, "PVEFW-FWBR-IN", $ipversion, $hostfw_options, $cluster_conf, $loglevel);
@@ -3716,17 +3740,148 @@ sub compile_ipsets {
     return $ipset_ruleset;
 }
 
+sub compile_ebtables_filter {
+    my ($cluster_conf, $hostfw_conf, $vmfw_configs, $vmdata, $verbose) = @_;
+
+    if (!($cluster_conf->{options}->{ebtables} // 1)) {
+	return {};
+    }
+
+    my $ruleset = {};
+
+    ruleset_create_chain($ruleset, "PVEFW-FORWARD");
+
+    ruleset_create_chain($ruleset, "PVEFW-FWBR-OUT");
+    #for ipv4 and ipv6, check macaddress in iptables, so we use conntrack 'ESTABLISHED', to speedup rules
+    ruleset_addrule($ruleset, 'PVEFW-FORWARD', '-p IPv4', '-j ACCEPT');
+    ruleset_addrule($ruleset, 'PVEFW-FORWARD', '-p IPv6', '-j ACCEPT');
+    ruleset_addrule($ruleset, 'PVEFW-FORWARD', '-o fwln+', '-j PVEFW-FWBR-OUT');
+
+    # generate firewall rules for QEMU VMs
+    foreach my $vmid (sort keys %{$vmdata->{qemu}}) {
+	eval {
+	    my $conf = $vmdata->{qemu}->{$vmid};
+	    my $vmfw_conf = $vmfw_configs->{$vmid};
+	    return if !$vmfw_conf;
+	    my $ipsets = $vmfw_conf->{ipset};
+
+	    foreach my $netid (sort keys %$conf) {
+		next if $netid !~ m/^net(\d+)$/;
+		my $net = PVE::QemuServer::parse_net($conf->{$netid});
+		next if !$net->{firewall};
+		my $iface = "tap${vmid}i$1";
+		my $macaddr = $net->{macaddr};
+		my $arpfilter = [];
+		if (defined(my $ipset = $ipsets->{"ipfilter-$netid"})) {
+		    foreach my $ipaddr (@$ipset) {
+			my($ip, $version) = parse_ip_or_cidr($ipaddr->{cidr});
+			next if !$ip || ($version && $version != 4);
+			push(@$arpfilter, $ip);
+		    }
+		}
+		generate_tap_layer2filter($ruleset, $iface, $macaddr, $vmfw_conf, $vmid, $arpfilter);
+	    }
+	};
+	warn $@ if $@; # just to be sure - should not happen
+    }
+
+    # generate firewall rules for LXC containers
+    foreach my $vmid (sort keys %{$vmdata->{lxc}}) {
+	eval {
+	    my $conf = $vmdata->{lxc}->{$vmid};
+
+	    my $vmfw_conf = $vmfw_configs->{$vmid};
+	    return if !$vmfw_conf || !$vmfw_conf->{options}->{enable};
+	    my $ipsets = $vmfw_conf->{ipset};
+
+	    foreach my $netid (sort keys %$conf) {
+		next if $netid !~ m/^net(\d+)$/;
+		my $net = PVE::LXC::Config->parse_lxc_network($conf->{$netid});
+		next if !$net->{firewall};
+		my $iface = "veth${vmid}i$1";
+		my $macaddr = $net->{hwaddr};
+		my $arpfilter = [];
+		if (defined(my $ipset = $ipsets->{"ipfilter-$netid"})) {
+		    foreach my $ipaddr (@$ipset) {
+			my($ip, $version) = parse_ip_or_cidr($ipaddr->{cidr});
+			next if !$ip || ($version && $version != 4);
+			push(@$arpfilter, $ip);
+		    }
+		}
+		push(@$arpfilter, $net->{ip}) if $net->{ip} && $vmfw_conf->{options}->{ipfilter};
+		generate_tap_layer2filter($ruleset, $iface, $macaddr, $vmfw_conf, $vmid, $arpfilter);
+	    }
+	};
+	warn $@ if $@; # just to be sure - should not happen
+    }
+
+    return $ruleset;
+}
+
+sub generate_tap_layer2filter {
+    my ($ruleset, $iface, $macaddr, $vmfw_conf, $vmid, $arpfilter) = @_;
+    my $options = $vmfw_conf->{options};
+
+    my $tapchain = $iface."-OUT";
+
+    # ebtables remove zeros from mac pairs
+    $macaddr =~ s/0([0-9a-f])/$1/ig;
+    $macaddr = lc($macaddr);
+
+    ruleset_create_chain($ruleset, $tapchain);
+
+    if (defined($macaddr) && !(defined($options->{macfilter}) && $options->{macfilter} == 0)) {
+	    ruleset_addrule($ruleset, $tapchain, "-s ! $macaddr", '-j DROP');
+    }
+
+    if (@$arpfilter){
+	my $arpchain = $tapchain."-ARP";
+	ruleset_addrule($ruleset, $tapchain, "-p ARP", "-j $arpchain");
+	ruleset_create_chain($ruleset, $arpchain);
+
+	foreach my $ip (@{$arpfilter}) {
+	    ruleset_addrule($ruleset, $arpchain, "-p ARP --arp-ip-src $ip", '-j RETURN');
+	}
+	ruleset_addrule($ruleset, $arpchain, '', '-j DROP');
+    }
+
+    if (defined($options->{layer2_protocols})){
+	my $protochain = $tapchain."-PROTO";
+	ruleset_addrule($ruleset, $tapchain, '', "-j $protochain");
+	ruleset_create_chain($ruleset, $protochain);
+
+	foreach my $proto (split(/,/, $options->{layer2_protocols})) {
+	    ruleset_addrule($ruleset, $protochain, "-p $proto", '-j RETURN');
+	}
+	ruleset_addrule($ruleset, $protochain, '', '-j DROP');
+    }
+
+    ruleset_addrule($ruleset, $tapchain, '', '-j ACCEPT');
+
+    ruleset_addrule($ruleset, 'PVEFW-FWBR-OUT', "-i $iface", "-j $tapchain");
+}
+
+# the parameter $change_only_regex changes two things if defined:
+# * all chains not matching it will be left intact
+# * both the $active_chains hash and the returned status_hash have different
+#   structure (they contain a key named 'rules').
 sub get_ruleset_status {
-    my ($ruleset, $active_chains, $digest_fn, $verbose) = @_;
+    my ($ruleset, $active_chains, $digest_fn, $verbose, $change_only_regex) = @_;
 
     my $statushash = {};
 
     foreach my $chain (sort keys %$ruleset) {
-	my $sig = &$digest_fn($ruleset->{$chain});
+	my $rules = $ruleset->{$chain};
+	my $sig = &$digest_fn($rules);
+	my $oldsig;
 
 	$statushash->{$chain}->{sig} = $sig;
-
-	my $oldsig = $active_chains->{$chain};
+	if (defined($change_only_regex)) {
+	    $oldsig = $active_chains->{$chain}->{sig};
+	    $statushash->{$chain}->{rules} = $rules;
+	} else {
+	    $oldsig = $active_chains->{$chain};
+	}
 	if (!defined($oldsig)) {
 	    $statushash->{$chain}->{action} = 'create';
 	} else {
@@ -3736,19 +3891,26 @@ sub get_ruleset_status {
 		$statushash->{$chain}->{action} = 'update';
 	    }
 	}
-	print "$statushash->{$chain}->{action} $chain ($sig)\n" if $verbose;
-	foreach my $cmd (@{$ruleset->{$chain}}) {
-	    print "\t$cmd\n" if $verbose;
+	if ($verbose) {
+	    print "$statushash->{$chain}->{action} $chain ($sig)\n";
+	    foreach my $cmd (@{$rules}) {
+		print "\t$cmd\n";
+	    }
 	}
     }
 
     foreach my $chain (sort keys %$active_chains) {
-	if (!defined($ruleset->{$chain})) {
-	    my $sig = $active_chains->{$chain};
-	    $statushash->{$chain}->{action} = 'delete';
-	    $statushash->{$chain}->{sig} = $sig;
-	    print "delete $chain ($sig)\n" if $verbose;
+	next if defined($ruleset->{$chain});
+	my $action = 'delete';
+	my $sig = $active_chains->{$chain};
+	if (defined($change_only_regex)) {
+	    $action = 'ignore' if ($chain !~ m/$change_only_regex/);
+	    $statushash->{$chain}->{rules} = $active_chains->{$chain}->{rules};
+	    $sig = $sig->{sig};
 	}
+	$statushash->{$chain}->{action} = $action;
+	$statushash->{$chain}->{sig} = $sig;
+	print "$action $chain ($sig)\n" if $verbose;
     }
 
     return $statushash;
@@ -3823,6 +3985,46 @@ sub get_ruleset_cmdlist {
     return wantarray ? ($cmdlist, $changes) : $cmdlist;
 }
 
+my $pve_ebtables_chainname_regex = qr/PVEFW-\S+|(?:tap|veth)\d+i\d+-(?:IN|OUT)/;
+
+sub get_ebtables_cmdlist {
+    my ($ruleset, $verbose) = @_;
+
+    my $changes = 0;
+    my $cmdlist = "*filter\n";
+
+    my $active_chains = ebtables_get_chains();
+    my $statushash = get_ruleset_status($ruleset, $active_chains,
+					\&iptables_chain_digest, $verbose,
+					$pve_ebtables_chainname_regex);
+
+    # create chains first and make sure PVE rules are evaluated if active
+    my $append_pve_to_forward = '-A FORWARD -j PVEFW-FORWARD';
+    my $pve_include = 0;
+    foreach my $chain (sort keys %$statushash) {
+	next if ($statushash->{$chain}->{action} eq 'delete');
+	$cmdlist .= ":$chain ACCEPT\n";
+	$pve_include = 1 if ($chain eq 'PVEFW-FORWARD');
+    }
+
+    foreach my $chain (sort keys %$statushash) {
+	my $stat = $statushash->{$chain};
+	next if ($stat->{action} eq 'delete');
+	$changes = 1 if ($stat->{action} !~ 'ignore|exists');
+
+	foreach my $cmd (@{$statushash->{$chain}->{'rules'}}) {
+	    if ($chain eq 'FORWARD' && $cmd eq $append_pve_to_forward) {
+		next if ! $pve_include;
+		$pve_include = 0;
+	    }
+	    $cmdlist .= "$cmd\n";
+	}
+    }
+    $cmdlist .= "$append_pve_to_forward\n" if $pve_include;
+
+    return wantarray ? ($cmdlist, $changes) : $cmdlist;
+}
+
 sub get_ipset_cmdlist {
     my ($ruleset, $verbose) = @_;
 
@@ -3882,7 +4084,7 @@ sub get_ipset_cmdlist {
 }
 
 sub apply_ruleset {
-    my ($ruleset, $hostfw_conf, $ipset_ruleset, $rulesetv6, $verbose) = @_;
+    my ($ruleset, $hostfw_conf, $ipset_ruleset, $rulesetv6, $ebtables_ruleset, $verbose) = @_;
 
     enable_bridge_firewall();
 
@@ -3891,6 +4093,7 @@ sub apply_ruleset {
 
     my ($cmdlist, $changes) = get_ruleset_cmdlist($ruleset, $verbose);
     my ($cmdlistv6, $changesv6) = get_ruleset_cmdlist($rulesetv6, $verbose, "ip6tables");
+    my ($ebtables_cmdlist, $ebtables_changes) = get_ebtables_cmdlist($ebtables_ruleset, $verbose);
 
     if ($verbose) {
 	if ($ipset_changes) {
@@ -3908,6 +4111,11 @@ sub apply_ruleset {
 	    print "ip6tables changes:\n";
 	    print $cmdlistv6;
 	}
+
+	if ($ebtables_changes) {
+	    print "ebtables changes:\n";
+	    print $ebtables_cmdlist;
+	}
     }
 
     my $tmpfile = "$pve_fw_status_dir/ipsetcmdlist1";
@@ -3930,6 +4138,11 @@ sub apply_ruleset {
 
     ipset_restore_cmdlist($ipset_delete_cmdlist) if $ipset_delete_cmdlist;
 
+    ebtables_restore_cmdlist($ebtables_cmdlist);
+
+    $tmpfile = "$pve_fw_status_dir/ebtablescmdlist";
+    PVE::Tools::file_set_contents($tmpfile, $ebtables_cmdlist || '');
+
     # test: re-read status and check if everything is up to date
     my $active_chains = iptables_get_chains();
     my $statushash = get_ruleset_status($ruleset, $active_chains, \&iptables_chain_digest, 0);
@@ -3954,12 +4167,26 @@ sub apply_ruleset {
 	}
     }
 
+    my $active_ebtables_chains = ebtables_get_chains();
+    my $ebtables_statushash = get_ruleset_status($ebtables_ruleset,
+				$active_ebtables_chains, \&iptables_chain_digest,
+				0, $pve_ebtables_chainname_regex);
+
+    foreach my $chain (sort keys %$ebtables_ruleset) {
+	my $stat = $ebtables_statushash->{$chain};
+	if ($stat->{action} ne 'exists') {
+	    warn "ebtables : unable to update chain '$chain'\n";
+	    $errors = 1;
+	}
+    }
+
     die "unable to apply firewall changes\n" if $errors;
 
     update_nf_conntrack_max($hostfw_conf);
 
     update_nf_conntrack_tcp_timeout_established($hostfw_conf);
 
+    update_nf_conntrack_logging($hostfw_conf);
 }
 
 sub update_nf_conntrack_max {
@@ -3996,6 +4223,23 @@ sub update_nf_conntrack_tcp_timeout_established {
     PVE::ProcFSTools::write_proc_entry("/proc/sys/net/netfilter/nf_conntrack_tcp_timeout_established", $value);
 }
 
+my $log_nf_conntrack_enabled = undef;
+sub update_nf_conntrack_logging {
+    my ($hostfw_conf) = @_;
+
+    my $options = $hostfw_conf->{options} || {};
+    my $value = $options->{log_nf_conntrack} || 0;
+    if (!defined($log_nf_conntrack_enabled)
+	|| $value != $log_nf_conntrack_enabled)
+    {
+	my $tmpfile = "$pve_fw_status_dir/log_nf_conntrack";
+	PVE::Tools::file_set_contents($tmpfile, $value);
+
+	PVE::Tools::run_command([qw(systemctl try-reload-or-restart pvefw-logger.service)]);
+	$log_nf_conntrack_enabled = $value;
+    }
+}
+
 sub remove_pvefw_chains {
 
     PVE::Firewall::remove_pvefw_chains_iptables("iptables");
@@ -4069,9 +4313,9 @@ sub update {
 
 	my $hostfw_conf = load_hostfw_conf($cluster_conf);
 
-	my ($ruleset, $ipset_ruleset, $rulesetv6) = compile($cluster_conf, $hostfw_conf);
+	my ($ruleset, $ipset_ruleset, $rulesetv6, $ebtables_ruleset) = compile($cluster_conf, $hostfw_conf);
 
-	apply_ruleset($ruleset, $hostfw_conf, $ipset_ruleset, $rulesetv6);
+	apply_ruleset($ruleset, $hostfw_conf, $ipset_ruleset, $rulesetv6, $ebtables_ruleset);
     };
 
     run_locked($code);