X-Git-Url: https://git.proxmox.com/?p=pve-firewall.git;a=blobdiff_plain;f=src%2FPVE%2FFirewall.pm;h=6d4127e8585917c51c841497959d902a8a8620cd;hp=45f6a5d8d00275b02e31c6b5e1fb9209abb0ec91;hb=5b7974dfa29048836d6ca5f66a05c96e54732cbf;hpb=dd7a13fddc9f4c727fdd537618e43a4b8967e079

diff --git a/src/PVE/Firewall.pm b/src/PVE/Firewall.pm
index 45f6a5d..6d4127e 100644
--- a/src/PVE/Firewall.pm
+++ b/src/PVE/Firewall.pm
@@ -2,6 +2,7 @@ package PVE::Firewall;
 
 use warnings;
 use strict;
+use POSIX;
 use Data::Dumper;
 use Digest::SHA;
 use PVE::INotify;
@@ -16,9 +17,10 @@ use Net::IP;
 use PVE::Tools qw(run_command lock_file);
 use Encode;
 
+my $hostfw_conf_filename = "/etc/pve/local/host.fw";
 my $clusterfw_conf_filename = "/etc/pve/firewall/cluster.fw";
 
-# dynamically include PVE::QemuServer and PVE::OpenVZ 
+# dynamically include PVE::QemuServer and PVE::OpenVZ
 # to avoid dependency problems
 my $have_qemu_server;
 eval {
@@ -32,6 +34,17 @@ eval {
     $have_pve_manager = 1;
 };
 
+my $feature_ipset_nomatch = 0;
+eval  {
+    my (undef, undef, $release) = POSIX::uname();
+    if ($release =~ m/^(\d+)\.(\d+)\.\d+-/) {
+	my ($major, $minor) = ($1, $2);
+	$feature_ipset_nomatch = 1 if ($major > 3) ||
+	    ($major == 3 && $minor >= 7);
+    }
+
+};
+
 use Data::Dumper;
 
 my $nodename = PVE::INotify::nodename();
@@ -396,12 +409,12 @@ my $pve_std_chains = {
 	{ action => 'DROP', dsttype => 'BROADCAST' },
 	{ action => 'DROP', dsttype => 'MULTICAST' },
 	{ action => 'DROP', dsttype => 'ANYCAST' },
-	{ action => 'DROP', dest => '224.0.0.0/4' }, 
+	{ action => 'DROP', dest => '224.0.0.0/4' },
     ],
     'PVEFW-reject' => [
 	# same as shorewall 'reject'
 	{ action => 'DROP', dsttype => 'BROADCAST' },
-	{ action => 'DROP', source => '224.0.0.0/4' }, 
+	{ action => 'DROP', source => '224.0.0.0/4' },
 	{ action => 'DROP', proto => 'icmp' },
 	"-p tcp -j REJECT --reject-with tcp-reset",
 	"-p udp -j REJECT --reject-with icmp-port-unreachable",
@@ -409,7 +422,7 @@ my $pve_std_chains = {
 	"-j REJECT --reject-with icmp-host-prohibited",
     ],
     'PVEFW-Drop' => [
-	# same as shorewall 'Drop', which is equal to DROP, 
+	# same as shorewall 'Drop', which is equal to DROP,
 	# but REJECT/DROP some packages to reduce logging,
 	# and ACCEPT critical ICMP types
 	{ action => 'PVEFW-reject',  proto => 'tcp', dport => '43' }, # REJECT 'auth'
@@ -432,7 +445,7 @@ my $pve_std_chains = {
 	{ action => 'DROP', proto => 'udp', sport => 53 },
     ],
     'PVEFW-Reject' => [
-	# same as shorewall 'Reject', which is equal to Reject, 
+	# same as shorewall 'Reject', which is equal to Reject,
 	# but REJECT/DROP some packages to reduce logging,
 	# and ACCEPT critical ICMP types
 	{ action => 'PVEFW-reject',  proto => 'tcp', dport => '43' }, # REJECT 'auth'
@@ -600,16 +613,20 @@ sub get_etc_protocols {
 sub parse_address_list {
     my ($str) = @_;
 
-    my $nbaor = 0;
-    foreach my $aor (split(/,/, $str)) {
-	if (!Net::IP->new($aor)) {
+    return if $str !~ m/^(\+)(\S+)$/; # ipset ref
+
+    my $count = 0;
+    my $iprange = 0;
+    foreach my $elem (split(/,/, $str)) {
+	$count++;
+	if (!Net::IP->new($elem)) {
 	    my $err = Net::IP::Error();
 	    die "invalid IP address: $err\n";
-	}else{
-	    $nbaor++;
 	}
+	$iprange = 1 if $elem =~ m/-/;
     }
-    return $nbaor;
+    
+    die "you can use a range in a list\n" if $iprange && $count > 1;
 }
 
 sub parse_port_name_number_or_range {
@@ -750,12 +767,28 @@ sub add_rule_properties {
     my ($properties) = @_;
 
     foreach my $k (keys %$rule_properties) {
-	$properties->{$k} = $rule_properties->{$k};
+	my $h = $rule_properties->{$k};
+	# copy data, so that we can modify later without side effects
+	foreach my $opt (keys %$h) { $properties->{$k}->{$opt} = $h->{$opt}; }
     }
-    
+
     return $properties;
 }
 
+sub delete_rule_properties {
+    my ($rule, $delete_str) = @_;
+    
+    foreach my $opt (PVE::Tools::split_list($delete_str)) {
+	raise_param_exc({ 'delete' => "no such property ('$opt')"})
+	    if !defined($rule_properties->{$opt});
+	raise_param_exc({ 'delete' => "unable to delete required property '$opt'"})
+	    if $opt eq 'type' || $opt eq 'action';
+	delete $rule->{$opt};
+    }
+
+    return $rule;
+}
+
 sub copy_rule_data {
     my ($rule, $param) = @_;
 
@@ -892,8 +925,10 @@ sub ipset_get_chains {
 
 	return if $line =~ m/^#/;
 	return if $line =~ m/^\s*$/;
-	if ($line =~ m/^(\S+)\s(\S+)\s(\S+)/) {
-	    push @{$chains->{$2}}, $line;
+	if ($line =~ m/^(?:\S+)\s(PVEFW-\S+)\s(?:\S+).*/) {
+	    my $chain = $1;
+	    $line =~ s/\s+$//; # delete trailing white space
+	    push @{$chains->{$chain}}, $line;
 	} else {
 	    # simply ignore the rest
 	    return;
@@ -911,7 +946,7 @@ sub ipset_get_chains {
 }
 
 sub ruleset_generate_cmdstr {
-    my ($ruleset, $chain, $rule, $actions, $goto) = @_;
+    my ($ruleset, $chain, $rule, $actions, $goto, $cluster_conf) = @_;
 
     return if defined($rule->{enable}) && !$rule->{enable};
 
@@ -919,18 +954,40 @@ sub ruleset_generate_cmdstr {
 
     my $nbdport = defined($rule->{dport}) ? parse_port_name_number_or_range($rule->{dport}) : 0;
     my $nbsport = defined($rule->{sport}) ? parse_port_name_number_or_range($rule->{sport}) : 0;
-    my $nbsource = $rule->{source} ? parse_address_list( $rule->{source}) : 0;
-    my $nbdest = $rule->{dest} ? parse_address_list($rule->{dest}) : 0;
 
     my @cmd = ();
 
     push @cmd, "-i $rule->{iface_in}" if $rule->{iface_in};
     push @cmd, "-o $rule->{iface_out}" if $rule->{iface_out};
 
-    push @cmd, "-m iprange --src-range" if $nbsource > 1;
-    push @cmd, "-s $rule->{source}" if $rule->{source};
-    push @cmd, "-m iprange --dst-range" if $nbdest > 1;
-    push @cmd, "-d $rule->{dest}" if $rule->{dest};
+    my $source = $rule->{source};
+    my $dest = $rule->{dest};
+
+    if ($source) {
+        if ($source =~ m/^(\+)(\S+)$/) {
+	    die "no such ipset $2" if !$cluster_conf->{ipset}->{$2};
+	    push @cmd, "-m set --match-set PVEFW-$2 src";
+
+        } elsif ($source =~ m/\-/){
+	    push @cmd, "-m iprange --src-range $source";
+
+	} else {
+	    push @cmd, "-s $source";
+        }
+    }
+
+    if ($dest) {
+        if ($dest =~ m/^(\+)(\S+)$/) {
+	    die "no such ipset $2" if !$cluster_conf->{ipset}->{$2};
+	    push @cmd, "-m set --match-set PVEFW-$2 dst";
+
+        } elsif ($dest =~ m/^(\d+)\.(\d+).(\d+).(\d+)\-(\d+)\.(\d+).(\d+).(\d+)$/){
+	    push @cmd, "-m iprange --dst-range $dest";
+
+	} else {
+	    push @cmd, "-s $dest";
+        }
+    }
 
     if ($rule->{proto}) {
 	push @cmd, "-p $rule->{proto}";
@@ -941,7 +998,7 @@ sub ruleset_generate_cmdstr {
 
 	push @cmd, "--match multiport" if $multiport;
 
-	die "multiport: option '--sports' cannot be used together with '--dports'\n" 
+	die "multiport: option '--sports' cannot be used together with '--dports'\n"
 	    if ($multiport == 2) && ($rule->{dport} ne $rule->{sport});
 
 	if ($rule->{dport}) {
@@ -977,7 +1034,7 @@ sub ruleset_generate_cmdstr {
     push @cmd, "-m addrtype --dst-type $rule->{dsttype}" if $rule->{dsttype};
 
     if (my $action = $rule->{action}) {
-	$action = $actions->{$action} if defined($actions->{$action}); 
+	$action = $actions->{$action} if defined($actions->{$action});
 	$goto = 1 if !defined($goto) && $action eq 'PVEFW-SET-ACCEPT-MARK';
 	push @cmd, $goto ? "-g $action" : "-j $action";
     }
@@ -1030,7 +1087,7 @@ my $apply_macro = sub {
 };
 
 sub ruleset_generate_rule {
-    my ($ruleset, $chain, $rule, $actions, $goto) = @_;
+    my ($ruleset, $chain, $rule, $actions, $goto, $cluster_conf) = @_;
 
     my $rules;
 
@@ -1040,8 +1097,8 @@ sub ruleset_generate_rule {
 	$rules = [ $rule ];
     }
 
-    foreach my $tmp (@$rules) { 
-	if (my $cmdstr = ruleset_generate_cmdstr($ruleset, $chain, $tmp, $actions, $goto)) {
+    foreach my $tmp (@$rules) {
+	if (my $cmdstr = ruleset_generate_cmdstr($ruleset, $chain, $tmp, $actions, $goto, $cluster_conf)) {
 	    ruleset_addrule($ruleset, $chain, $cmdstr);
 	}
     }
@@ -1092,12 +1149,12 @@ sub ruleset_insertrule {
 
 sub get_log_rule_base {
     my ($chain, $vmid, $msg, $loglevel) = @_;
-    
+
     die "internal error - no log level" if !defined($loglevel);
 
     $vmid = 0 if !defined($vmid);
 
-    # Note: we use special format for prefix to pass further 
+    # Note: we use special format for prefix to pass further
     # info to log daemon (VMID, LOGVELEL and CHAIN)
 
     return "-j NFLOG --nflog-prefix \":$vmid:$loglevel:$chain: $msg\"";
@@ -1183,10 +1240,10 @@ sub ruleset_create_vm_chain {
 
     if (!(defined($options->{dhcp}) && $options->{dhcp} == 0)) {
 	if ($direction eq 'OUT') {
-	    ruleset_generate_rule($ruleset, $chain, { action => 'PVEFW-SET-ACCEPT-MARK', 
+	    ruleset_generate_rule($ruleset, $chain, { action => 'PVEFW-SET-ACCEPT-MARK',
 						      proto => 'udp', sport => 68, dport => 67 });
 	} else {
-	    ruleset_generate_rule($ruleset, $chain, { action => 'ACCEPT', 
+	    ruleset_generate_rule($ruleset, $chain, { action => 'ACCEPT',
 						      proto => 'udp', sport => 67, dport => 68 });
 	}
     }
@@ -1196,20 +1253,18 @@ sub ruleset_create_vm_chain {
     }
 
     ruleset_addrule($ruleset, $chain, "-m conntrack --ctstate INVALID -j DROP");
-    if($direction eq 'OUT'){
+    if ($direction eq 'OUT') {
 	ruleset_addrule($ruleset, $chain, "-m conntrack --ctstate RELATED,ESTABLISHED -g PVEFW-SET-ACCEPT-MARK");
-
-    }else{
+    } else {
 	ruleset_addrule($ruleset, $chain, "-m conntrack --ctstate RELATED,ESTABLISHED -j $accept");
     }
+
     if ($direction eq 'OUT') {
 	if (defined($macaddr) && !(defined($options->{macfilter}) && $options->{macfilter} == 0)) {
 	    ruleset_addrule($ruleset, $chain, "-m mac ! --mac-source $macaddr -j DROP");
 	}
 	ruleset_addrule($ruleset, $chain, "-j MARK --set-mark 0"); # clear mark
     }
-
-
 }
 
 sub ruleset_generate_vm_rules {
@@ -1221,7 +1276,7 @@ sub ruleset_generate_vm_rules {
 	next if $rule->{iface} && $rule->{iface} ne $netid;
 	next if !$rule->{enable};
 	if ($rule->{type} eq 'group') {
-	    my $group_chain = "GROUP-$rule->{action}-$direction"; 
+	    my $group_chain = "GROUP-$rule->{action}-$direction";
 	    if(!ruleset_chain_exist($ruleset, $group_chain)){
 		generate_group_rules($ruleset, $cluster_conf, $rule->{action});
 	    }
@@ -1236,11 +1291,11 @@ sub ruleset_generate_vm_rules {
 	} else {
 	    next if $rule->{type} ne $lc_direction;
 	    if ($direction eq 'OUT') {
-		ruleset_generate_rule($ruleset, $chain, $rule, 
-				      { ACCEPT => "PVEFW-SET-ACCEPT-MARK", REJECT => "PVEFW-reject" });
+		ruleset_generate_rule($ruleset, $chain, $rule,
+				      { ACCEPT => "PVEFW-SET-ACCEPT-MARK", REJECT => "PVEFW-reject" }, undef, $cluster_conf);
 	    } else {
 		my $accept = generate_nfqueue($options);
-		ruleset_generate_rule($ruleset, $chain, $rule, { ACCEPT => $accept , REJECT => "PVEFW-reject" });
+		ruleset_generate_rule($ruleset, $chain, $rule, { ACCEPT => $accept , REJECT => "PVEFW-reject" }, undef, $cluster_conf);
 	    }
 	}
     }
@@ -1416,7 +1471,7 @@ sub enable_host_firewall {
 
     foreach my $rule (@$rules) {
 	next if $rule->{type} ne 'in';
-	ruleset_generate_rule($ruleset, $chain, $rule, { ACCEPT => $accept_action, REJECT => "PVEFW-reject" });
+	ruleset_generate_rule($ruleset, $chain, $rule, { ACCEPT => $accept_action, REJECT => "PVEFW-reject" }, undef, $cluster_conf);
     }
 
     # implement input policy
@@ -1441,7 +1496,7 @@ sub enable_host_firewall {
 
     foreach my $rule (@$rules) {
 	next if $rule->{type} ne 'out';
-	ruleset_generate_rule($ruleset, $chain, $rule, { ACCEPT => $accept_action, REJECT => "PVEFW-reject" });
+	ruleset_generate_rule($ruleset, $chain, $rule, { ACCEPT => $accept_action, REJECT => "PVEFW-reject" }, undef, $cluster_conf);
     }
 
     # implement output policy
@@ -1465,7 +1520,7 @@ sub generate_group_rules {
 
     foreach my $rule (@$rules) {
 	next if $rule->{type} ne 'in';
-	ruleset_generate_rule($ruleset, $chain, $rule, { ACCEPT => "PVEFW-SET-ACCEPT-MARK", REJECT => "PVEFW-reject" });
+	ruleset_generate_rule($ruleset, $chain, $rule, { ACCEPT => "PVEFW-SET-ACCEPT-MARK", REJECT => "PVEFW-reject" }, undef, $cluster_conf);
     }
 
     $chain = "GROUP-${group}-OUT";
@@ -1477,8 +1532,8 @@ sub generate_group_rules {
 	next if $rule->{type} ne 'out';
 	# we use PVEFW-SET-ACCEPT-MARK (Instead of ACCEPT) because we need to
 	# check also other tap rules later
-	ruleset_generate_rule($ruleset, $chain, $rule, 
-			      { ACCEPT => 'PVEFW-SET-ACCEPT-MARK', REJECT => "PVEFW-reject" });
+	ruleset_generate_rule($ruleset, $chain, $rule,
+			      { ACCEPT => 'PVEFW-SET-ACCEPT-MARK', REJECT => "PVEFW-reject" }, undef, $cluster_conf);
     }
 }
 
@@ -1533,7 +1588,7 @@ sub parse_fw_rule {
 	die "wrong number of rule elements\n" if scalar(@data) != 3;
 	die "groups disabled\n" if !$allow_groups;
 
-	die "invalid characters in group name\n" if $action !~ m/^[A-Za-z0-9_\-]+$/;	
+	die "invalid characters in group name\n" if $action !~ m/^[A-Za-z0-9_\-]+$/;
     } else {
 	die "unknown rule type '$type'\n";
     }
@@ -1553,7 +1608,7 @@ sub parse_fw_rule {
 
     parse_port_name_number_or_range($dport) if defined($dport);
     parse_port_name_number_or_range($sport) if defined($sport);
- 
+
     parse_address_list($source) if $source;
     parse_address_list($dest) if $dest;
 
@@ -1741,11 +1796,11 @@ sub parse_host_fw_rules {
 	    warn "$prefix: $err";
 	    next;
 	}
-	
+
 	push @{$res->{$section}}, $rule;
     }
 
-    $res->{digest} = $digest->b64digest;
+$res->{digest} = $digest->b64digest;
 
     return $res;
 }
@@ -1784,15 +1839,15 @@ sub parse_cluster_fw_rules {
 	    $section = 'rules';
 	    next;
 	}
-    
-	if ($line =~ m/^\[netgroup\s+(\S+)\]\s*$/i) {
+
+	if ($line =~ m/^\[ipset\s+(\S+)\]\s*$/i) {
 	    $section = 'ipset';
 	    $group = lc($1);
 	    next;
 	}
 
 	if (!$section) {
-	    warn "$prefix: skip line - no section";
+	    warn "$prefix: skip line - no section\n";
 	    next;
 	}
 
@@ -1820,21 +1875,28 @@ sub parse_cluster_fw_rules {
 	    push @{$res->{$section}->{$group}}, $rule;
 	} elsif ($section eq 'ipset') {
 	    chomp $line;
-	    $line =~ m/^(\!)?(\s)?((\d+)\.(\d+)\.(\d+)\.(\d+)(\/(\d+))?)/;
+	    $line =~ m/^(\!)?\s*((\d+)\.(\d+)\.(\d+)\.(\d+)(\/(\d+))?)/;
 	    my $nomatch = $1;
-	    my $ip = $3;
+	my $ip = $2;
 
 	    if(!$ip){
-		warn "$prefix: $line is not an valid ip address";
+		warn "$prefix: $line is not an valid ip address\n";
 		next;
 	    }
 	    if (!Net::IP->new($ip)) {
-		warn "$prefix: $line is not an valid ip address";
+		warn "$prefix: $line is not an valid ip address\n";
 		next;
 	    }
-	    $ip .= " nomatch" if $nomatch;
 
-	    push @{$res->{$section}->{$group}}, $ip;
+	    if ($nomatch) {
+		if ($feature_ipset_nomatch) {
+		    push @{$res->{$section}->{$group}}, "$ip nomatch";
+		} else {
+		    warn "$prefix: ignore $line - nomatch not supported by kernel\n";
+		}
+	    } else {
+		push @{$res->{$section}->{$group}}, $ip;
+	    }
 	}
     }
 
@@ -1903,6 +1965,66 @@ sub load_vmfw_conf {
     return $vmfw_conf;
 }
 
+my $format_rules = sub {
+    my ($rules, $need_iface) = @_;
+
+    my $raw = '';
+
+    foreach my $rule (@$rules) {
+	if ($rule->{type} eq  'in' || $rule->{type} eq 'out') {
+	    $raw .= '|' if defined($rule->{enable}) && !$rule->{enable};
+	    $raw .= uc($rule->{type});
+	    $raw .= " " . $rule->{action};
+	    $raw .= " " . ($rule->{iface} || '-') if $need_iface;
+	    $raw .= " " . ($rule->{source} || '-');
+	    $raw .= " " . ($rule->{dest} || '-');
+	    $raw .= " " . ($rule->{proto} || '-');
+	    $raw .= " " . ($rule->{dport} || '-');
+	    $raw .= " " . ($rule->{sport} || '-');
+	    $raw .= " # " . encode('utf8', $rule->{comment})
+		if $rule->{comment} && $rule->{comment} !~ m/^\s*$/;
+	    $raw .= "\n";
+	} else {
+	    die "implement me '$rule->{type}'";
+	}
+    }
+
+    return $raw;
+};
+
+my $format_options = sub {
+    my ($options) = @_;
+
+    my $raw = '';
+
+    $raw .= "[OPTIONS]\n\n";
+    foreach my $opt (keys %$options) {
+	$raw .= "$opt: $options->{$opt}\n";
+    }
+    $raw .= "\n";
+
+    return $raw;
+};
+
+sub save_vmfw_conf {
+    my ($vmid, $vmfw_conf) = @_;
+
+    my $raw = '';
+
+    my $options = $vmfw_conf->{options};
+    $raw .= &$format_options($options) if scalar(keys %$options);
+
+    my $rules = $vmfw_conf->{rules};
+    if (scalar(@$rules)) {
+	$raw .= "[RULES]\n\n";
+	$raw .= &$format_rules($rules, 1);
+	$raw .= "\n";
+    }
+
+    my $filename = "/etc/pve/firewall/$vmid.fw";
+    PVE::Tools::file_set_contents($filename, $raw);
+}
+
 sub read_vm_firewall_configs {
     my ($vmdata) = @_;
     my $vmfw_configs = {};
@@ -1935,7 +2057,7 @@ sub get_option_log_level {
 
 sub generate_std_chains {
     my ($ruleset, $options) = @_;
-    
+
     my $loglevel = get_option_log_level($options, 'smurf_log_level');
 
     # same as shorewall smurflog.
@@ -1967,7 +2089,7 @@ sub generate_ipset_chains {
     my ($ipset_ruleset, $fw_conf) = @_;
 
     foreach my $ipset (keys %{$fw_conf->{ipset}}) {
-	generate_ipset($ipset_ruleset, $ipset, $fw_conf->{ipset}->{$ipset});
+	generate_ipset($ipset_ruleset, "PVEFW-$ipset", $fw_conf->{ipset}->{$ipset});
     }
 }
 
@@ -2058,60 +2180,27 @@ sub load_clusterfw_conf {
     return $cluster_conf;
 }
 
-my $rules_to_conf = sub {
-    my ($rules, $need_iface) = @_;
-
-    my $raw = '';
-
-    foreach my $rule (@$rules) {
-	if ($rule->{type} eq  'in' || $rule->{type} eq 'out') {
-	    $raw .= '|' if defined($rule->{enable}) && !$rule->{enable};
-	    $raw .= uc($rule->{type});
-	    $raw .= " " . $rule->{action};
-	    $raw .= " " . ($rule->{iface} || '-') if $need_iface;
-	    $raw .= " " . ($rule->{source} || '-');
-	    $raw .= " " . ($rule->{dest} || '-');
-	    $raw .= " " . ($rule->{proto} || '-');
-	    $raw .= " " . ($rule->{dport} || '-');
-	    $raw .= " " . ($rule->{sport} || '-');
-	    $raw .= " # " . encode('utf8', $rule->{comment}) 
-		if $rule->{comment} && $rule->{comment} !~ m/^\s*$/;
-	    $raw .= "\n";
-	} else {
-	    die "implement me '$rule->{type}'";
-	}
-    }
-
-    return $raw;
-};
-
 sub save_clusterfw_conf {
     my ($cluster_conf) = @_;
 
     my $raw = '';
 
     my $options = $cluster_conf->{options};
-    if (scalar(keys %$options)) {
-	$raw .= "[OPTIONS]\n\n";
-	foreach my $opt (keys %$options) {
-	    $raw .= "$opt: $options->{$opt}\n";
-	}
-	$raw .= "\n";
-    }
+    $raw .= &$format_options($options) if scalar(keys %$options);
 
     # fixme: save ipset
 
     my $rules = $cluster_conf->{rules};
     if (scalar(@$rules)) {
 	$raw .= "[RULES]\n\n";
-	$raw .= &$rules_to_conf($rules, 1);
+	$raw .= &$format_rules($rules, 1);
 	$raw .= "\n";
     }
 
     foreach my $group (sort keys %{$cluster_conf->{groups}}) {
 	my $rules = $cluster_conf->{groups}->{$group};
 	$raw .= "[group $group]\n\n";
-	$raw .= &$rules_to_conf($rules, 0);
+	$raw .= &$format_rules($rules, 0);
 	$raw .= "\n";
     }
 
@@ -2121,13 +2210,30 @@ sub save_clusterfw_conf {
 sub load_hostfw_conf {
 
     my $hostfw_conf = {};
-    my $filename = "/etc/pve/local/host.fw";
-    if (my $fh = IO::File->new($filename, O_RDONLY)) {
-	$hostfw_conf = parse_host_fw_rules($filename, $fh);
+    if (my $fh = IO::File->new($hostfw_conf_filename, O_RDONLY)) {
+	$hostfw_conf = parse_host_fw_rules($hostfw_conf_filename, $fh);
     }
     return $hostfw_conf;
 }
 
+sub save_hostfw_conf {
+    my ($hostfw_conf) = @_;
+
+    my $raw = '';
+
+    my $options = $hostfw_conf->{options};
+    $raw .= &$format_options($options) if scalar(keys %$options);
+
+    my $rules = $hostfw_conf->{rules};
+    if (scalar(@$rules)) {
+	$raw .= "[RULES]\n\n";
+	$raw .= &$format_rules($rules, 1);
+	$raw .= "\n";
+    }
+
+    PVE::Tools::file_set_contents($hostfw_conf_filename, $raw);
+}
+
 sub compile {
     my $vmdata = read_local_vm_config();
     my $vmfw_configs = read_vm_firewall_configs($vmdata);
@@ -2176,9 +2282,9 @@ sub compile {
 	    generate_bridge_chains($ruleset, $hostfw_conf, $bridge, $routing_table);
 
 	    my $macaddr = $net->{macaddr};
-	    generate_tap_rules_direction($ruleset, $cluster_conf, $iface, $netid, $macaddr, 
+	    generate_tap_rules_direction($ruleset, $cluster_conf, $iface, $netid, $macaddr,
 					 $vmfw_conf, $vmid, $bridge, 'IN');
-	    generate_tap_rules_direction($ruleset, $cluster_conf, $iface, $netid, $macaddr, 
+	    generate_tap_rules_direction($ruleset, $cluster_conf, $iface, $netid, $macaddr,
 					 $vmfw_conf, $vmid, $bridge, 'OUT');
 	}
     }
@@ -2206,14 +2312,14 @@ sub compile {
 		    warn "no bridge device for CT $vmid iface '$netid'\n";
 		    next; # fixme?
 		}
-		
+
 		generate_bridge_chains($ruleset, $hostfw_conf, $bridge, $routing_table);
 
 		my $macaddr = $d->{mac};
 		my $iface = $d->{host_ifname};
-		generate_tap_rules_direction($ruleset, $cluster_conf, $iface, $netid, $macaddr, 
+		generate_tap_rules_direction($ruleset, $cluster_conf, $iface, $netid, $macaddr,
 					     $vmfw_conf, $vmid, $bridge, 'IN');
-		generate_tap_rules_direction($ruleset, $cluster_conf, $iface, $netid, $macaddr, 
+		generate_tap_rules_direction($ruleset, $cluster_conf, $iface, $netid, $macaddr,
 					     $vmfw_conf, $vmid, $bridge, 'OUT');
 	    }
 	}
@@ -2235,11 +2341,11 @@ sub compile {
     ruleset_addrule($ruleset, "PVEFW-FORWARD", "-i venet0 -j RETURN");
 
     # disable interbridge routing
-    ruleset_addrule($ruleset, "PVEFW-FORWARD", "-o vmbr+ -j PVEFW-Drop"); 
+    ruleset_addrule($ruleset, "PVEFW-FORWARD", "-o vmbr+ -j PVEFW-Drop");
     ruleset_addrule($ruleset, "PVEFW-FORWARD", "-i vmbr+ -j PVEFW-Drop");
-    ruleset_addlog($ruleset, "PVEFW-FORWARD", 0, "DROP: ", $loglevel, "-o vmbr+");  
-    ruleset_addlog($ruleset, "PVEFW-FORWARD", 0, "DROP: ", $loglevel, "-i vmbr+");  
-    ruleset_addrule($ruleset, "PVEFW-FORWARD", "-o vmbr+ -j DROP");  
+    ruleset_addlog($ruleset, "PVEFW-FORWARD", 0, "DROP: ", $loglevel, "-o vmbr+");
+    ruleset_addlog($ruleset, "PVEFW-FORWARD", 0, "DROP: ", $loglevel, "-i vmbr+");
+    ruleset_addrule($ruleset, "PVEFW-FORWARD", "-o vmbr+ -j DROP");
     ruleset_addrule($ruleset, "PVEFW-FORWARD", "-i vmbr+ -j DROP");
 
     return wantarray ? ($ruleset, $hostfw_conf, $ipset_ruleset) : $ruleset;
@@ -2294,7 +2400,7 @@ sub get_ruleset_cmdlist {
     my ($ruleset, $verbose) = @_;
 
     my $cmdlist = "*filter\n"; # we pass this to iptables-restore;
- 
+
     my ($active_chains, $hooks) = iptables_get_chains();
     my $statushash = get_ruleset_status($ruleset, $active_chains, \&iptables_chain_digest, $verbose);
 
@@ -2373,7 +2479,7 @@ sub get_ipset_cmdlist {
 
 	if ($stat->{action} eq 'update') {
 	    my $chain_swap = $chain."_swap";
-	    
+
 	    foreach my $cmd (@{$ruleset->{$chain}}) {
 		$cmd =~ s/$chain/$chain_swap/;
 		$cmdlist .= "$cmd\n";
@@ -2393,7 +2499,7 @@ sub get_ipset_cmdlist {
     }
 
     my $changes = ($cmdlist || $delete_cmdlist) ? 1 : 0;
-    
+
     return ($cmdlist, $delete_cmdlist, $changes);
 }
 
@@ -2404,15 +2510,23 @@ sub apply_ruleset {
 
     update_nf_conntrack_max($hostfw_conf);
 
-    my ($ipset_create_cmdlist, $ipset_delete_cmdlist) = get_ipset_cmdlist($ipset_ruleset, undef, $verbose);
+    my ($ipset_create_cmdlist, $ipset_delete_cmdlist, $ipset_changes) =
+	get_ipset_cmdlist($ipset_ruleset, undef, $verbose);
 
-    my $cmdlist = get_ruleset_cmdlist($ruleset, $verbose);
+    my ($cmdlist, $changes) = get_ruleset_cmdlist($ruleset, $verbose);
 
-    print $ipset_create_cmdlist if $verbose;
-
-    print $ipset_delete_cmdlist if $verbose;
+    if ($verbose) {
+	if ($ipset_changes) {
+	    print "ipset changes:\n";
+	    print $ipset_create_cmdlist if $ipset_create_cmdlist;
+	    print $ipset_delete_cmdlist if $ipset_delete_cmdlist;
+	}
 
-    print $cmdlist if $verbose;
+	if ($changes) {
+	    print "iptables changes:\n";
+	    print $cmdlist;
+	}
+    }
 
     ipset_restore_cmdlist($ipset_create_cmdlist);
 
@@ -2422,7 +2536,7 @@ sub apply_ruleset {
 
     # test: re-read status and check if everything is up to date
     my $active_chains = iptables_get_chains();
-    my $statushash = get_ruleset_status($ruleset, $active_chains, \&iptables_chain_digest, $verbose);
+    my $statushash = get_ruleset_status($ruleset, $active_chains, \&iptables_chain_digest, 0);
 
     my $errors;
     foreach my $chain (sort keys %$ruleset) {
@@ -2470,7 +2584,7 @@ sub remove_pvefw_chains {
 	    $cmdlist .= "-D $h -j PVEFW-$h\n";
 	}
     }
- 
+
     foreach my $chain (keys %$chash) {
 	$cmdlist .= "-F $chain\n";
     }