manager: refactor fence processing and rework fence-but-no-service log

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm
index 317c1566c9c4e758192614db68845511d84729a3..2deea57769f9aafaf9e4a8ec62fbf940a51e9272 100644 (file)
--- a/src/PVE/HA/Manager.pm
+++ b/src/PVE/HA/Manager.pm
@@ -458,13 +458,11 @@ sub manage {
        # handle fencing
        my $fenced_nodes = {};
        foreach my $sid (sort keys %$ss) {
-           my $sd = $ss->{$sid};
-           next if $sd->{state} ne 'fence';
-
-           my $service_node = $sd->{node};
+           my ($service_state, $service_node) = $ss->{$sid}->@{'state', 'node'};
+           next if $service_state ne 'fence';
 
            if (!defined($fenced_nodes->{$service_node})) {
-               $fenced_nodes->{$service_node} = $ns->fence_node($sd->{node}) || 0;
+               $fenced_nodes->{$service_node} = $ns->fence_node($service_node) || 0;
            }
 
            next if !$fenced_nodes->{$service_node};
@@ -474,13 +472,13 @@ sub manage {
            $repeat = 1; # for faster recovery execution
        }
 
-       # Avoid that a node without services in 'fence' state gets stuck in 'fence' state.
-       for my $node (sort keys $ns->{status}->%*) {
+       # Avoid that a node without services in 'fence' state (e.g., removed
+       # manually by admin) is stuck with the 'fence' node state.
+       for my $node (sort grep { !defined($fenced_nodes->{$_}) } keys $ns->{status}->%*) {
            next if $ns->get_node_state($node) ne 'fence';
-           next if defined($fenced_nodes->{$node});
 
-           $haenv->log('info', "fence of node '$node' without any service left");
-           $fenced_nodes->{$node} = $ns->fence_node($node) || 0;
+           $haenv->log('notice', "node '$node' in fence state but no services to-fence! admin interference?!");
+           $repeat = 1 if $ns->fence_node($node);
        }
 
        last if !$repeat;

diff --git a/src/PVE/HA/Sim/TestEnv.pm b/src/PVE/HA/Sim/TestEnv.pm
index 6718d8c2ca7625cf4b009b4d251209726f147e6a..b448d72d7d01d97653251df4a755a7d5c710ce7f 100644 (file)
--- a/src/PVE/HA/Sim/TestEnv.pm
+++ b/src/PVE/HA/Sim/TestEnv.pm
@@ -104,7 +104,7 @@ sub loop_end_hook {
     my $delay = $self->{loop_delay};
     $self->{loop_delay} = 0;
 
-    die "loop take too long ($delay seconds)\n" if $delay > 30;
+    die "loop took too long ($delay seconds)\n" if $delay > 300;
 
     # $self->{cur_time} += $delay;
 

diff --git a/src/test/test-service-command8/log.expect b/src/test/test-service-command8/log.expect
index ff0c17ca6d2518fefe100406aaa349384340f748..b1a2e0e3027e4def54f05dd8469db114bf03a114 100644 (file)
--- a/src/test/test-service-command8/log.expect
+++ b/src/test/test-service-command8/log.expect
@@ -10,7 +10,7 @@ info     20    node3/crm: status change startup => wait_for_quorum
 info     20    node3/lrm: status change startup => wait_for_agent_lock
 info     20    node1/crm: got lock 'ha_manager_lock'
 info     20    node1/crm: status change wait_for_quorum => master
-info     20    node1/crm: fence of node 'node3' without any service left
+noti     20    node1/crm: node 'node3' in fence state but no services to-fence! admin interference?!
 info     20    node1/crm: got lock 'ha_agent_node3_lock'
 info     20    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
 info     20    node1/crm: node 'node3': state changed from 'fence' => 'unknown'

diff --git a/src/test/test-service-command9/log.expect b/src/test/test-service-command9/log.expect
index e27c50d3daf89befae3375e587fca9e1cf58f266..ea7d1ccd615c737e74b627ed11fd5649e4adc0a8 100644 (file)
--- a/src/test/test-service-command9/log.expect
+++ b/src/test/test-service-command9/log.expect
@@ -18,7 +18,7 @@ info     20    node1/lrm: service status vm:101 started
 info     40     run-loop: skipping CRM round
 info     60    node1/crm: got lock 'ha_manager_lock'
 info     60    node1/crm: status change wait_for_quorum => master
-info     60    node1/crm: fence of node 'node3' without any service left
+noti     60    node1/crm: node 'node3' in fence state but no services to-fence! admin interference?!
 info     60    node1/crm: got lock 'ha_agent_node3_lock'
 info     60    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
 info     60    node1/crm: node 'node3': state changed from 'fence' => 'unknown'