manage: handle edge case where a node gets stuck in 'fence' state

If all services in 'fence' state are gone from a node (e.g. by
removing the services) before fence_node() was successful, a node
would get stuck in the 'fence' state. Avoid this by calling
fence_node() if the node is in 'fence' state, regardless of service
state.

Reported in the community forum:
https://forum.proxmox.com/threads/ha-migration-stuck-is-doing-nothing.94469/

Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
[ T: track test change of new test ]
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm
index 11336135706746804aa56066475a2b4c1997903d..317c1566c9c4e758192614db68845511d84729a3 100644 (file)
--- a/src/PVE/HA/Manager.pm
+++ b/src/PVE/HA/Manager.pm
@@ -474,6 +474,15 @@ sub manage {
            $repeat = 1; # for faster recovery execution
        }
 
+       # Avoid that a node without services in 'fence' state gets stuck in 'fence' state.
+       for my $node (sort keys $ns->{status}->%*) {
+           next if $ns->get_node_state($node) ne 'fence';
+           next if defined($fenced_nodes->{$node});
+
+           $haenv->log('info', "fence of node '$node' without any service left");
+           $fenced_nodes->{$node} = $ns->fence_node($node) || 0;
+       }
+
        last if !$repeat;
     }
 

diff --git a/src/test/test-service-command8/log.expect b/src/test/test-service-command8/log.expect
index 72eb36923d51dbb8518e45c65c3648ed4f07d03d..ff0c17ca6d2518fefe100406aaa349384340f748 100644 (file)
--- a/src/test/test-service-command8/log.expect
+++ b/src/test/test-service-command8/log.expect
@@ -10,13 +10,25 @@ info     20    node3/crm: status change startup => wait_for_quorum
 info     20    node3/lrm: status change startup => wait_for_agent_lock
 info     20    node1/crm: got lock 'ha_manager_lock'
 info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: fence of node 'node3' without any service left
+info     20    node1/crm: got lock 'ha_agent_node3_lock'
+info     20    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
+info     20    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai     20    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
 info     21    node1/lrm: got lock 'ha_agent_node1_lock'
 info     21    node1/lrm: status change wait_for_agent_lock => active
 info     21    node1/lrm: starting service vm:101
 info     21    node1/lrm: service status vm:101 started
 info     22    node2/crm: status change wait_for_quorum => slave
 info     24    node3/crm: status change wait_for_quorum => slave
+info     40    node1/crm: node 'node3': state changed from 'unknown' => 'online'
 info    120      cmdlist: execute service vm:103 add node3 stopped
 info    120    node1/crm: adding new service 'vm:103' on node 'node3'
+info    145    node3/lrm: got lock 'ha_agent_node3_lock'
+info    145    node3/lrm: status change wait_for_agent_lock => active
+info    160    node1/crm: service 'vm:103': state changed from 'request_stop' to 'stopped'
 info    220      cmdlist: execute service vm:103 started
+info    220    node1/crm: service 'vm:103': state changed from 'stopped' to 'started'  (node = node3)
+info    225    node3/lrm: starting service vm:103
+info    225    node3/lrm: service status vm:103 started
 info    820     hardware: exit simulation - done

diff --git a/src/test/test-service-command9/log.expect b/src/test/test-service-command9/log.expect
index 40de86bb14e1ac5f5175ab383bb4488ba7f195b2..e27c50d3daf89befae3375e587fca9e1cf58f266 100644 (file)
--- a/src/test/test-service-command9/log.expect
+++ b/src/test/test-service-command9/log.expect
@@ -18,6 +18,16 @@ info     20    node1/lrm: service status vm:101 started
 info     40     run-loop: skipping CRM round
 info     60    node1/crm: got lock 'ha_manager_lock'
 info     60    node1/crm: status change wait_for_quorum => master
+info     60    node1/crm: fence of node 'node3' without any service left
+info     60    node1/crm: got lock 'ha_agent_node3_lock'
+info     60    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
+info     60    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai     60    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
 info     62    node2/crm: status change wait_for_quorum => slave
 info     64    node3/crm: status change wait_for_quorum => slave
+info     80    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info    185    node3/lrm: got lock 'ha_agent_node3_lock'
+info    185    node3/lrm: status change wait_for_agent_lock => active
+info    185    node3/lrm: starting service vm:103
+info    185    node3/lrm: service status vm:103 started
 info    620     hardware: exit simulation - done