manager: make recovery actual state in FSM

This basically makes recovery just an active state transition, as can
be seen from the regression tests - no other semantic change is
caused.

For the admin this is much better to grasp than services still marked
as "fence" when the failed node is already fenced or even already up
again.

Code-wise it makes sense too, to make the recovery part not so hidden
anymore, but show it was it is: an actual part of the FSM

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm
index 9b76354e2bf32056170d337415cd4774500fc710..d65fb885ba2a8661f94929d581774d632ae4a738 100644 (file)
--- a/src/PVE/HA/Manager.pm
+++ b/src/PVE/HA/Manager.pm
@@ -166,6 +166,7 @@ my $valid_service_states = {
     request_stop => 1,
     started => 1,
     fence => 1,
+    recovery => 1,
     migrate => 1,
     relocate => 1,
     freeze => 1,
@@ -187,8 +188,10 @@ sub recompute_online_node_usage {
        my $sd = $self->{ss}->{$sid};
        my $state = $sd->{state};
        if (defined($online_node_usage->{$sd->{node}})) {
-           if (($state eq 'started') || ($state eq 'request_stop') ||
-               ($state eq 'fence') || ($state eq 'freeze') || ($state eq 'error')) {
+           if (
+               $state eq 'started' || $state eq 'request_stop' || $state eq 'fence' ||
+               $state eq 'freeze' || $state eq 'error' || $state eq 'recovery'
+           ) {
                $online_node_usage->{$sd->{node}}++;
            } elsif (($state eq 'migrate') || ($state eq 'relocate')) {
                # count it for both, source and target as load is put on both
@@ -276,49 +279,6 @@ my $fence_recovery_cleanup = sub {
     }
 };
 
-# after a node was fenced this recovers the service to a new node
-my $recover_fenced_service = sub {
-    my ($self, $sid, $cd) = @_;
-
-    my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
-
-    my $sd = $ss->{$sid};
-
-    if ($sd->{state} ne 'fence') { # should not happen
-       $haenv->log('err', "cannot recover service '$sid' from fencing, wrong state '$sd->{state}'");
-       return;
-    }
-
-    my $fenced_node = $sd->{node}; # for logging purpose
-
-    $self->recompute_online_node_usage(); # we want the most current node state
-
-    my $recovery_node = select_service_node(
-       $self->{groups},
-       $self->{online_node_usage},
-       $cd,
-       $sd->{node},
-    );
-
-    if ($recovery_node) {
-       $haenv->log('info', "recover service '$sid' from fenced node '$fenced_node' to node '$recovery_node'");
-
-       &$fence_recovery_cleanup($self, $sid, $fenced_node);
-
-       $haenv->steal_service($sid, $sd->{node}, $recovery_node);
-       $self->{online_node_usage}->{$recovery_node}++;
-
-       # $sd *is normally read-only*, fencing is the exception
-       $cd->{node} = $sd->{node} = $recovery_node;
-       my $new_state = ($cd->{state} eq 'started') ? 'started' : 'request_stop';
-       &$change_service_state($self, $sid, $new_state, node => $recovery_node);
-    } else {
-       # no possible node found, cannot recover
-       $haenv->log('err', "recovering service '$sid' from fenced node '$fenced_node' failed, no recovery node found");
-       &$change_service_state($self, $sid, 'error');
-    }
-};
-
 # read LRM status for all nodes
 sub read_lrm_status {
     my ($self) = @_;
@@ -460,6 +420,10 @@ sub manage {
 
                # do nothing here - wait until fenced
 
+           } elsif ($last_state eq 'recovery') {
+
+               $self->next_state_recovery($sid, $cd, $sd, $lrm_res);
+
            } elsif ($last_state eq 'request_stop') {
 
                $self->next_state_request_stop($sid, $cd, $sd, $lrm_res);
@@ -505,7 +469,8 @@ sub manage {
            next if !$fenced_nodes->{$sd->{node}};
 
            # node fence was successful - recover service
-           &$recover_fenced_service($self, $sid, $sc->{$sid});
+           $change_service_state->($self, $sid, 'recovery');
+           $repeat = 1; # for faster execution
        }
 
        last if !$repeat;
@@ -821,4 +786,47 @@ sub next_state_error {
 
 }
 
+# after a node was fenced this recovers the service to a new node
+sub next_state_recovery {
+    my ($self, $sid, $cd, $sd, $lrm_res) = @_;
+
+    my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
+    my $ns = $self->{ns};
+    my $ms = $self->{ms};
+
+    if ($sd->{state} ne 'recovery') { # should not happen
+       $haenv->log('err', "cannot recover service '$sid' from fencing, wrong state '$sd->{state}'");
+       return;
+    }
+
+    my $fenced_node = $sd->{node}; # for logging purpose
+
+    $self->recompute_online_node_usage(); # we want the most current node state
+
+    my $recovery_node = select_service_node(
+       $self->{groups},
+       $self->{online_node_usage},
+       $cd,
+       $sd->{node},
+    );
+
+    if ($recovery_node) {
+       $haenv->log('info', "recover service '$sid' from fenced node '$fenced_node' to node '$recovery_node'");
+
+       $fence_recovery_cleanup->($self, $sid, $fenced_node);
+
+       $haenv->steal_service($sid, $sd->{node}, $recovery_node);
+       $self->{online_node_usage}->{$recovery_node}++;
+
+       # NOTE: $sd *is normally read-only*, fencing is the exception
+       $cd->{node} = $sd->{node} = $recovery_node;
+       my $new_state = ($cd->{state} eq 'started') ? 'started' : 'request_stop';
+       $change_service_state->($self, $sid, $new_state, node => $recovery_node);
+    } else {
+       # no possible node found, cannot recover - but retry later, as we always try to make it available
+       $haenv->log('err', "recovering service '$sid' from fenced node '$fenced_node' failed, no recovery node found");
+       $change_service_state->($self, $sid, 'error');
+    }
+}
+
 1;

diff --git a/src/test/test-basic1/log.expect b/src/test/test-basic1/log.expect
index 93c7dfb35149122d15c99859d94b36ffb6d0dca2..c61850dc1aceeda57dd910029a78dd43cbefd7cb 100644 (file)
--- a/src/test/test-basic1/log.expect
+++ b/src/test/test-basic1/log.expect
@@ -44,8 +44,9 @@ info    240    node1/crm: got lock 'ha_agent_node3_lock'
 info    240    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
 info    240    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
 emai    240    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info    240    node1/crm: service 'vm:103': state changed from 'fence' to 'recovery'
 info    240    node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node2'
-info    240    node1/crm: service 'vm:103': state changed from 'fence' to 'started'  (node = node2)
+info    240    node1/crm: service 'vm:103': state changed from 'recovery' to 'started'  (node = node2)
 info    243    node2/lrm: starting service vm:103
 info    243    node2/lrm: service status vm:103 started
 info    720     hardware: exit simulation - done

diff --git a/src/test/test-basic2/log.expect b/src/test/test-basic2/log.expect
index 8ccce20528a95d0e0fd89acbfca0e4a1b620ec73..d2e40ac7a5b55e7af6efcd8e9252f5bb997a7b73 100644 (file)
--- a/src/test/test-basic2/log.expect
+++ b/src/test/test-basic2/log.expect
@@ -12,8 +12,9 @@ info     22    node3/crm: got lock 'ha_agent_node1_lock'
 info     22    node3/crm: fencing: acknowledged - got agent lock for node 'node1'
 info     22    node3/crm: node 'node1': state changed from 'fence' => 'unknown'
 emai     22    node3/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1'
+info     22    node3/crm: service 'vm:101': state changed from 'fence' to 'recovery'
 info     22    node3/crm: recover service 'vm:101' from fenced node 'node1' to node 'node3'
-info     22    node3/crm: service 'vm:101': state changed from 'fence' to 'started'  (node = node3)
+info     22    node3/crm: service 'vm:101': state changed from 'recovery' to 'started'  (node = node3)
 info     23    node3/lrm: got lock 'ha_agent_node3_lock'
 info     23    node3/lrm: status change wait_for_agent_lock => active
 info     23    node3/lrm: starting service vm:101

diff --git a/src/test/test-basic5/log.expect b/src/test/test-basic5/log.expect
index 8274f44797286b66d094c0e199007b9cfb335bc7..4af7447e2173b7e26fb34b4fb5a575292d23f505 100644 (file)
--- a/src/test/test-basic5/log.expect
+++ b/src/test/test-basic5/log.expect
@@ -48,8 +48,9 @@ info    282    node3/crm: got lock 'ha_agent_node1_lock'
 info    282    node3/crm: fencing: acknowledged - got agent lock for node 'node1'
 info    282    node3/crm: node 'node1': state changed from 'fence' => 'unknown'
 emai    282    node3/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1'
+info    282    node3/crm: service 'vm:101': state changed from 'fence' to 'recovery'
 info    282    node3/crm: recover service 'vm:101' from fenced node 'node1' to node 'node2'
-info    282    node3/crm: service 'vm:101': state changed from 'fence' to 'started'  (node = node2)
+info    282    node3/crm: service 'vm:101': state changed from 'recovery' to 'started'  (node = node2)
 info    301    node2/lrm: starting service vm:101
 info    301    node2/lrm: service status vm:101 started
 info    500      cmdlist: execute power node1 on

diff --git a/src/test/test-locked-service1/log.expect b/src/test/test-locked-service1/log.expect
index ebd3805436f118ee574d07ec3415b3910d69dd4a..ca8b008b3882de3121eac1f6c5c1755258fc1c38 100644 (file)
--- a/src/test/test-locked-service1/log.expect
+++ b/src/test/test-locked-service1/log.expect
@@ -36,9 +36,10 @@ info    340    node1/crm: got lock 'ha_agent_node3_lock'
 info    340    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
 info    340    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
 emai    340    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info    340    node1/crm: service 'vm:103': state changed from 'fence' to 'recovery'
 info    340    node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node1'
 warn    340    node1/crm: removed leftover lock 'backup' from recovered service 'vm:103' to allow its start.
-info    340    node1/crm: service 'vm:103': state changed from 'fence' to 'started'  (node = node1)
+info    340    node1/crm: service 'vm:103': state changed from 'recovery' to 'started'  (node = node1)
 info    341    node1/lrm: got lock 'ha_agent_node1_lock'
 info    341    node1/lrm: status change wait_for_agent_lock => active
 info    341    node1/lrm: starting service vm:103

diff --git a/src/test/test-locked-service2/log.expect b/src/test/test-locked-service2/log.expect
index 8e8e1a51082a787e9be0c8a1566d156a01df9242..6d564ca0cac312b424e1527e466e4804f0d6ce18 100644 (file)
--- a/src/test/test-locked-service2/log.expect
+++ b/src/test/test-locked-service2/log.expect
@@ -36,8 +36,9 @@ info    340    node1/crm: got lock 'ha_agent_node3_lock'
 info    340    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
 info    340    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
 emai    340    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info    340    node1/crm: service 'vm:103': state changed from 'fence' to 'recovery'
 info    340    node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node1'
-info    340    node1/crm: service 'vm:103': state changed from 'fence' to 'started'  (node = node1)
+info    340    node1/crm: service 'vm:103': state changed from 'recovery' to 'started'  (node = node1)
 info    341    node1/lrm: got lock 'ha_agent_node1_lock'
 info    341    node1/lrm: status change wait_for_agent_lock => active
 info    341    node1/lrm: starting service vm:103

diff --git a/src/test/test-recovery1/log.expect b/src/test/test-recovery1/log.expect
index 62b7c90233397e2279163bbad413db27f7b59128..e4496b575f3f85f36ef4c9b97dcbbb28a32299ef 100644 (file)
--- a/src/test/test-recovery1/log.expect
+++ b/src/test/test-recovery1/log.expect
@@ -35,6 +35,7 @@ info    240    node1/crm: got lock 'ha_agent_node2_lock'
 info    240    node1/crm: fencing: acknowledged - got agent lock for node 'node2'
 info    240    node1/crm: node 'node2': state changed from 'fence' => 'unknown'
 emai    240    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node2'
+info    240    node1/crm: service 'vm:102': state changed from 'fence' to 'recovery'
 err     240    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
-info    240    node1/crm: service 'vm:102': state changed from 'fence' to 'error'
+info    240    node1/crm: service 'vm:102': state changed from 'recovery' to 'error'
 info    720     hardware: exit simulation - done

diff --git a/src/test/test-service-ignore2/log.expect b/src/test/test-service-ignore2/log.expect
index e089d719ae1358425c69beefcb533af221d62c70..b6f3eef2f9a05301ea357fa3cbb59b9543770096 100644 (file)
--- a/src/test/test-service-ignore2/log.expect
+++ b/src/test/test-service-ignore2/log.expect
@@ -39,8 +39,9 @@ info    340    node1/crm: got lock 'ha_agent_node3_lock'
 info    340    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
 info    340    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
 emai    340    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info    340    node1/crm: service 'vm:103': state changed from 'fence' to 'recovery'
 info    340    node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node1'
-info    340    node1/crm: service 'vm:103': state changed from 'fence' to 'started'  (node = node1)
+info    340    node1/crm: service 'vm:103': state changed from 'recovery' to 'started'  (node = node1)
 info    341    node1/lrm: got lock 'ha_agent_node1_lock'
 info    341    node1/lrm: status change wait_for_agent_lock => active
 info    341    node1/lrm: starting service vm:103

diff --git a/src/test/test-service-stopped2/log.expect b/src/test/test-service-stopped2/log.expect
index 585621dc8d5431c954ea21c8ec78f754d9a3b361..dde219ed04ab317a5fe0875be79c0563f6eebecc 100644 (file)
--- a/src/test/test-service-stopped2/log.expect
+++ b/src/test/test-service-stopped2/log.expect
@@ -35,8 +35,9 @@ info    340    node1/crm: got lock 'ha_agent_node3_lock'
 info    340    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
 info    340    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
 emai    340    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info    340    node1/crm: service 'fa:1501': state changed from 'fence' to 'recovery'
 info    340    node1/crm: recover service 'fa:1501' from fenced node 'node3' to node 'node1'
-info    340    node1/crm: service 'fa:1501': state changed from 'fence' to 'request_stop'  (node = node1)
+info    340    node1/crm: service 'fa:1501': state changed from 'recovery' to 'request_stop'  (node = node1)
 info    341    node1/lrm: got lock 'ha_agent_node1_lock'
 info    341    node1/lrm: status change wait_for_agent_lock => active
 info    360    node1/crm: service 'fa:1501': state changed from 'request_stop' to 'stopped'

diff --git a/src/test/test-shutdown-policy4/log.expect b/src/test/test-shutdown-policy4/log.expect
index 4afafd3979f99ee595523cb17364934d19f1d8d4..71c234c686ed1b8487c1129c695d98d317eae958 100644 (file)
--- a/src/test/test-shutdown-policy4/log.expect
+++ b/src/test/test-shutdown-policy4/log.expect
@@ -65,8 +65,9 @@ info    220    node1/crm: got lock 'ha_agent_node3_lock'
 info    220    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
 info    220    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
 emai    220    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info    220    node1/crm: service 'ct:105': state changed from 'fence' to 'recovery'
 info    220    node1/crm: recover service 'ct:105' from fenced node 'node3' to node 'node2'
-info    220    node1/crm: service 'ct:105': state changed from 'fence' to 'started'  (node = node2)
+info    220    node1/crm: service 'ct:105': state changed from 'recovery' to 'started'  (node = node2)
 info    223    node2/lrm: got lock 'ha_agent_node2_lock'
 info    223    node2/lrm: status change wait_for_agent_lock => active
 info    223    node2/lrm: starting service ct:105

diff --git a/src/test/test-shutdown1/log.expect b/src/test/test-shutdown1/log.expect
index e46f13f8e6e8cb1dbcc417b03292891fb539f000..6b5d896ff3f3872db83f87aafdf5b0a02adf2e68 100644 (file)
--- a/src/test/test-shutdown1/log.expect
+++ b/src/test/test-shutdown1/log.expect
@@ -38,8 +38,9 @@ info    200    node1/crm: got lock 'ha_agent_node3_lock'
 info    200    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
 info    200    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
 emai    200    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info    200    node1/crm: service 'vm:103': state changed from 'fence' to 'recovery'
 info    200    node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node1'
-info    200    node1/crm: service 'vm:103': state changed from 'fence' to 'started'  (node = node1)
+info    200    node1/crm: service 'vm:103': state changed from 'recovery' to 'started'  (node = node1)
 info    201    node1/lrm: got lock 'ha_agent_node1_lock'
 info    201    node1/lrm: status change wait_for_agent_lock => active
 info    201    node1/lrm: starting service vm:103

diff --git a/src/test/test-shutdown2/log.expect b/src/test/test-shutdown2/log.expect
index bf4c319bbca309025355640a44b3481a047ad3ad..9ef2c8dd7345e9109c84980d9f2b0b3cbb8435f7 100644 (file)
--- a/src/test/test-shutdown2/log.expect
+++ b/src/test/test-shutdown2/log.expect
@@ -38,8 +38,9 @@ info    200    node1/crm: got lock 'ha_agent_node3_lock'
 info    200    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
 info    200    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
 emai    200    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info    200    node1/crm: service 'vm:103': state changed from 'fence' to 'recovery'
 info    200    node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node1'
-info    200    node1/crm: service 'vm:103': state changed from 'fence' to 'started'  (node = node1)
+info    200    node1/crm: service 'vm:103': state changed from 'recovery' to 'started'  (node = node1)
 info    201    node1/lrm: got lock 'ha_agent_node1_lock'
 info    201    node1/lrm: status change wait_for_agent_lock => active
 info    201    node1/lrm: starting service vm:103

diff --git a/src/test/test-shutdown3/log.expect b/src/test/test-shutdown3/log.expect
index 571cb89022e055f4d209a66504d9c3df308ddb52..85a63b4cc73843762c544ab7feae48d01b51dfcb 100644 (file)
--- a/src/test/test-shutdown3/log.expect
+++ b/src/test/test-shutdown3/log.expect
@@ -38,8 +38,9 @@ info    200    node1/crm: got lock 'ha_agent_node3_lock'
 info    200    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
 info    200    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
 emai    200    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info    200    node1/crm: service 'ct:103': state changed from 'fence' to 'recovery'
 info    200    node1/crm: recover service 'ct:103' from fenced node 'node3' to node 'node1'
-info    200    node1/crm: service 'ct:103': state changed from 'fence' to 'started'  (node = node1)
+info    200    node1/crm: service 'ct:103': state changed from 'recovery' to 'started'  (node = node1)
 info    201    node1/lrm: got lock 'ha_agent_node1_lock'
 info    201    node1/lrm: status change wait_for_agent_lock => active
 info    201    node1/lrm: starting service ct:103

diff --git a/src/test/test-shutdown4/log.expect b/src/test/test-shutdown4/log.expect
index afebeff7761c582a18f3c66f8fcf7f1664bd6fbd..4aaadfb607ccdf86b565cb6e38a050828dc17676 100644 (file)
--- a/src/test/test-shutdown4/log.expect
+++ b/src/test/test-shutdown4/log.expect
@@ -41,8 +41,9 @@ info    220    node2/crm: got lock 'ha_agent_node1_lock'
 info    220    node2/crm: fencing: acknowledged - got agent lock for node 'node1'
 info    220    node2/crm: node 'node1': state changed from 'fence' => 'unknown'
 emai    220    node2/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1'
+info    220    node2/crm: service 'vm:100': state changed from 'fence' to 'recovery'
 info    220    node2/crm: recover service 'vm:100' from fenced node 'node1' to node 'node2'
-info    220    node2/crm: service 'vm:100': state changed from 'fence' to 'started'  (node = node2)
+info    220    node2/crm: service 'vm:100': state changed from 'recovery' to 'started'  (node = node2)
 info    221    node2/lrm: got lock 'ha_agent_node2_lock'
 info    221    node2/lrm: status change wait_for_agent_lock => active
 info    221    node2/lrm: starting service vm:100