manager: add new intermediate state for stop->start transitions

We always check for re-starting a service if its in the started
state, but for those that go from a (request_)stop to the stopped
state it can be useful to explicitly have a separate transition.

The newly introduced `request_start` state can also be used for CRS
to opt-into starting a service up on a load-wise better suited node
in the future.

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

diff --git a/src/PVE/HA/LRM.pm b/src/PVE/HA/LRM.pm
index 5d2fa2c21656e787275e5d780e1e1047ada9ec76..828ba97bff1d97ad65ddcc3cea705f6a2f9711bb 100644 (file)
--- a/src/PVE/HA/LRM.pm
+++ b/src/PVE/HA/LRM.pm
@@ -273,6 +273,8 @@ sub active_service_count {
        next if $req_state eq 'freeze';
        # erroneous services are not managed by HA, don't count them as active
        next if $req_state eq 'error';
+       # request_start is for (optional) better node selection for stop -> started transition
+       next if $req_state eq 'request_start';
 
        $count++;
     }
@@ -568,6 +570,7 @@ sub run_workers {
            # higher try-count means higher priority especially compared to newly queued jobs, so
            # count every try to avoid starvation
            $w->{start_tries}++;
+           # FIXME: should be last and ensure that check_active_workers is called sooner
            next if $count >= $max_workers && $max_workers > 0;
 
            # only fork if we may, else call exec_resource_agent directly (e.g. for tests)
@@ -642,6 +645,8 @@ sub manage_resources {
        # we can just continue normally. But we must NOT do anything with it while still in recovery
        next if $request_state eq 'recovery';
        next if $request_state eq 'freeze';
+       # intermediate step for optional better node selection on stop -> start request state change
+       next if $request_state eq 'request_start';
 
        $self->queue_resource_command($sid, $sd->{uid}, $request_state, {
            'target' => $sd->{target},

diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm
index cf9bdbd459970a87eb60a5b405981273cba194af..fde71b447facde491a2d6fa3f23a00df51c103e5 100644 (file)
--- a/src/PVE/HA/Manager.pm
+++ b/src/PVE/HA/Manager.pm
@@ -209,6 +209,7 @@ sub compute_new_uuid {
 my $valid_service_states = {
     stopped => 1,
     request_stop => 1,
+    request_start => 1,
     started => 1,
     fence => 1,
     recovery => 1,
@@ -257,8 +258,8 @@ sub recompute_online_node_usage {
        my $target = $sd->{target}; # optional
        if ($online_node_usage->contains_node($sd->{node})) {
            if (
-               $state eq 'started' || $state eq 'request_stop' || $state eq 'fence' ||
-               $state eq 'freeze' || $state eq 'error' || $state eq 'recovery'
+               $state eq 'started' || $state eq 'request_stop' || $state eq 'fence'
+               || $state eq 'freeze' || $state eq 'error' || $state eq 'recovery'
            ) {
                $online_node_usage->add_service_usage_to_node($sd->{node}, $sid, $sd->{node});
            } elsif (($state eq 'migrate') || ($state eq 'relocate')) {
@@ -266,7 +267,7 @@ sub recompute_online_node_usage {
                # count it for both, source and target as load is put on both
                $online_node_usage->add_service_usage_to_node($source, $sid, $source, $target);
                $online_node_usage->add_service_usage_to_node($target, $sid, $source, $target);
-           } elsif ($state eq 'stopped') {
+           } elsif ($state eq 'stopped' || $state eq 'request_start') {
                # do nothing
            } else {
                die "should not be reached (sid = '$sid', state = '$state')";
@@ -490,6 +491,10 @@ sub manage {
 
                $self->next_state_started($sid, $cd, $sd, $lrm_res);
 
+           } elsif ($last_state eq 'request_start') {
+
+               $self->next_state_request_start($sid, $cd, $sd, $lrm_res);
+
            } elsif ($last_state eq 'migrate' || $last_state eq 'relocate') {
 
                $self->next_state_migrate_relocate($sid, $cd, $sd, $lrm_res);
@@ -651,7 +656,7 @@ sub next_state_stopped {
            } elsif ($sd->{node} eq $target) {
                $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
            } else {
-               $change_service_state->($self, $sid, $cmd, node => $sd->{node}, target => $target);
+               &$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
                return;
            }
        } elsif ($cmd eq 'stop') {
@@ -680,15 +685,20 @@ sub next_state_stopped {
     }
 
     if ($cd->{state} eq 'started') {
-       # simply mark it started, if it's on the wrong node
-       # next_state_started will fix that for us
-       &$change_service_state($self, $sid, 'started', node => $sd->{node});
+       # simply mark it started, if it's on the wrong node next_state_started will fix that for us
+       $change_service_state->($self, $sid, 'request_start', node => $sd->{node});
        return;
     }
 
     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 }
 
+sub next_state_request_start {
+    my ($self, $sid, $cd, $sd, $lrm_res) = @_;
+
+    $change_service_state->($self, $sid, 'started', node => $sd->{node});
+}
+
 sub record_service_failed_on_node {
     my ($self, $sid, $node) = @_;
 

diff --git a/src/test/test-relocate-policy-default-group/log.expect b/src/test/test-relocate-policy-default-group/log.expect
index e0ae1ab5291cd324acd29a1b3aac784dfcc6040f..83366dd85222f0abdd5e619d8e82c6d1357f1d18 100644 (file)
--- a/src/test/test-relocate-policy-default-group/log.expect
+++ b/src/test/test-relocate-policy-default-group/log.expect
@@ -20,7 +20,8 @@ info     23    node2/lrm: status change wait_for_agent_lock => active
 info     24    node3/crm: status change wait_for_quorum => slave
 info     40    node1/crm: service 'fa:130': state changed from 'request_stop' to 'stopped'
 info    120      cmdlist: execute service fa:130 started
-info    120    node1/crm: service 'fa:130': state changed from 'stopped' to 'started'  (node = node2)
+info    120    node1/crm: service 'fa:130': state changed from 'stopped' to 'request_start'  (node = node2)
+info    120    node1/crm: service 'fa:130': state changed from 'request_start' to 'started'  (node = node2)
 info    123    node2/lrm: starting service fa:130
 warn    123    node2/lrm: unable to start service fa:130
 err     123    node2/lrm: unable to start service fa:130 on local node after 0 retries

diff --git a/src/test/test-relocate-policy1/log.expect b/src/test/test-relocate-policy1/log.expect
index e1c9f01783a40a3f08f4ea93acc0ebba015b8009..0975ffdb8612cb55ebc62d7e008d2e9579829054 100644 (file)
--- a/src/test/test-relocate-policy1/log.expect
+++ b/src/test/test-relocate-policy1/log.expect
@@ -33,7 +33,8 @@ info     25    node3/lrm: starting service vm:105
 info     25    node3/lrm: service status vm:105 started
 info     40    node1/crm: service 'fa:130': state changed from 'request_stop' to 'stopped'
 info    120      cmdlist: execute service fa:130 started
-info    120    node1/crm: service 'fa:130': state changed from 'stopped' to 'started'  (node = node3)
+info    120    node1/crm: service 'fa:130': state changed from 'stopped' to 'request_start'  (node = node3)
+info    120    node1/crm: service 'fa:130': state changed from 'request_start' to 'started'  (node = node3)
 info    125    node3/lrm: starting service fa:130
 warn    125    node3/lrm: unable to start service fa:130
 err     125    node3/lrm: unable to start service fa:130 on local node after 0 retries

diff --git a/src/test/test-resource-failure1/log.expect b/src/test/test-resource-failure1/log.expect
index 129d66e6c702b698a04a6de971fc3cb127991e4d..84e235c0e865e9559275c6b2d5531c37194fa84e 100644 (file)
--- a/src/test/test-resource-failure1/log.expect
+++ b/src/test/test-resource-failure1/log.expect
@@ -20,7 +20,8 @@ info     23    node2/lrm: status change wait_for_agent_lock => active
 info     24    node3/crm: status change wait_for_quorum => slave
 info     40    node1/crm: service 'fa:110': state changed from 'request_stop' to 'stopped'
 info    120      cmdlist: execute service fa:110 started
-info    120    node1/crm: service 'fa:110': state changed from 'stopped' to 'started'  (node = node2)
+info    120    node1/crm: service 'fa:110': state changed from 'stopped' to 'request_start'  (node = node2)
+info    120    node1/crm: service 'fa:110': state changed from 'request_start' to 'started'  (node = node2)
 info    123    node2/lrm: starting service fa:110
 warn    123    node2/lrm: unable to start service fa:110
 warn    123    node2/lrm: restart policy: retry number 1 for service 'fa:110'

diff --git a/src/test/test-resource-failure2/log.expect b/src/test/test-resource-failure2/log.expect
index b672dbbee6b36d277304219fe805c16127fba6f1..070cb0192d52d305035d2d2058bc6dbec6967c16 100644 (file)
--- a/src/test/test-resource-failure2/log.expect
+++ b/src/test/test-resource-failure2/log.expect
@@ -20,7 +20,8 @@ info     23    node2/lrm: status change wait_for_agent_lock => active
 info     24    node3/crm: status change wait_for_quorum => slave
 info     40    node1/crm: service 'fa:130': state changed from 'request_stop' to 'stopped'
 info    120      cmdlist: execute service fa:130 started
-info    120    node1/crm: service 'fa:130': state changed from 'stopped' to 'started'  (node = node2)
+info    120    node1/crm: service 'fa:130': state changed from 'stopped' to 'request_start'  (node = node2)
+info    120    node1/crm: service 'fa:130': state changed from 'request_start' to 'started'  (node = node2)
 info    123    node2/lrm: starting service fa:130
 warn    123    node2/lrm: unable to start service fa:130
 warn    123    node2/lrm: restart policy: retry number 1 for service 'fa:130'

diff --git a/src/test/test-resource-failure5/log.expect b/src/test/test-resource-failure5/log.expect
index 2fbd5ef953b532d09d7608694427087de7dc6e7e..03cb61af2106db633f01277a1f21c2dc988fd1db 100644 (file)
--- a/src/test/test-resource-failure5/log.expect
+++ b/src/test/test-resource-failure5/log.expect
@@ -20,7 +20,8 @@ info     23    node2/lrm: status change wait_for_agent_lock => active
 info     24    node3/crm: status change wait_for_quorum => slave
 info     40    node1/crm: service 'fa:130': state changed from 'request_stop' to 'stopped'
 info    120      cmdlist: execute service fa:130 started
-info    120    node1/crm: service 'fa:130': state changed from 'stopped' to 'started'  (node = node2)
+info    120    node1/crm: service 'fa:130': state changed from 'stopped' to 'request_start'  (node = node2)
+info    120    node1/crm: service 'fa:130': state changed from 'request_start' to 'started'  (node = node2)
 info    123    node2/lrm: starting service fa:130
 warn    123    node2/lrm: unable to start service fa:130
 warn    123    node2/lrm: restart policy: retry number 1 for service 'fa:130'

diff --git a/src/test/test-resource-failure6/log.expect b/src/test/test-resource-failure6/log.expect
index 61992cba01ca9e8a3fd8d39cc0ed1a6f984a8fea..35a13b654e7c2267a9d7ba735c4c1de060ff4184 100644 (file)
--- a/src/test/test-resource-failure6/log.expect
+++ b/src/test/test-resource-failure6/log.expect
@@ -20,7 +20,8 @@ info     23    node2/lrm: status change wait_for_agent_lock => active
 info     24    node3/crm: status change wait_for_quorum => slave
 info     40    node1/crm: service 'fa:130': state changed from 'request_stop' to 'stopped'
 info    120      cmdlist: execute service fa:130 started
-info    120    node1/crm: service 'fa:130': state changed from 'stopped' to 'started'  (node = node2)
+info    120    node1/crm: service 'fa:130': state changed from 'stopped' to 'request_start'  (node = node2)
+info    120    node1/crm: service 'fa:130': state changed from 'request_start' to 'started'  (node = node2)
 info    123    node2/lrm: starting service fa:130
 warn    123    node2/lrm: unable to start service fa:130
 err     123    node2/lrm: unable to start service fa:130 on local node after 0 retries

diff --git a/src/test/test-service-command1/log.expect b/src/test/test-service-command1/log.expect
index 02ca900a154c631fea92b68a04d1b81baf24a0fb..bfd18d686061d7a97f93bcdb97243b78ecec2d4d 100644 (file)
--- a/src/test/test-service-command1/log.expect
+++ b/src/test/test-service-command1/log.expect
@@ -26,7 +26,8 @@ info    125    node3/lrm: stopping service vm:103
 info    125    node3/lrm: service status vm:103 stopped
 info    140    node1/crm: service 'vm:103': state changed from 'request_stop' to 'stopped'
 info    220      cmdlist: execute service vm:103 started
-info    220    node1/crm: service 'vm:103': state changed from 'stopped' to 'started'  (node = node3)
+info    220    node1/crm: service 'vm:103': state changed from 'stopped' to 'request_start'  (node = node3)
+info    220    node1/crm: service 'vm:103': state changed from 'request_start' to 'started'  (node = node3)
 info    225    node3/lrm: starting service vm:103
 info    225    node3/lrm: service status vm:103 started
 info    820     hardware: exit simulation - done

diff --git a/src/test/test-service-command8/log.expect b/src/test/test-service-command8/log.expect
index b1a2e0e3027e4def54f05dd8469db114bf03a114..1afcd07ee49fa08c78c61114a604a28b4a80139a 100644 (file)
--- a/src/test/test-service-command8/log.expect
+++ b/src/test/test-service-command8/log.expect
@@ -28,7 +28,8 @@ info    145    node3/lrm: got lock 'ha_agent_node3_lock'
 info    145    node3/lrm: status change wait_for_agent_lock => active
 info    160    node1/crm: service 'vm:103': state changed from 'request_stop' to 'stopped'
 info    220      cmdlist: execute service vm:103 started
-info    220    node1/crm: service 'vm:103': state changed from 'stopped' to 'started'  (node = node3)
+info    220    node1/crm: service 'vm:103': state changed from 'stopped' to 'request_start'  (node = node3)
+info    220    node1/crm: service 'vm:103': state changed from 'request_start' to 'started'  (node = node3)
 info    225    node3/lrm: starting service vm:103
 info    225    node3/lrm: service status vm:103 started
 info    820     hardware: exit simulation - done

diff --git a/src/test/test-shutdown-policy4/log.expect b/src/test/test-shutdown-policy4/log.expect
index a6a9055c39192bff84e909b0bea11d2f9394a1a6..021306375fdcaac026d3992f7b93a93def8c8645 100644 (file)
--- a/src/test/test-shutdown-policy4/log.expect
+++ b/src/test/test-shutdown-policy4/log.expect
@@ -56,7 +56,8 @@ info    145    node3/crm: server received shutdown request
 info    165    node3/crm: exit (loop end)
 info    165     shutdown: execute power node3 off
 info    220      cmdlist: execute service ct:105 started
-info    220    node1/crm: service 'ct:105': state changed from 'stopped' to 'started'  (node = node3)
+info    220    node1/crm: service 'ct:105': state changed from 'stopped' to 'request_start'  (node = node3)
+info    220    node1/crm: service 'ct:105': state changed from 'request_start' to 'started'  (node = node3)
 info    220    node1/crm: service 'ct:105': state changed from 'started' to 'fence'
 info    220    node1/crm: relocate service 'ct:105' to node 'node1'
 info    220    node1/crm: service 'ct:105': state changed from 'fence' to 'relocate'  (node = node3, target = node1)