Add crm command 'stop'

Not every command parameter is 'target' anymore, so
it was necessary to modify the parsing of $sd->{cmd}.

Just changing the state to request_stop is not enough,
we need to actually update the service configuration as well.

Add a simple test for the stop command

Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm
index 5137de8fc37eb8179e2538283ed3fd721a60b24a..617e3692c7e492a4b640847efd64d3fca6405060 100644 (file)
--- a/src/PVE/HA/Manager.pm
+++ b/src/PVE/HA/Manager.pm
@@ -349,6 +349,14 @@ sub update_crm_commands {
                $haenv->log('err', "crm command error - no such service: $cmd");
            }
 
+       } elsif ($cmd =~ m/^stop\s+(\S+)\s+(\S+)$/) {
+           my ($sid, $timeout) = ($1, $2);
+           if (my $sd = $ss->{$sid}) {
+               $haenv->log('info', "got crm command: $cmd");
+               $ss->{$sid}->{cmd} = [ 'stop', $timeout ];
+           } else {
+               $haenv->log('err', "crm command error - no such service: $cmd");
+           }
        } else {
            $haenv->log('err', "unable to parse crm command: $cmd");
        }
@@ -561,10 +569,10 @@ sub next_state_stopped {
     }
 
     if ($sd->{cmd}) {
-       my ($cmd, $target) = @{$sd->{cmd}};
-       delete $sd->{cmd};
+       my $cmd = shift @{$sd->{cmd}};
 
        if ($cmd eq 'migrate' || $cmd eq 'relocate') {
+           my $target = shift @{$sd->{cmd}};
            if (!$ns->node_is_online($target)) {
                $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
            } elsif ($sd->{node} eq $target) {
@@ -574,9 +582,12 @@ sub next_state_stopped {
                                       target => $target);
                return;
            }
+       } elsif ($cmd eq 'stop') {
+               $haenv->log('info', "ignore service '$sid' $cmd request - service already stopped");
        } else {
            $haenv->log('err', "unknown command '$cmd' for service '$sid'");
        }
+       delete $sd->{cmd};
     }
 
     if ($cd->{state} eq 'disabled') {
@@ -638,10 +649,10 @@ sub next_state_started {
     if ($cd->{state} eq 'started') {
 
        if ($sd->{cmd}) {
-           my ($cmd, $target) = @{$sd->{cmd}};
-           delete $sd->{cmd};
+           my $cmd = shift @{$sd->{cmd}};
 
            if ($cmd eq 'migrate' || $cmd eq 'relocate') {
+               my $target = shift @{$sd->{cmd}};
                if (!$ns->node_is_online($target)) {
                    $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
                } elsif ($sd->{node} eq $target) {
@@ -650,9 +661,17 @@ sub next_state_started {
                    $haenv->log('info', "$cmd service '$sid' to node '$target'");
                    &$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
                }
+           } elsif ($cmd eq 'stop') {
+               my $timeout = shift @{$sd->{cmd}};
+               $haenv->log('info', "$cmd service with timeout '$timeout'");
+               &$change_service_state($self, $sid, 'request_stop', timeout => $timeout);
+               $haenv->update_service_config($sid, {'state' => 'stopped'});
            } else {
                $haenv->log('err', "unknown command '$cmd' for service '$sid'");
            }
+
+           delete $sd->{cmd};
+
        } else {
 
            my $try_next = 0;

diff --git a/src/PVE/HA/Sim/Hardware.pm b/src/PVE/HA/Sim/Hardware.pm
index 3cdc85bd56ba91daa0f0f54f613bd0aa6cbba27e..121cd1b1cf3093f429701266c8422b149b63499f 100644 (file)
--- a/src/PVE/HA/Sim/Hardware.pm
+++ b/src/PVE/HA/Sim/Hardware.pm
@@ -541,6 +541,7 @@ sub get_cfs_state {
 # restart-lrm <node>
 # service <sid> <started|disabled|stopped|ignored>
 # service <sid> <migrate|relocate> <target>
+# service <sid> stop <timeout>
 # service <sid> lock/unlock [lockname]
 
 sub sim_hardware_cmd {
@@ -658,6 +659,13 @@ sub sim_hardware_cmd {
 
                $self->queue_crm_commands_nolock("$action $sid $param");
 
+           } elsif ($action eq 'stop') {
+
+               die "sim_hardware_cmd: missing timeout for '$action' command"
+                   if !defined($param);
+
+               $self->queue_crm_commands_nolock("$action $sid $param");
+
            } elsif ($action eq 'add') {
 
                $self->add_service($sid, {state => 'started', node => $param});

diff --git a/src/test/test-stop-command1/README b/src/test/test-stop-command1/README
new file mode 100644 (file)
index 0000000..d725380
--- /dev/null
+++ b/src/test/test-stop-command1/README
@@ -0,0 +1,2 @@
+Test the stop command with different timeouts
+and what happens with a failed service.

diff --git a/src/test/test-stop-command1/cmdlist b/src/test/test-stop-command1/cmdlist
new file mode 100644 (file)
index 0000000..c7c9e67
--- /dev/null
+++ b/src/test/test-stop-command1/cmdlist
@@ -0,0 +1,8 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on" ],
+    [ "service vm:101 stop 0" ],
+    [ "service vm:101 stop 1" ],
+    [ "service vm:102 stop 37" ],
+    [ "service vm:103 stop 60" ],
+    [ "service fa:1001 stop 0" ]
+]

diff --git a/src/test/test-stop-command1/hardware_status b/src/test/test-stop-command1/hardware_status
new file mode 100644 (file)
index 0000000..451beb1
--- /dev/null
+++ b/src/test/test-stop-command1/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}

diff --git a/src/test/test-stop-command1/log.expect b/src/test/test-stop-command1/log.expect
new file mode 100644 (file)
index 0000000..05f9712
--- /dev/null
+++ b/src/test/test-stop-command1/log.expect
@@ -0,0 +1,69 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'fa:1001' on node 'node3'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     23    node2/lrm: starting service vm:102
+info     23    node2/lrm: service status vm:102 started
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service fa:1001
+info     25    node3/lrm: service status fa:1001 started
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+info    120      cmdlist: execute service vm:101 stop 0
+info    120    node1/crm: got crm command: stop vm:101 0
+info    120    node1/crm: stop service with timeout '0'
+info    120    node1/crm: service 'vm:101': state changed from 'started' to 'request_stop'  (timeout = 0)
+info    121    node1/lrm: stopping service vm:101 (timeout=0)
+info    121    node1/lrm: service status vm:101 stopped
+info    140    node1/crm: service 'vm:101': state changed from 'request_stop' to 'stopped'
+info    220      cmdlist: execute service vm:101 stop 1
+info    220    node1/crm: got crm command: stop vm:101 1
+info    220    node1/crm: ignore service 'vm:101' stop request - service already stopped
+info    320      cmdlist: execute service vm:102 stop 37
+info    320    node1/crm: got crm command: stop vm:102 37
+info    320    node1/crm: stop service with timeout '37'
+info    320    node1/crm: service 'vm:102': state changed from 'started' to 'request_stop'  (timeout = 37)
+info    323    node2/lrm: stopping service vm:102 (timeout=37)
+info    323    node2/lrm: service status vm:102 stopped
+info    340    node1/crm: service 'vm:102': state changed from 'request_stop' to 'stopped'
+info    420      cmdlist: execute service vm:103 stop 60
+info    420    node1/crm: got crm command: stop vm:103 60
+info    420    node1/crm: stop service with timeout '60'
+info    420    node1/crm: service 'vm:103': state changed from 'started' to 'request_stop'  (timeout = 60)
+info    425    node3/lrm: stopping service vm:103 (timeout=60)
+info    425    node3/lrm: service status vm:103 stopped
+info    440    node1/crm: service 'vm:103': state changed from 'request_stop' to 'stopped'
+info    520      cmdlist: execute service fa:1001 stop 0
+info    520    node1/crm: got crm command: stop fa:1001 0
+info    520    node1/crm: stop service with timeout '0'
+info    520    node1/crm: service 'fa:1001': state changed from 'started' to 'request_stop'  (timeout = 0)
+info    525    node3/lrm: stopping service fa:1001 (timeout=0)
+info    525    node3/lrm: unable to stop stop service fa:1001 (still running)
+err     540    node1/crm: service 'fa:1001' stop failed (exit code 1)
+info    540    node1/crm: service 'fa:1001': state changed from 'request_stop' to 'error'
+err     545    node3/lrm: service fa:1001 is in an error state and needs manual intervention. Look up 'ERROR RECOVERY' in the documentation.
+info   1120     hardware: exit simulation - done

diff --git a/src/test/test-stop-command1/manager_status b/src/test/test-stop-command1/manager_status
new file mode 100644 (file)
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-stop-command1/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file

diff --git a/src/test/test-stop-command1/service_config b/src/test/test-stop-command1/service_config
new file mode 100644 (file)
index 0000000..4a126df
--- /dev/null
+++ b/src/test/test-stop-command1/service_config
@@ -0,0 +1,6 @@
+{
+    "vm:101": { "node": "node1", "state": "started" },
+    "vm:102": { "node": "node2", "state": "started" },
+    "vm:103": { "node": "node3", "state": "started" },
+    "fa:1001": { "node": "node3", "state": "started" }
+}