tests: cover request-state changes and crm-cmds for in-recovery services

Add a test which covers that the recovery state does not allows
things like stop or migrate to happen.

Also add one for disabling at the end, this is currently blocked too
but will change in the next patch, as it can be a safe way out for
the admin to reset the service without removing it.

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

diff --git a/src/test/test-recovery4/README b/src/test/test-recovery4/README
new file mode 100644 (file)
index 0000000..98c248b
--- /dev/null
+++ b/src/test/test-recovery4/README
@@ -0,0 +1,7 @@
+Test what happens if a service needs to get recovered but select_service_node
+cannot return any possible node, and then the user triggers some CRM comand
+like `migration`, `stop` and finally a `disable`
+
+Expected Result:
+Try recovery continuosly, ignore the migrate and stop command, we cannot honor
+them for a in `recovery` service. For now also the `disabled` one is a no-op.

diff --git a/src/test/test-recovery4/cmdlist b/src/test/test-recovery4/cmdlist
new file mode 100644 (file)
index 0000000..8273d21
--- /dev/null
+++ b/src/test/test-recovery4/cmdlist
@@ -0,0 +1,8 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "network node2 off" ],
+    [ "delay 0" ],
+    [ "service vm:102 migrate node3" ],
+    [ "service vm:102 stop 5" ],
+    [ "service vm:102 disabled" ]
+]

diff --git a/src/test/test-recovery4/groups b/src/test/test-recovery4/groups
new file mode 100644 (file)
index 0000000..06c7f76
--- /dev/null
+++ b/src/test/test-recovery4/groups
@@ -0,0 +1,4 @@
+group: prefer_node2
+       nodes node2
+       restricted 1
+

diff --git a/src/test/test-recovery4/hardware_status b/src/test/test-recovery4/hardware_status
new file mode 100644 (file)
index 0000000..451beb1
--- /dev/null
+++ b/src/test/test-recovery4/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}

diff --git a/src/test/test-recovery4/log.expect b/src/test/test-recovery4/log.expect
new file mode 100644 (file)
index 0000000..599acf6
--- /dev/null
+++ b/src/test/test-recovery4/log.expect
@@ -0,0 +1,88 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     23    node2/lrm: starting service vm:102
+info     23    node2/lrm: service status vm:102 started
+info     24    node3/crm: status change wait_for_quorum => slave
+info    120      cmdlist: execute network node2 off
+info    120    node1/crm: node 'node2': state changed from 'online' => 'unknown'
+info    122    node2/crm: status change slave => wait_for_quorum
+info    123    node2/lrm: status change active => lost_agent_lock
+info    160    node1/crm: service 'vm:102': state changed from 'started' to 'fence'
+info    160    node1/crm: node 'node2': state changed from 'unknown' => 'fence'
+emai    160    node1/crm: FENCE: Try to fence node 'node2'
+info    164     watchdog: execute power node2 off
+info    163    node2/crm: killed by poweroff
+info    164    node2/lrm: killed by poweroff
+info    164     hardware: server 'node2' stopped by poweroff (watchdog)
+info    240    node1/crm: got lock 'ha_agent_node2_lock'
+info    240    node1/crm: fencing: acknowledged - got agent lock for node 'node2'
+info    240    node1/crm: node 'node2': state changed from 'fence' => 'unknown'
+emai    240    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node2'
+info    240    node1/crm: service 'vm:102': state changed from 'fence' to 'recovery'
+err     240    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     260    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     280    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     300    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+info    320      cmdlist: execute service vm:102 migrate node3
+info    320    node1/crm: got crm command: migrate vm:102 node3
+err     320    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     340    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     360    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     380    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     400    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+info    420      cmdlist: execute service vm:102 stop 5
+info    420    node1/crm: got crm command: stop vm:102 5
+err     420    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     440    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     460    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     480    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     500    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+info    520      cmdlist: execute service vm:102 disabled
+err     520    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     540    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     560    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     580    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     600    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     620    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     640    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     660    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     680    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     700    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     720    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     740    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     760    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     780    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     800    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     820    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     840    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     860    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     880    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     900    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     920    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     940    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     960    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err     980    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err    1000    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err    1020    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err    1040    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err    1060    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err    1080    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+err    1100    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
+info   1120     hardware: exit simulation - done

diff --git a/src/test/test-recovery4/manager_status b/src/test/test-recovery4/manager_status
new file mode 100644 (file)
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-recovery4/manager_status
@@ -0,0 +1 @@
+{}

diff --git a/src/test/test-recovery4/service_config b/src/test/test-recovery4/service_config
new file mode 100644 (file)
index 0000000..39a05e5
--- /dev/null
+++ b/src/test/test-recovery4/service_config
@@ -0,0 +1,3 @@
+{
+    "vm:102": { "node": "node2", "state": "enabled", "group": "prefer_node2" }
+}