From: Fiona Ebner Date: Thu, 17 Nov 2022 14:00:15 +0000 (+0100) Subject: test: add tests for static resource scheduling X-Git-Url: https://git.proxmox.com/?p=pve-ha-manager.git;a=commitdiff_plain;h=f348399fe4a8e48172ae7a5899dfb610fc9adb8e test: add tests for static resource scheduling See the READMEs for more information about the tests. Signed-off-by: Fiona Ebner --- diff --git a/src/test/test-crs-static1/README b/src/test/test-crs-static1/README new file mode 100644 index 0000000..483f265 --- /dev/null +++ b/src/test/test-crs-static1/README @@ -0,0 +1,4 @@ +Test how service recovery works with the 'static' resource scheduling mode. + +Expect that the single service gets recovered to the node with the most +available resources. diff --git a/src/test/test-crs-static1/cmdlist b/src/test/test-crs-static1/cmdlist new file mode 100644 index 0000000..8684073 --- /dev/null +++ b/src/test/test-crs-static1/cmdlist @@ -0,0 +1,4 @@ +[ + [ "power node1 on", "power node2 on", "power node3 on"], + [ "network node1 off" ] +] diff --git a/src/test/test-crs-static1/datacenter.cfg b/src/test/test-crs-static1/datacenter.cfg new file mode 100644 index 0000000..8f83457 --- /dev/null +++ b/src/test/test-crs-static1/datacenter.cfg @@ -0,0 +1,6 @@ +{ + "crs": { + "ha": "static" + } +} + diff --git a/src/test/test-crs-static1/hardware_status b/src/test/test-crs-static1/hardware_status new file mode 100644 index 0000000..0fa8c26 --- /dev/null +++ b/src/test/test-crs-static1/hardware_status @@ -0,0 +1,5 @@ +{ + "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 }, + "node2": { "power": "off", "network": "off", "cpus": 32, "memory": 200000000000 }, + "node3": { "power": "off", "network": "off", "cpus": 32, "memory": 300000000000 } +} diff --git a/src/test/test-crs-static1/log.expect b/src/test/test-crs-static1/log.expect new file mode 100644 index 0000000..2b06b3c --- /dev/null +++ b/src/test/test-crs-static1/log.expect @@ -0,0 +1,50 @@ +info 0 hardware: starting simulation +info 20 cmdlist: execute power node1 on +info 20 node1/crm: status change startup => wait_for_quorum +info 20 node1/lrm: status change startup => wait_for_agent_lock +info 20 cmdlist: execute power node2 on +info 20 node2/crm: status change startup => wait_for_quorum +info 20 node2/lrm: status change startup => wait_for_agent_lock +info 20 cmdlist: execute power node3 on +info 20 node3/crm: status change startup => wait_for_quorum +info 20 node3/lrm: status change startup => wait_for_agent_lock +info 20 node1/crm: got lock 'ha_manager_lock' +info 20 node1/crm: status change wait_for_quorum => master +info 20 node1/crm: using scheduler mode 'static' +info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online' +info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online' +info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online' +info 20 node1/crm: adding new service 'vm:102' on node 'node1' +info 21 node1/lrm: got lock 'ha_agent_node1_lock' +info 21 node1/lrm: status change wait_for_agent_lock => active +info 21 node1/lrm: starting service vm:102 +info 21 node1/lrm: service status vm:102 started +info 22 node2/crm: status change wait_for_quorum => slave +info 24 node3/crm: status change wait_for_quorum => slave +info 120 cmdlist: execute network node1 off +info 120 node1/crm: status change master => lost_manager_lock +info 120 node1/crm: status change lost_manager_lock => wait_for_quorum +info 121 node1/lrm: status change active => lost_agent_lock +info 162 watchdog: execute power node1 off +info 161 node1/crm: killed by poweroff +info 162 node1/lrm: killed by poweroff +info 162 hardware: server 'node1' stopped by poweroff (watchdog) +info 222 node3/crm: got lock 'ha_manager_lock' +info 222 node3/crm: status change slave => master +info 222 node3/crm: using scheduler mode 'static' +info 222 node3/crm: node 'node1': state changed from 'online' => 'unknown' +info 282 node3/crm: service 'vm:102': state changed from 'started' to 'fence' +info 282 node3/crm: node 'node1': state changed from 'unknown' => 'fence' +emai 282 node3/crm: FENCE: Try to fence node 'node1' +info 282 node3/crm: got lock 'ha_agent_node1_lock' +info 282 node3/crm: fencing: acknowledged - got agent lock for node 'node1' +info 282 node3/crm: node 'node1': state changed from 'fence' => 'unknown' +emai 282 node3/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1' +info 282 node3/crm: service 'vm:102': state changed from 'fence' to 'recovery' +info 282 node3/crm: recover service 'vm:102' from fenced node 'node1' to node 'node3' +info 282 node3/crm: service 'vm:102': state changed from 'recovery' to 'started' (node = node3) +info 283 node3/lrm: got lock 'ha_agent_node3_lock' +info 283 node3/lrm: status change wait_for_agent_lock => active +info 283 node3/lrm: starting service vm:102 +info 283 node3/lrm: service status vm:102 started +info 720 hardware: exit simulation - done diff --git a/src/test/test-crs-static1/manager_status b/src/test/test-crs-static1/manager_status new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/src/test/test-crs-static1/manager_status @@ -0,0 +1 @@ +{} diff --git a/src/test/test-crs-static1/service_config b/src/test/test-crs-static1/service_config new file mode 100644 index 0000000..9c12447 --- /dev/null +++ b/src/test/test-crs-static1/service_config @@ -0,0 +1,3 @@ +{ + "vm:102": { "node": "node1", "state": "enabled" } +} diff --git a/src/test/test-crs-static1/static_service_stats b/src/test/test-crs-static1/static_service_stats new file mode 100644 index 0000000..7fb992d --- /dev/null +++ b/src/test/test-crs-static1/static_service_stats @@ -0,0 +1,3 @@ +{ + "vm:102": { "maxcpu": 2, "maxmem": 4000000000 } +} diff --git a/src/test/test-crs-static2/README b/src/test/test-crs-static2/README new file mode 100644 index 0000000..61530a7 --- /dev/null +++ b/src/test/test-crs-static2/README @@ -0,0 +1,4 @@ +Test how service recovery works with the 'static' resource scheduling mode. + +Expect that the single service always gets recovered to the node with the most +available resources. Also tests that the group priority still takes precedence. diff --git a/src/test/test-crs-static2/cmdlist b/src/test/test-crs-static2/cmdlist new file mode 100644 index 0000000..bada1bb --- /dev/null +++ b/src/test/test-crs-static2/cmdlist @@ -0,0 +1,20 @@ +[ + [ "power node1 on", "power node2 on", "power node3 on", "power node5 on" ], + [ "power node1 off" ], + [ "delay 300" ], + [ "power node1 on" ], + [ "delay 300" ], + [ "power node4 on" ], + [ "power node1 off" ], + [ "delay 300" ], + [ "power node1 on" ], + [ "delay 300" ], + [ "power node2 off" ], + [ "power node1 off" ], + [ "delay 300" ], + [ "power node1 on" ], + [ "delay 300" ], + [ "power node2 on" ], + [ "power node3 off" ], + [ "power node1 off" ] +] diff --git a/src/test/test-crs-static2/datacenter.cfg b/src/test/test-crs-static2/datacenter.cfg new file mode 100644 index 0000000..8f83457 --- /dev/null +++ b/src/test/test-crs-static2/datacenter.cfg @@ -0,0 +1,6 @@ +{ + "crs": { + "ha": "static" + } +} + diff --git a/src/test/test-crs-static2/groups b/src/test/test-crs-static2/groups new file mode 100644 index 0000000..43e9bf5 --- /dev/null +++ b/src/test/test-crs-static2/groups @@ -0,0 +1,2 @@ +group: prefer_node1 + nodes node1 diff --git a/src/test/test-crs-static2/hardware_status b/src/test/test-crs-static2/hardware_status new file mode 100644 index 0000000..d426023 --- /dev/null +++ b/src/test/test-crs-static2/hardware_status @@ -0,0 +1,7 @@ +{ + "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 }, + "node2": { "power": "off", "network": "off", "cpus": 32, "memory": 200000000000 }, + "node3": { "power": "off", "network": "off", "cpus": 32, "memory": 300000000000 }, + "node4": { "power": "off", "network": "off", "cpus": 64, "memory": 300000000000 }, + "node5": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 } +} diff --git a/src/test/test-crs-static2/log.expect b/src/test/test-crs-static2/log.expect new file mode 100644 index 0000000..ee4416c --- /dev/null +++ b/src/test/test-crs-static2/log.expect @@ -0,0 +1,171 @@ +info 0 hardware: starting simulation +info 20 cmdlist: execute power node1 on +info 20 node1/crm: status change startup => wait_for_quorum +info 20 node1/lrm: status change startup => wait_for_agent_lock +info 20 cmdlist: execute power node2 on +info 20 node2/crm: status change startup => wait_for_quorum +info 20 node2/lrm: status change startup => wait_for_agent_lock +info 20 cmdlist: execute power node3 on +info 20 node3/crm: status change startup => wait_for_quorum +info 20 node3/lrm: status change startup => wait_for_agent_lock +info 20 cmdlist: execute power node5 on +info 20 node5/crm: status change startup => wait_for_quorum +info 20 node5/lrm: status change startup => wait_for_agent_lock +info 20 node1/crm: got lock 'ha_manager_lock' +info 20 node1/crm: status change wait_for_quorum => master +info 20 node1/crm: using scheduler mode 'static' +info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online' +info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online' +info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online' +info 20 node1/crm: node 'node5': state changed from 'unknown' => 'online' +info 20 node1/crm: adding new service 'vm:102' on node 'node1' +info 21 node1/lrm: got lock 'ha_agent_node1_lock' +info 21 node1/lrm: status change wait_for_agent_lock => active +info 21 node1/lrm: starting service vm:102 +info 21 node1/lrm: service status vm:102 started +info 22 node2/crm: status change wait_for_quorum => slave +info 24 node3/crm: status change wait_for_quorum => slave +info 26 node5/crm: status change wait_for_quorum => slave +info 120 cmdlist: execute power node1 off +info 120 node1/crm: killed by poweroff +info 120 node1/lrm: killed by poweroff +info 220 cmdlist: execute delay 300 +info 222 node3/crm: got lock 'ha_manager_lock' +info 222 node3/crm: status change slave => master +info 222 node3/crm: using scheduler mode 'static' +info 222 node3/crm: node 'node1': state changed from 'online' => 'unknown' +info 282 node3/crm: service 'vm:102': state changed from 'started' to 'fence' +info 282 node3/crm: node 'node1': state changed from 'unknown' => 'fence' +emai 282 node3/crm: FENCE: Try to fence node 'node1' +info 282 node3/crm: got lock 'ha_agent_node1_lock' +info 282 node3/crm: fencing: acknowledged - got agent lock for node 'node1' +info 282 node3/crm: node 'node1': state changed from 'fence' => 'unknown' +emai 282 node3/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1' +info 282 node3/crm: service 'vm:102': state changed from 'fence' to 'recovery' +info 282 node3/crm: recover service 'vm:102' from fenced node 'node1' to node 'node3' +info 282 node3/crm: service 'vm:102': state changed from 'recovery' to 'started' (node = node3) +info 283 node3/lrm: got lock 'ha_agent_node3_lock' +info 283 node3/lrm: status change wait_for_agent_lock => active +info 283 node3/lrm: starting service vm:102 +info 283 node3/lrm: service status vm:102 started +info 600 cmdlist: execute power node1 on +info 600 node1/crm: status change startup => wait_for_quorum +info 600 node1/lrm: status change startup => wait_for_agent_lock +info 600 node1/crm: status change wait_for_quorum => slave +info 604 node3/crm: node 'node1': state changed from 'unknown' => 'online' +info 604 node3/crm: migrate service 'vm:102' to node 'node1' (running) +info 604 node3/crm: service 'vm:102': state changed from 'started' to 'migrate' (node = node3, target = node1) +info 605 node3/lrm: service vm:102 - start migrate to node 'node1' +info 605 node3/lrm: service vm:102 - end migrate to node 'node1' +info 624 node3/crm: service 'vm:102': state changed from 'migrate' to 'started' (node = node1) +info 641 node1/lrm: got lock 'ha_agent_node1_lock' +info 641 node1/lrm: status change wait_for_agent_lock => active +info 641 node1/lrm: starting service vm:102 +info 641 node1/lrm: service status vm:102 started +info 700 cmdlist: execute delay 300 +info 1080 cmdlist: execute power node4 on +info 1080 node4/crm: status change startup => wait_for_quorum +info 1080 node4/lrm: status change startup => wait_for_agent_lock +info 1084 node3/crm: node 'node4': state changed from 'unknown' => 'online' +info 1086 node4/crm: status change wait_for_quorum => slave +info 1180 cmdlist: execute power node1 off +info 1180 node1/crm: killed by poweroff +info 1180 node1/lrm: killed by poweroff +info 1182 node3/crm: node 'node1': state changed from 'online' => 'unknown' +info 1242 node3/crm: service 'vm:102': state changed from 'started' to 'fence' +info 1242 node3/crm: node 'node1': state changed from 'unknown' => 'fence' +emai 1242 node3/crm: FENCE: Try to fence node 'node1' +info 1280 cmdlist: execute delay 300 +info 1282 node3/crm: got lock 'ha_agent_node1_lock' +info 1282 node3/crm: fencing: acknowledged - got agent lock for node 'node1' +info 1282 node3/crm: node 'node1': state changed from 'fence' => 'unknown' +emai 1282 node3/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1' +info 1282 node3/crm: service 'vm:102': state changed from 'fence' to 'recovery' +info 1282 node3/crm: recover service 'vm:102' from fenced node 'node1' to node 'node4' +info 1282 node3/crm: service 'vm:102': state changed from 'recovery' to 'started' (node = node4) +info 1285 node4/lrm: got lock 'ha_agent_node4_lock' +info 1285 node4/lrm: status change wait_for_agent_lock => active +info 1285 node4/lrm: starting service vm:102 +info 1285 node4/lrm: service status vm:102 started +info 1660 cmdlist: execute power node1 on +info 1660 node1/crm: status change startup => wait_for_quorum +info 1660 node1/lrm: status change startup => wait_for_agent_lock +info 1660 node1/crm: status change wait_for_quorum => slave +info 1664 node3/crm: node 'node1': state changed from 'unknown' => 'online' +info 1664 node3/crm: migrate service 'vm:102' to node 'node1' (running) +info 1664 node3/crm: service 'vm:102': state changed from 'started' to 'migrate' (node = node4, target = node1) +info 1667 node4/lrm: service vm:102 - start migrate to node 'node1' +info 1667 node4/lrm: service vm:102 - end migrate to node 'node1' +info 1684 node3/crm: service 'vm:102': state changed from 'migrate' to 'started' (node = node1) +info 1701 node1/lrm: got lock 'ha_agent_node1_lock' +info 1701 node1/lrm: status change wait_for_agent_lock => active +info 1701 node1/lrm: starting service vm:102 +info 1701 node1/lrm: service status vm:102 started +info 1760 cmdlist: execute delay 300 +info 1825 node3/lrm: node had no service configured for 60 rounds, going idle. +info 1825 node3/lrm: status change active => wait_for_agent_lock +info 2140 cmdlist: execute power node2 off +info 2140 node2/crm: killed by poweroff +info 2140 node2/lrm: killed by poweroff +info 2142 node3/crm: node 'node2': state changed from 'online' => 'unknown' +info 2240 cmdlist: execute power node1 off +info 2240 node1/crm: killed by poweroff +info 2240 node1/lrm: killed by poweroff +info 2240 node3/crm: node 'node1': state changed from 'online' => 'unknown' +info 2300 node3/crm: service 'vm:102': state changed from 'started' to 'fence' +info 2300 node3/crm: node 'node1': state changed from 'unknown' => 'fence' +emai 2300 node3/crm: FENCE: Try to fence node 'node1' +info 2340 cmdlist: execute delay 300 +info 2360 node3/crm: got lock 'ha_agent_node1_lock' +info 2360 node3/crm: fencing: acknowledged - got agent lock for node 'node1' +info 2360 node3/crm: node 'node1': state changed from 'fence' => 'unknown' +emai 2360 node3/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1' +info 2360 node3/crm: service 'vm:102': state changed from 'fence' to 'recovery' +info 2360 node3/crm: recover service 'vm:102' from fenced node 'node1' to node 'node4' +info 2360 node3/crm: service 'vm:102': state changed from 'recovery' to 'started' (node = node4) +info 2363 node4/lrm: starting service vm:102 +info 2363 node4/lrm: service status vm:102 started +info 2720 cmdlist: execute power node1 on +info 2720 node1/crm: status change startup => wait_for_quorum +info 2720 node1/lrm: status change startup => wait_for_agent_lock +info 2720 node1/crm: status change wait_for_quorum => slave +info 2722 node3/crm: node 'node1': state changed from 'unknown' => 'online' +info 2722 node3/crm: migrate service 'vm:102' to node 'node1' (running) +info 2722 node3/crm: service 'vm:102': state changed from 'started' to 'migrate' (node = node4, target = node1) +info 2725 node4/lrm: service vm:102 - start migrate to node 'node1' +info 2725 node4/lrm: service vm:102 - end migrate to node 'node1' +info 2742 node3/crm: service 'vm:102': state changed from 'migrate' to 'started' (node = node1) +info 2761 node1/lrm: got lock 'ha_agent_node1_lock' +info 2761 node1/lrm: status change wait_for_agent_lock => active +info 2761 node1/lrm: starting service vm:102 +info 2761 node1/lrm: service status vm:102 started +info 2820 cmdlist: execute delay 300 +info 3200 cmdlist: execute power node2 on +info 3200 node2/crm: status change startup => wait_for_quorum +info 3200 node2/lrm: status change startup => wait_for_agent_lock +info 3202 node2/crm: status change wait_for_quorum => slave +info 3204 node3/crm: node 'node2': state changed from 'unknown' => 'online' +info 3300 cmdlist: execute power node3 off +info 3300 node3/crm: killed by poweroff +info 3300 node3/lrm: killed by poweroff +info 3400 cmdlist: execute power node1 off +info 3400 node1/crm: killed by poweroff +info 3400 node1/lrm: killed by poweroff +info 3420 node2/crm: got lock 'ha_manager_lock' +info 3420 node2/crm: status change slave => master +info 3420 node2/crm: using scheduler mode 'static' +info 3420 node2/crm: node 'node1': state changed from 'online' => 'unknown' +info 3420 node2/crm: node 'node3': state changed from 'online' => 'unknown' +info 3480 node2/crm: service 'vm:102': state changed from 'started' to 'fence' +info 3480 node2/crm: node 'node1': state changed from 'unknown' => 'fence' +emai 3480 node2/crm: FENCE: Try to fence node 'node1' +info 3520 node2/crm: got lock 'ha_agent_node1_lock' +info 3520 node2/crm: fencing: acknowledged - got agent lock for node 'node1' +info 3520 node2/crm: node 'node1': state changed from 'fence' => 'unknown' +emai 3520 node2/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1' +info 3520 node2/crm: service 'vm:102': state changed from 'fence' to 'recovery' +info 3520 node2/crm: recover service 'vm:102' from fenced node 'node1' to node 'node4' +info 3520 node2/crm: service 'vm:102': state changed from 'recovery' to 'started' (node = node4) +info 3523 node4/lrm: starting service vm:102 +info 3523 node4/lrm: service status vm:102 started +info 4000 hardware: exit simulation - done diff --git a/src/test/test-crs-static2/manager_status b/src/test/test-crs-static2/manager_status new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/src/test/test-crs-static2/manager_status @@ -0,0 +1 @@ +{} diff --git a/src/test/test-crs-static2/service_config b/src/test/test-crs-static2/service_config new file mode 100644 index 0000000..1f2333d --- /dev/null +++ b/src/test/test-crs-static2/service_config @@ -0,0 +1,3 @@ +{ + "vm:102": { "node": "node1", "state": "enabled", "group": "prefer_node1" } +} diff --git a/src/test/test-crs-static2/static_service_stats b/src/test/test-crs-static2/static_service_stats new file mode 100644 index 0000000..7fb992d --- /dev/null +++ b/src/test/test-crs-static2/static_service_stats @@ -0,0 +1,3 @@ +{ + "vm:102": { "maxcpu": 2, "maxmem": 4000000000 } +} diff --git a/src/test/test-crs-static3/README b/src/test/test-crs-static3/README new file mode 100644 index 0000000..db929e1 --- /dev/null +++ b/src/test/test-crs-static3/README @@ -0,0 +1,5 @@ +Test how shutdown migrate policy works with the 'static' resource scheduling +mode. + +Expect that when node1 is shut down the services get migrated in the repeating +sequence node2 node2 node3, because node 2 has twice the resources of node 3. diff --git a/src/test/test-crs-static3/cmdlist b/src/test/test-crs-static3/cmdlist new file mode 100644 index 0000000..e84297f --- /dev/null +++ b/src/test/test-crs-static3/cmdlist @@ -0,0 +1,4 @@ +[ + [ "power node1 on", "power node2 on", "power node3 on"], + [ "shutdown node1" ] +] diff --git a/src/test/test-crs-static3/datacenter.cfg b/src/test/test-crs-static3/datacenter.cfg new file mode 100644 index 0000000..caa8148 --- /dev/null +++ b/src/test/test-crs-static3/datacenter.cfg @@ -0,0 +1,9 @@ +{ + "crs": { + "ha": "static" + }, + "ha": { + "shutdown_policy": "migrate" + } +} + diff --git a/src/test/test-crs-static3/hardware_status b/src/test/test-crs-static3/hardware_status new file mode 100644 index 0000000..dfbf496 --- /dev/null +++ b/src/test/test-crs-static3/hardware_status @@ -0,0 +1,5 @@ +{ + "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 }, + "node2": { "power": "off", "network": "off", "cpus": 64, "memory": 200000000000 }, + "node3": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 } +} diff --git a/src/test/test-crs-static3/log.expect b/src/test/test-crs-static3/log.expect new file mode 100644 index 0000000..00cfefb --- /dev/null +++ b/src/test/test-crs-static3/log.expect @@ -0,0 +1,131 @@ +info 0 hardware: starting simulation +info 20 cmdlist: execute power node1 on +info 20 node1/crm: status change startup => wait_for_quorum +info 20 node1/lrm: status change startup => wait_for_agent_lock +info 20 cmdlist: execute power node2 on +info 20 node2/crm: status change startup => wait_for_quorum +info 20 node2/lrm: status change startup => wait_for_agent_lock +info 20 cmdlist: execute power node3 on +info 20 node3/crm: status change startup => wait_for_quorum +info 20 node3/lrm: status change startup => wait_for_agent_lock +info 20 node1/crm: got lock 'ha_manager_lock' +info 20 node1/crm: status change wait_for_quorum => master +info 20 node1/crm: using scheduler mode 'static' +info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online' +info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online' +info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online' +info 20 node1/crm: adding new service 'vm:100' on node 'node1' +info 20 node1/crm: adding new service 'vm:101' on node 'node1' +info 20 node1/crm: adding new service 'vm:102' on node 'node1' +info 20 node1/crm: adding new service 'vm:103' on node 'node1' +info 20 node1/crm: adding new service 'vm:104' on node 'node1' +info 20 node1/crm: adding new service 'vm:105' on node 'node1' +info 20 node1/crm: adding new service 'vm:106' on node 'node1' +info 20 node1/crm: adding new service 'vm:107' on node 'node1' +info 20 node1/crm: adding new service 'vm:108' on node 'node1' +info 20 node1/crm: adding new service 'vm:109' on node 'node1' +info 21 node1/lrm: got lock 'ha_agent_node1_lock' +info 21 node1/lrm: status change wait_for_agent_lock => active +info 21 node1/lrm: starting service vm:101 +info 21 node1/lrm: service status vm:101 started +info 21 node1/lrm: starting service vm:102 +info 21 node1/lrm: service status vm:102 started +info 21 node1/lrm: starting service vm:103 +info 21 node1/lrm: service status vm:103 started +info 21 node1/lrm: starting service vm:104 +info 21 node1/lrm: service status vm:104 started +info 21 node1/lrm: starting service vm:105 +info 21 node1/lrm: service status vm:105 started +info 21 node1/lrm: starting service vm:106 +info 21 node1/lrm: service status vm:106 started +info 21 node1/lrm: starting service vm:107 +info 21 node1/lrm: service status vm:107 started +info 21 node1/lrm: starting service vm:108 +info 21 node1/lrm: service status vm:108 started +info 21 node1/lrm: starting service vm:109 +info 21 node1/lrm: service status vm:109 started +info 22 node2/crm: status change wait_for_quorum => slave +info 24 node3/crm: status change wait_for_quorum => slave +info 40 node1/crm: service 'vm:100': state changed from 'request_stop' to 'stopped' +info 120 cmdlist: execute shutdown node1 +info 120 node1/lrm: got shutdown request with shutdown policy 'migrate' +info 120 node1/lrm: shutdown LRM, doing maintenance, removing this node from active list +info 120 node1/crm: node 'node1': state changed from 'online' => 'maintenance' +info 120 node1/crm: migrate service 'vm:101' to node 'node2' (running) +info 120 node1/crm: service 'vm:101': state changed from 'started' to 'migrate' (node = node1, target = node2) +info 120 node1/crm: migrate service 'vm:102' to node 'node2' (running) +info 120 node1/crm: service 'vm:102': state changed from 'started' to 'migrate' (node = node1, target = node2) +info 120 node1/crm: migrate service 'vm:103' to node 'node3' (running) +info 120 node1/crm: service 'vm:103': state changed from 'started' to 'migrate' (node = node1, target = node3) +info 120 node1/crm: migrate service 'vm:104' to node 'node2' (running) +info 120 node1/crm: service 'vm:104': state changed from 'started' to 'migrate' (node = node1, target = node2) +info 120 node1/crm: migrate service 'vm:105' to node 'node2' (running) +info 120 node1/crm: service 'vm:105': state changed from 'started' to 'migrate' (node = node1, target = node2) +info 120 node1/crm: migrate service 'vm:106' to node 'node3' (running) +info 120 node1/crm: service 'vm:106': state changed from 'started' to 'migrate' (node = node1, target = node3) +info 120 node1/crm: migrate service 'vm:107' to node 'node2' (running) +info 120 node1/crm: service 'vm:107': state changed from 'started' to 'migrate' (node = node1, target = node2) +info 120 node1/crm: migrate service 'vm:108' to node 'node2' (running) +info 120 node1/crm: service 'vm:108': state changed from 'started' to 'migrate' (node = node1, target = node2) +info 120 node1/crm: migrate service 'vm:109' to node 'node3' (running) +info 120 node1/crm: service 'vm:109': state changed from 'started' to 'migrate' (node = node1, target = node3) +info 121 node1/lrm: status change active => maintenance +info 121 node1/lrm: service vm:101 - start migrate to node 'node2' +info 121 node1/lrm: service vm:101 - end migrate to node 'node2' +info 121 node1/lrm: service vm:102 - start migrate to node 'node2' +info 121 node1/lrm: service vm:102 - end migrate to node 'node2' +info 121 node1/lrm: service vm:103 - start migrate to node 'node3' +info 121 node1/lrm: service vm:103 - end migrate to node 'node3' +info 121 node1/lrm: service vm:104 - start migrate to node 'node2' +info 121 node1/lrm: service vm:104 - end migrate to node 'node2' +info 121 node1/lrm: service vm:105 - start migrate to node 'node2' +info 121 node1/lrm: service vm:105 - end migrate to node 'node2' +info 121 node1/lrm: service vm:106 - start migrate to node 'node3' +info 121 node1/lrm: service vm:106 - end migrate to node 'node3' +info 121 node1/lrm: service vm:107 - start migrate to node 'node2' +info 121 node1/lrm: service vm:107 - end migrate to node 'node2' +info 121 node1/lrm: service vm:108 - start migrate to node 'node2' +info 121 node1/lrm: service vm:108 - end migrate to node 'node2' +info 121 node1/lrm: service vm:109 - start migrate to node 'node3' +info 121 node1/lrm: service vm:109 - end migrate to node 'node3' +info 140 node1/crm: service 'vm:101': state changed from 'migrate' to 'started' (node = node2) +info 140 node1/crm: service 'vm:102': state changed from 'migrate' to 'started' (node = node2) +info 140 node1/crm: service 'vm:103': state changed from 'migrate' to 'started' (node = node3) +info 140 node1/crm: service 'vm:104': state changed from 'migrate' to 'started' (node = node2) +info 140 node1/crm: service 'vm:105': state changed from 'migrate' to 'started' (node = node2) +info 140 node1/crm: service 'vm:106': state changed from 'migrate' to 'started' (node = node3) +info 140 node1/crm: service 'vm:107': state changed from 'migrate' to 'started' (node = node2) +info 140 node1/crm: service 'vm:108': state changed from 'migrate' to 'started' (node = node2) +info 140 node1/crm: service 'vm:109': state changed from 'migrate' to 'started' (node = node3) +info 142 node1/lrm: exit (loop end) +info 142 shutdown: execute crm node1 stop +info 141 node1/crm: server received shutdown request +info 143 node2/lrm: got lock 'ha_agent_node2_lock' +info 143 node2/lrm: status change wait_for_agent_lock => active +info 143 node2/lrm: starting service vm:101 +info 143 node2/lrm: service status vm:101 started +info 143 node2/lrm: starting service vm:102 +info 143 node2/lrm: service status vm:102 started +info 143 node2/lrm: starting service vm:104 +info 143 node2/lrm: service status vm:104 started +info 143 node2/lrm: starting service vm:105 +info 143 node2/lrm: service status vm:105 started +info 143 node2/lrm: starting service vm:107 +info 143 node2/lrm: service status vm:107 started +info 143 node2/lrm: starting service vm:108 +info 143 node2/lrm: service status vm:108 started +info 145 node3/lrm: got lock 'ha_agent_node3_lock' +info 145 node3/lrm: status change wait_for_agent_lock => active +info 145 node3/lrm: starting service vm:103 +info 145 node3/lrm: service status vm:103 started +info 145 node3/lrm: starting service vm:106 +info 145 node3/lrm: service status vm:106 started +info 145 node3/lrm: starting service vm:109 +info 145 node3/lrm: service status vm:109 started +info 160 node1/crm: voluntary release CRM lock +info 161 node1/crm: exit (loop end) +info 161 shutdown: execute power node1 off +info 161 node2/crm: got lock 'ha_manager_lock' +info 161 node2/crm: status change slave => master +info 161 node2/crm: using scheduler mode 'static' +info 720 hardware: exit simulation - done diff --git a/src/test/test-crs-static3/manager_status b/src/test/test-crs-static3/manager_status new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/src/test/test-crs-static3/manager_status @@ -0,0 +1 @@ +{} diff --git a/src/test/test-crs-static3/service_config b/src/test/test-crs-static3/service_config new file mode 100644 index 0000000..47f94d3 --- /dev/null +++ b/src/test/test-crs-static3/service_config @@ -0,0 +1,12 @@ +{ + "vm:100": { "node": "node1", "state": "stopped" }, + "vm:101": { "node": "node1", "state": "enabled" }, + "vm:102": { "node": "node1", "state": "enabled" }, + "vm:103": { "node": "node1", "state": "enabled" }, + "vm:104": { "node": "node1", "state": "enabled" }, + "vm:105": { "node": "node1", "state": "enabled" }, + "vm:106": { "node": "node1", "state": "enabled" }, + "vm:107": { "node": "node1", "state": "enabled" }, + "vm:108": { "node": "node1", "state": "enabled" }, + "vm:109": { "node": "node1", "state": "enabled" } +} diff --git a/src/test/test-crs-static3/static_service_stats b/src/test/test-crs-static3/static_service_stats new file mode 100644 index 0000000..bca71cb --- /dev/null +++ b/src/test/test-crs-static3/static_service_stats @@ -0,0 +1,12 @@ +{ + "vm:100": { "maxcpu": 2, "maxmem": 4000000000 }, + "vm:101": { "maxcpu": 2, "maxmem": 4000000000 }, + "vm:102": { "maxcpu": 2, "maxmem": 4000000000 }, + "vm:103": { "maxcpu": 2, "maxmem": 4000000000 }, + "vm:104": { "maxcpu": 2, "maxmem": 4000000000 }, + "vm:105": { "maxcpu": 2, "maxmem": 4000000000 }, + "vm:106": { "maxcpu": 2, "maxmem": 4000000000 }, + "vm:107": { "maxcpu": 2, "maxmem": 4000000000 }, + "vm:108": { "maxcpu": 2, "maxmem": 4000000000 }, + "vm:109": { "maxcpu": 2, "maxmem": 4000000000 } +} diff --git a/src/test/test-crs-static4/README b/src/test/test-crs-static4/README new file mode 100644 index 0000000..4dfb1bc --- /dev/null +++ b/src/test/test-crs-static4/README @@ -0,0 +1,6 @@ +Test how shutdown migrate policy works with the 'static' resource scheduling +mode. + +Expect that, when node1 is shut down, the first service is migrated to node2 and +all others to node 3, because the first service is very resource-heavy compared +to the others. diff --git a/src/test/test-crs-static4/cmdlist b/src/test/test-crs-static4/cmdlist new file mode 100644 index 0000000..e84297f --- /dev/null +++ b/src/test/test-crs-static4/cmdlist @@ -0,0 +1,4 @@ +[ + [ "power node1 on", "power node2 on", "power node3 on"], + [ "shutdown node1" ] +] diff --git a/src/test/test-crs-static4/datacenter.cfg b/src/test/test-crs-static4/datacenter.cfg new file mode 100644 index 0000000..caa8148 --- /dev/null +++ b/src/test/test-crs-static4/datacenter.cfg @@ -0,0 +1,9 @@ +{ + "crs": { + "ha": "static" + }, + "ha": { + "shutdown_policy": "migrate" + } +} + diff --git a/src/test/test-crs-static4/hardware_status b/src/test/test-crs-static4/hardware_status new file mode 100644 index 0000000..a83a2dc --- /dev/null +++ b/src/test/test-crs-static4/hardware_status @@ -0,0 +1,5 @@ +{ + "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 }, + "node2": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 }, + "node3": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 } +} diff --git a/src/test/test-crs-static4/log.expect b/src/test/test-crs-static4/log.expect new file mode 100644 index 0000000..3eedc23 --- /dev/null +++ b/src/test/test-crs-static4/log.expect @@ -0,0 +1,149 @@ +info 0 hardware: starting simulation +info 20 cmdlist: execute power node1 on +info 20 node1/crm: status change startup => wait_for_quorum +info 20 node1/lrm: status change startup => wait_for_agent_lock +info 20 cmdlist: execute power node2 on +info 20 node2/crm: status change startup => wait_for_quorum +info 20 node2/lrm: status change startup => wait_for_agent_lock +info 20 cmdlist: execute power node3 on +info 20 node3/crm: status change startup => wait_for_quorum +info 20 node3/lrm: status change startup => wait_for_agent_lock +info 20 node1/crm: got lock 'ha_manager_lock' +info 20 node1/crm: status change wait_for_quorum => master +info 20 node1/crm: using scheduler mode 'static' +info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online' +info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online' +info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online' +info 20 node1/crm: adding new service 'ct:100' on node 'node1' +info 20 node1/crm: adding new service 'ct:101' on node 'node1' +info 20 node1/crm: adding new service 'ct:102' on node 'node1' +info 20 node1/crm: adding new service 'ct:103' on node 'node1' +info 20 node1/crm: adding new service 'ct:104' on node 'node1' +info 20 node1/crm: adding new service 'ct:105' on node 'node1' +info 20 node1/crm: adding new service 'ct:106' on node 'node1' +info 20 node1/crm: adding new service 'ct:107' on node 'node1' +info 20 node1/crm: adding new service 'ct:108' on node 'node1' +info 20 node1/crm: adding new service 'ct:109' on node 'node1' +info 21 node1/lrm: got lock 'ha_agent_node1_lock' +info 21 node1/lrm: status change wait_for_agent_lock => active +info 21 node1/lrm: starting service ct:101 +info 21 node1/lrm: service status ct:101 started +info 21 node1/lrm: starting service ct:102 +info 21 node1/lrm: service status ct:102 started +info 21 node1/lrm: starting service ct:103 +info 21 node1/lrm: service status ct:103 started +info 21 node1/lrm: starting service ct:104 +info 21 node1/lrm: service status ct:104 started +info 21 node1/lrm: starting service ct:105 +info 21 node1/lrm: service status ct:105 started +info 21 node1/lrm: starting service ct:106 +info 21 node1/lrm: service status ct:106 started +info 21 node1/lrm: starting service ct:107 +info 21 node1/lrm: service status ct:107 started +info 21 node1/lrm: starting service ct:108 +info 21 node1/lrm: service status ct:108 started +info 21 node1/lrm: starting service ct:109 +info 21 node1/lrm: service status ct:109 started +info 22 node2/crm: status change wait_for_quorum => slave +info 24 node3/crm: status change wait_for_quorum => slave +info 40 node1/crm: service 'ct:100': state changed from 'request_stop' to 'stopped' +info 120 cmdlist: execute shutdown node1 +info 120 node1/lrm: got shutdown request with shutdown policy 'migrate' +info 120 node1/lrm: shutdown LRM, doing maintenance, removing this node from active list +info 120 node1/crm: node 'node1': state changed from 'online' => 'maintenance' +info 120 node1/crm: relocate service 'ct:101' to node 'node2' +info 120 node1/crm: service 'ct:101': state changed from 'started' to 'relocate' (node = node1, target = node2) +info 120 node1/crm: relocate service 'ct:102' to node 'node3' +info 120 node1/crm: service 'ct:102': state changed from 'started' to 'relocate' (node = node1, target = node3) +info 120 node1/crm: relocate service 'ct:103' to node 'node3' +info 120 node1/crm: service 'ct:103': state changed from 'started' to 'relocate' (node = node1, target = node3) +info 120 node1/crm: relocate service 'ct:104' to node 'node3' +info 120 node1/crm: service 'ct:104': state changed from 'started' to 'relocate' (node = node1, target = node3) +info 120 node1/crm: relocate service 'ct:105' to node 'node3' +info 120 node1/crm: service 'ct:105': state changed from 'started' to 'relocate' (node = node1, target = node3) +info 120 node1/crm: relocate service 'ct:106' to node 'node3' +info 120 node1/crm: service 'ct:106': state changed from 'started' to 'relocate' (node = node1, target = node3) +info 120 node1/crm: relocate service 'ct:107' to node 'node3' +info 120 node1/crm: service 'ct:107': state changed from 'started' to 'relocate' (node = node1, target = node3) +info 120 node1/crm: relocate service 'ct:108' to node 'node3' +info 120 node1/crm: service 'ct:108': state changed from 'started' to 'relocate' (node = node1, target = node3) +info 120 node1/crm: relocate service 'ct:109' to node 'node3' +info 120 node1/crm: service 'ct:109': state changed from 'started' to 'relocate' (node = node1, target = node3) +info 121 node1/lrm: status change active => maintenance +info 121 node1/lrm: service ct:101 - start relocate to node 'node2' +info 121 node1/lrm: stopping service ct:101 (relocate) +info 121 node1/lrm: service status ct:101 stopped +info 121 node1/lrm: service ct:101 - end relocate to node 'node2' +info 121 node1/lrm: service ct:102 - start relocate to node 'node3' +info 121 node1/lrm: stopping service ct:102 (relocate) +info 121 node1/lrm: service status ct:102 stopped +info 121 node1/lrm: service ct:102 - end relocate to node 'node3' +info 121 node1/lrm: service ct:103 - start relocate to node 'node3' +info 121 node1/lrm: stopping service ct:103 (relocate) +info 121 node1/lrm: service status ct:103 stopped +info 121 node1/lrm: service ct:103 - end relocate to node 'node3' +info 121 node1/lrm: service ct:104 - start relocate to node 'node3' +info 121 node1/lrm: stopping service ct:104 (relocate) +info 121 node1/lrm: service status ct:104 stopped +info 121 node1/lrm: service ct:104 - end relocate to node 'node3' +info 121 node1/lrm: service ct:105 - start relocate to node 'node3' +info 121 node1/lrm: stopping service ct:105 (relocate) +info 121 node1/lrm: service status ct:105 stopped +info 121 node1/lrm: service ct:105 - end relocate to node 'node3' +info 121 node1/lrm: service ct:106 - start relocate to node 'node3' +info 121 node1/lrm: stopping service ct:106 (relocate) +info 121 node1/lrm: service status ct:106 stopped +info 121 node1/lrm: service ct:106 - end relocate to node 'node3' +info 121 node1/lrm: service ct:107 - start relocate to node 'node3' +info 121 node1/lrm: stopping service ct:107 (relocate) +info 121 node1/lrm: service status ct:107 stopped +info 121 node1/lrm: service ct:107 - end relocate to node 'node3' +info 121 node1/lrm: service ct:108 - start relocate to node 'node3' +info 121 node1/lrm: stopping service ct:108 (relocate) +info 121 node1/lrm: service status ct:108 stopped +info 121 node1/lrm: service ct:108 - end relocate to node 'node3' +info 121 node1/lrm: service ct:109 - start relocate to node 'node3' +info 121 node1/lrm: stopping service ct:109 (relocate) +info 121 node1/lrm: service status ct:109 stopped +info 121 node1/lrm: service ct:109 - end relocate to node 'node3' +info 140 node1/crm: service 'ct:101': state changed from 'relocate' to 'started' (node = node2) +info 140 node1/crm: service 'ct:102': state changed from 'relocate' to 'started' (node = node3) +info 140 node1/crm: service 'ct:103': state changed from 'relocate' to 'started' (node = node3) +info 140 node1/crm: service 'ct:104': state changed from 'relocate' to 'started' (node = node3) +info 140 node1/crm: service 'ct:105': state changed from 'relocate' to 'started' (node = node3) +info 140 node1/crm: service 'ct:106': state changed from 'relocate' to 'started' (node = node3) +info 140 node1/crm: service 'ct:107': state changed from 'relocate' to 'started' (node = node3) +info 140 node1/crm: service 'ct:108': state changed from 'relocate' to 'started' (node = node3) +info 140 node1/crm: service 'ct:109': state changed from 'relocate' to 'started' (node = node3) +info 142 node1/lrm: exit (loop end) +info 142 shutdown: execute crm node1 stop +info 141 node1/crm: server received shutdown request +info 143 node2/lrm: got lock 'ha_agent_node2_lock' +info 143 node2/lrm: status change wait_for_agent_lock => active +info 143 node2/lrm: starting service ct:101 +info 143 node2/lrm: service status ct:101 started +info 145 node3/lrm: got lock 'ha_agent_node3_lock' +info 145 node3/lrm: status change wait_for_agent_lock => active +info 145 node3/lrm: starting service ct:102 +info 145 node3/lrm: service status ct:102 started +info 145 node3/lrm: starting service ct:103 +info 145 node3/lrm: service status ct:103 started +info 145 node3/lrm: starting service ct:104 +info 145 node3/lrm: service status ct:104 started +info 145 node3/lrm: starting service ct:105 +info 145 node3/lrm: service status ct:105 started +info 145 node3/lrm: starting service ct:106 +info 145 node3/lrm: service status ct:106 started +info 145 node3/lrm: starting service ct:107 +info 145 node3/lrm: service status ct:107 started +info 145 node3/lrm: starting service ct:108 +info 145 node3/lrm: service status ct:108 started +info 145 node3/lrm: starting service ct:109 +info 145 node3/lrm: service status ct:109 started +info 160 node1/crm: voluntary release CRM lock +info 161 node1/crm: exit (loop end) +info 161 shutdown: execute power node1 off +info 161 node2/crm: got lock 'ha_manager_lock' +info 161 node2/crm: status change slave => master +info 161 node2/crm: using scheduler mode 'static' +info 720 hardware: exit simulation - done diff --git a/src/test/test-crs-static4/manager_status b/src/test/test-crs-static4/manager_status new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/src/test/test-crs-static4/manager_status @@ -0,0 +1 @@ +{} diff --git a/src/test/test-crs-static4/service_config b/src/test/test-crs-static4/service_config new file mode 100644 index 0000000..b984a09 --- /dev/null +++ b/src/test/test-crs-static4/service_config @@ -0,0 +1,12 @@ +{ + "ct:100": { "node": "node1", "state": "stopped" }, + "ct:101": { "node": "node1", "state": "enabled" }, + "ct:102": { "node": "node1", "state": "enabled" }, + "ct:103": { "node": "node1", "state": "enabled" }, + "ct:104": { "node": "node1", "state": "enabled" }, + "ct:105": { "node": "node1", "state": "enabled" }, + "ct:106": { "node": "node1", "state": "enabled" }, + "ct:107": { "node": "node1", "state": "enabled" }, + "ct:108": { "node": "node1", "state": "enabled" }, + "ct:109": { "node": "node1", "state": "enabled" } +} diff --git a/src/test/test-crs-static4/static_service_stats b/src/test/test-crs-static4/static_service_stats new file mode 100644 index 0000000..878709b --- /dev/null +++ b/src/test/test-crs-static4/static_service_stats @@ -0,0 +1,12 @@ +{ + "ct:100": { "maxcpu": 2, "maxmem": 4000000000 }, + "ct:101": { "maxcpu": 0, "maxmem": 40000000000 }, + "ct:102": { "maxcpu": 2, "maxmem": 2000000000 }, + "ct:103": { "maxcpu": 2, "maxmem": 2000000000 }, + "ct:104": { "maxcpu": 2, "maxmem": 2000000000 }, + "ct:105": { "maxcpu": 2, "maxmem": 2000000000 }, + "ct:106": { "maxcpu": 2, "maxmem": 2000000000 }, + "ct:107": { "maxcpu": 2, "maxmem": 2000000000 }, + "ct:108": { "maxcpu": 2, "maxmem": 2000000000 }, + "ct:109": { "maxcpu": 2, "maxmem": 2000000000 } +} diff --git a/src/test/test-crs-static5/README b/src/test/test-crs-static5/README new file mode 100644 index 0000000..d9b5dc7 --- /dev/null +++ b/src/test/test-crs-static5/README @@ -0,0 +1,5 @@ +Test how recovery works with the 'static' resource scheduling mode. + +Expect that, when node1 is shut down, all services are migrated to node 3, +because the services don't have much memory, node 2 and 3 both already have a +service with high memory, but node 3 has much left-over CPU. diff --git a/src/test/test-crs-static5/cmdlist b/src/test/test-crs-static5/cmdlist new file mode 100644 index 0000000..e84297f --- /dev/null +++ b/src/test/test-crs-static5/cmdlist @@ -0,0 +1,4 @@ +[ + [ "power node1 on", "power node2 on", "power node3 on"], + [ "shutdown node1" ] +] diff --git a/src/test/test-crs-static5/datacenter.cfg b/src/test/test-crs-static5/datacenter.cfg new file mode 100644 index 0000000..caa8148 --- /dev/null +++ b/src/test/test-crs-static5/datacenter.cfg @@ -0,0 +1,9 @@ +{ + "crs": { + "ha": "static" + }, + "ha": { + "shutdown_policy": "migrate" + } +} + diff --git a/src/test/test-crs-static5/hardware_status b/src/test/test-crs-static5/hardware_status new file mode 100644 index 0000000..3eb9e73 --- /dev/null +++ b/src/test/test-crs-static5/hardware_status @@ -0,0 +1,5 @@ +{ + "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 }, + "node2": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 }, + "node3": { "power": "off", "network": "off", "cpus": 128, "memory": 100000000000 } +} diff --git a/src/test/test-crs-static5/log.expect b/src/test/test-crs-static5/log.expect new file mode 100644 index 0000000..cb6b0d5 --- /dev/null +++ b/src/test/test-crs-static5/log.expect @@ -0,0 +1,117 @@ +info 0 hardware: starting simulation +info 20 cmdlist: execute power node1 on +info 20 node1/crm: status change startup => wait_for_quorum +info 20 node1/lrm: status change startup => wait_for_agent_lock +info 20 cmdlist: execute power node2 on +info 20 node2/crm: status change startup => wait_for_quorum +info 20 node2/lrm: status change startup => wait_for_agent_lock +info 20 cmdlist: execute power node3 on +info 20 node3/crm: status change startup => wait_for_quorum +info 20 node3/lrm: status change startup => wait_for_agent_lock +info 20 node1/crm: got lock 'ha_manager_lock' +info 20 node1/crm: status change wait_for_quorum => master +info 20 node1/crm: using scheduler mode 'static' +info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online' +info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online' +info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online' +info 20 node1/crm: adding new service 'ct:102' on node 'node2' +info 20 node1/crm: adding new service 'ct:103' on node 'node3' +info 20 node1/crm: adding new service 'ct:104' on node 'node1' +info 20 node1/crm: adding new service 'ct:105' on node 'node1' +info 20 node1/crm: adding new service 'ct:106' on node 'node1' +info 20 node1/crm: adding new service 'ct:107' on node 'node1' +info 20 node1/crm: adding new service 'ct:108' on node 'node1' +info 20 node1/crm: adding new service 'ct:109' on node 'node1' +info 21 node1/lrm: got lock 'ha_agent_node1_lock' +info 21 node1/lrm: status change wait_for_agent_lock => active +info 21 node1/lrm: starting service ct:104 +info 21 node1/lrm: service status ct:104 started +info 21 node1/lrm: starting service ct:105 +info 21 node1/lrm: service status ct:105 started +info 21 node1/lrm: starting service ct:106 +info 21 node1/lrm: service status ct:106 started +info 21 node1/lrm: starting service ct:107 +info 21 node1/lrm: service status ct:107 started +info 21 node1/lrm: starting service ct:108 +info 21 node1/lrm: service status ct:108 started +info 21 node1/lrm: starting service ct:109 +info 21 node1/lrm: service status ct:109 started +info 22 node2/crm: status change wait_for_quorum => slave +info 23 node2/lrm: got lock 'ha_agent_node2_lock' +info 23 node2/lrm: status change wait_for_agent_lock => active +info 23 node2/lrm: starting service ct:102 +info 23 node2/lrm: service status ct:102 started +info 24 node3/crm: status change wait_for_quorum => slave +info 25 node3/lrm: got lock 'ha_agent_node3_lock' +info 25 node3/lrm: status change wait_for_agent_lock => active +info 25 node3/lrm: starting service ct:103 +info 25 node3/lrm: service status ct:103 started +info 120 cmdlist: execute shutdown node1 +info 120 node1/lrm: got shutdown request with shutdown policy 'migrate' +info 120 node1/lrm: shutdown LRM, doing maintenance, removing this node from active list +info 120 node1/crm: node 'node1': state changed from 'online' => 'maintenance' +info 120 node1/crm: relocate service 'ct:104' to node 'node3' +info 120 node1/crm: service 'ct:104': state changed from 'started' to 'relocate' (node = node1, target = node3) +info 120 node1/crm: relocate service 'ct:105' to node 'node3' +info 120 node1/crm: service 'ct:105': state changed from 'started' to 'relocate' (node = node1, target = node3) +info 120 node1/crm: relocate service 'ct:106' to node 'node3' +info 120 node1/crm: service 'ct:106': state changed from 'started' to 'relocate' (node = node1, target = node3) +info 120 node1/crm: relocate service 'ct:107' to node 'node3' +info 120 node1/crm: service 'ct:107': state changed from 'started' to 'relocate' (node = node1, target = node3) +info 120 node1/crm: relocate service 'ct:108' to node 'node3' +info 120 node1/crm: service 'ct:108': state changed from 'started' to 'relocate' (node = node1, target = node3) +info 120 node1/crm: relocate service 'ct:109' to node 'node3' +info 120 node1/crm: service 'ct:109': state changed from 'started' to 'relocate' (node = node1, target = node3) +info 121 node1/lrm: status change active => maintenance +info 121 node1/lrm: service ct:104 - start relocate to node 'node3' +info 121 node1/lrm: stopping service ct:104 (relocate) +info 121 node1/lrm: service status ct:104 stopped +info 121 node1/lrm: service ct:104 - end relocate to node 'node3' +info 121 node1/lrm: service ct:105 - start relocate to node 'node3' +info 121 node1/lrm: stopping service ct:105 (relocate) +info 121 node1/lrm: service status ct:105 stopped +info 121 node1/lrm: service ct:105 - end relocate to node 'node3' +info 121 node1/lrm: service ct:106 - start relocate to node 'node3' +info 121 node1/lrm: stopping service ct:106 (relocate) +info 121 node1/lrm: service status ct:106 stopped +info 121 node1/lrm: service ct:106 - end relocate to node 'node3' +info 121 node1/lrm: service ct:107 - start relocate to node 'node3' +info 121 node1/lrm: stopping service ct:107 (relocate) +info 121 node1/lrm: service status ct:107 stopped +info 121 node1/lrm: service ct:107 - end relocate to node 'node3' +info 121 node1/lrm: service ct:108 - start relocate to node 'node3' +info 121 node1/lrm: stopping service ct:108 (relocate) +info 121 node1/lrm: service status ct:108 stopped +info 121 node1/lrm: service ct:108 - end relocate to node 'node3' +info 121 node1/lrm: service ct:109 - start relocate to node 'node3' +info 121 node1/lrm: stopping service ct:109 (relocate) +info 121 node1/lrm: service status ct:109 stopped +info 121 node1/lrm: service ct:109 - end relocate to node 'node3' +info 140 node1/crm: service 'ct:104': state changed from 'relocate' to 'started' (node = node3) +info 140 node1/crm: service 'ct:105': state changed from 'relocate' to 'started' (node = node3) +info 140 node1/crm: service 'ct:106': state changed from 'relocate' to 'started' (node = node3) +info 140 node1/crm: service 'ct:107': state changed from 'relocate' to 'started' (node = node3) +info 140 node1/crm: service 'ct:108': state changed from 'relocate' to 'started' (node = node3) +info 140 node1/crm: service 'ct:109': state changed from 'relocate' to 'started' (node = node3) +info 142 node1/lrm: exit (loop end) +info 142 shutdown: execute crm node1 stop +info 141 node1/crm: server received shutdown request +info 145 node3/lrm: starting service ct:104 +info 145 node3/lrm: service status ct:104 started +info 145 node3/lrm: starting service ct:105 +info 145 node3/lrm: service status ct:105 started +info 145 node3/lrm: starting service ct:106 +info 145 node3/lrm: service status ct:106 started +info 145 node3/lrm: starting service ct:107 +info 145 node3/lrm: service status ct:107 started +info 145 node3/lrm: starting service ct:108 +info 145 node3/lrm: service status ct:108 started +info 145 node3/lrm: starting service ct:109 +info 145 node3/lrm: service status ct:109 started +info 160 node1/crm: voluntary release CRM lock +info 161 node1/crm: exit (loop end) +info 161 shutdown: execute power node1 off +info 161 node2/crm: got lock 'ha_manager_lock' +info 161 node2/crm: status change slave => master +info 161 node2/crm: using scheduler mode 'static' +info 720 hardware: exit simulation - done diff --git a/src/test/test-crs-static5/manager_status b/src/test/test-crs-static5/manager_status new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/src/test/test-crs-static5/manager_status @@ -0,0 +1 @@ +{} diff --git a/src/test/test-crs-static5/service_config b/src/test/test-crs-static5/service_config new file mode 100644 index 0000000..43c5f60 --- /dev/null +++ b/src/test/test-crs-static5/service_config @@ -0,0 +1,10 @@ +{ + "ct:102": { "node": "node2", "state": "enabled" }, + "ct:103": { "node": "node3", "state": "enabled" }, + "ct:104": { "node": "node1", "state": "enabled" }, + "ct:105": { "node": "node1", "state": "enabled" }, + "ct:106": { "node": "node1", "state": "enabled" }, + "ct:107": { "node": "node1", "state": "enabled" }, + "ct:108": { "node": "node1", "state": "enabled" }, + "ct:109": { "node": "node1", "state": "enabled" } +} diff --git a/src/test/test-crs-static5/static_service_stats b/src/test/test-crs-static5/static_service_stats new file mode 100644 index 0000000..6293f63 --- /dev/null +++ b/src/test/test-crs-static5/static_service_stats @@ -0,0 +1,11 @@ +{ + "ct:101": { "maxcpu": 0, "maxmem": 40000000000 }, + "ct:102": { "maxcpu": 0.5, "maxmem": 40000000000 }, + "ct:103": { "maxcpu": 0.5, "maxmem": 200000000 }, + "ct:104": { "maxcpu": 0.5, "maxmem": 200000000 }, + "ct:105": { "maxcpu": 0.5, "maxmem": 200000000 }, + "ct:106": { "maxcpu": 0.5, "maxmem": 200000000 }, + "ct:107": { "maxcpu": 0.5, "maxmem": 200000000 }, + "ct:108": { "maxcpu": 0.5, "maxmem": 200000000 }, + "ct:109": { "maxcpu": 0.5, "maxmem": 200000000 } +}