From 314ef2579e25298a74f05915fb1fe1b43cfad633 Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Sat, 19 Nov 2022 15:49:50 +0100 Subject: [PATCH] request start: allow to auto-rebalance on a new start request Signed-off-by: Thomas Lamprecht --- src/PVE/HA/LRM.pm | 2 +- src/PVE/HA/Manager.pm | 36 +- src/test/test-crs-static-rebalance1/README | 1 + src/test/test-crs-static-rebalance1/cmdlist | 4 + .../test-crs-static-rebalance1/datacenter.cfg | 7 + .../hardware_status | 5 + .../test-crs-static-rebalance1/log.expect | 489 ++++++++++++++++++ .../test-crs-static-rebalance1/manager_status | 1 + .../test-crs-static-rebalance1/service_config | 53 ++ .../static_service_stats | 3 + 10 files changed, 596 insertions(+), 5 deletions(-) create mode 100644 src/test/test-crs-static-rebalance1/README create mode 100644 src/test/test-crs-static-rebalance1/cmdlist create mode 100644 src/test/test-crs-static-rebalance1/datacenter.cfg create mode 100644 src/test/test-crs-static-rebalance1/hardware_status create mode 100644 src/test/test-crs-static-rebalance1/log.expect create mode 100644 src/test/test-crs-static-rebalance1/manager_status create mode 100644 src/test/test-crs-static-rebalance1/service_config create mode 100644 src/test/test-crs-static-rebalance1/static_service_stats diff --git a/src/PVE/HA/LRM.pm b/src/PVE/HA/LRM.pm index 828ba97..55ef250 100644 --- a/src/PVE/HA/LRM.pm +++ b/src/PVE/HA/LRM.pm @@ -906,7 +906,7 @@ sub exec_resource_agent { return ERROR; } - } elsif ($cmd eq 'migrate' || $cmd eq 'relocate') { + } elsif ($cmd eq 'migrate' || $cmd eq 'relocate' || $cmd eq 'request_start_balance') { my $target = $params->{target}; if (!defined($target)) { diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm index e6c4215..d325f06 100644 --- a/src/PVE/HA/Manager.pm +++ b/src/PVE/HA/Manager.pm @@ -64,6 +64,8 @@ sub update_crs_scheduler_mode { my $haenv = $self->{haenv}; my $dc_cfg = $haenv->get_datacenter_settings(); + $self->{crs}->{rebalance_on_request_start} = !!$dc_cfg->{crs}->{'ha-rebalance-on-start'}; + my $old_mode = $self->{crs}->{scheduler}; my $new_mode = $dc_cfg->{crs}->{ha} || 'basic'; @@ -210,6 +212,7 @@ my $valid_service_states = { stopped => 1, request_stop => 1, request_start => 1, + request_start_balance => 1, started => 1, fence => 1, recovery => 1, @@ -262,10 +265,11 @@ sub recompute_online_node_usage { || $state eq 'freeze' || $state eq 'error' || $state eq 'recovery' ) { $online_node_usage->add_service_usage_to_node($sd->{node}, $sid, $sd->{node}); - } elsif (($state eq 'migrate') || ($state eq 'relocate')) { + } elsif ($state eq 'migrate' || $state eq 'relocate' || $state eq 'request_start_balance') { my $source = $sd->{node}; # count it for both, source and target as load is put on both - $online_node_usage->add_service_usage_to_node($source, $sid, $source, $target); + $online_node_usage->add_service_usage_to_node($source, $sid, $source, $target) + if $state ne 'request_start_balance'; $online_node_usage->add_service_usage_to_node($target, $sid, $source, $target); } elsif ($state eq 'stopped' || $state eq 'request_start') { # do nothing @@ -495,7 +499,7 @@ sub manage { $self->next_state_request_start($sid, $cd, $sd, $lrm_res); - } elsif ($last_state eq 'migrate' || $last_state eq 'relocate') { + } elsif ($last_state eq 'migrate' || $last_state eq 'relocate' || $last_state eq 'request_start_balance') { $self->next_state_migrate_relocate($sid, $cd, $sd, $lrm_res); @@ -696,7 +700,31 @@ sub next_state_stopped { sub next_state_request_start { my ($self, $sid, $cd, $sd, $lrm_res) = @_; - $change_service_state->($self, $sid, 'started', node => $sd->{node}); + my $haenv = $self->{haenv}; + my $current_node = $sd->{node}; + + if ($self->{crs}->{rebalance_on_request_start}) { + my $selected_node = select_service_node( + $self->{groups}, + $self->{online_node_usage}, + $sid, + $cd, + $sd->{node}, + 0, # try_next + $sd->{failed_nodes}, + $sd->{maintenance_node}, + 1, # best_score + ); + my $select_text = $selected_node ne $current_node ? 'new' : 'current'; + $haenv->log('info', "service $sid: re-balance selected $select_text node $selected_node for startup"); + + if ($selected_node ne $current_node) { + $change_service_state->($self, $sid, 'request_start_balance', node => $current_node, target => $selected_node); + return; + } + } + + $change_service_state->($self, $sid, 'started', node => $current_node); } sub record_service_failed_on_node { diff --git a/src/test/test-crs-static-rebalance1/README b/src/test/test-crs-static-rebalance1/README new file mode 100644 index 0000000..de6a72f --- /dev/null +++ b/src/test/test-crs-static-rebalance1/README @@ -0,0 +1 @@ +Test failover and how recovery gets balanced out after single node network failure. diff --git a/src/test/test-crs-static-rebalance1/cmdlist b/src/test/test-crs-static-rebalance1/cmdlist new file mode 100644 index 0000000..eee0e40 --- /dev/null +++ b/src/test/test-crs-static-rebalance1/cmdlist @@ -0,0 +1,4 @@ +[ + [ "power node1 on", "power node2 on", "power node3 on"], + [ "network node3 off" ] +] diff --git a/src/test/test-crs-static-rebalance1/datacenter.cfg b/src/test/test-crs-static-rebalance1/datacenter.cfg new file mode 100644 index 0000000..9f5137b --- /dev/null +++ b/src/test/test-crs-static-rebalance1/datacenter.cfg @@ -0,0 +1,7 @@ +{ + "crs": { + "ha": "static", + "ha-rebalance-on-start": 1 + } +} + diff --git a/src/test/test-crs-static-rebalance1/hardware_status b/src/test/test-crs-static-rebalance1/hardware_status new file mode 100644 index 0000000..651ad79 --- /dev/null +++ b/src/test/test-crs-static-rebalance1/hardware_status @@ -0,0 +1,5 @@ +{ + "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 256000000000 }, + "node2": { "power": "off", "network": "off", "cpus": 32, "memory": 256000000000 }, + "node3": { "power": "off", "network": "off", "cpus": 32, "memory": 256000000000 } +} diff --git a/src/test/test-crs-static-rebalance1/log.expect b/src/test/test-crs-static-rebalance1/log.expect new file mode 100644 index 0000000..da3ed5b --- /dev/null +++ b/src/test/test-crs-static-rebalance1/log.expect @@ -0,0 +1,489 @@ +info 0 hardware: starting simulation +info 20 cmdlist: execute power node1 on +info 20 node1/crm: status change startup => wait_for_quorum +info 20 node1/lrm: status change startup => wait_for_agent_lock +info 20 cmdlist: execute power node2 on +info 20 node2/crm: status change startup => wait_for_quorum +info 20 node2/lrm: status change startup => wait_for_agent_lock +info 20 cmdlist: execute power node3 on +info 20 node3/crm: status change startup => wait_for_quorum +info 20 node3/lrm: status change startup => wait_for_agent_lock +info 20 node1/crm: got lock 'ha_manager_lock' +info 20 node1/crm: status change wait_for_quorum => master +info 20 node1/crm: using scheduler mode 'static' +info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online' +info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online' +info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online' +info 20 node1/crm: adding new service 'vm:101' on node 'node1' +info 20 node1/crm: adding new service 'vm:102' on node 'node2' +info 20 node1/crm: adding new service 'vm:103' on node 'node3' +info 20 node1/crm: adding new service 'vm:104' on node 'node3' +info 20 node1/crm: adding new service 'vm:105' on node 'node3' +info 20 node1/crm: adding new service 'vm:106' on node 'node3' +info 20 node1/crm: adding new service 'vm:107' on node 'node3' +info 20 node1/crm: adding new service 'vm:108' on node 'node3' +info 20 node1/crm: adding new service 'vm:109' on node 'node3' +info 20 node1/crm: adding new service 'vm:110' on node 'node3' +info 20 node1/crm: adding new service 'vm:111' on node 'node3' +info 20 node1/crm: adding new service 'vm:112' on node 'node3' +info 20 node1/crm: adding new service 'vm:113' on node 'node3' +info 20 node1/crm: adding new service 'vm:114' on node 'node3' +info 20 node1/crm: adding new service 'vm:115' on node 'node3' +info 20 node1/crm: adding new service 'vm:116' on node 'node3' +info 20 node1/crm: adding new service 'vm:117' on node 'node3' +info 20 node1/crm: adding new service 'vm:118' on node 'node3' +info 20 node1/crm: adding new service 'vm:119' on node 'node3' +info 20 node1/crm: adding new service 'vm:120' on node 'node3' +info 20 node1/crm: adding new service 'vm:121' on node 'node3' +info 20 node1/crm: adding new service 'vm:122' on node 'node3' +info 20 node1/crm: adding new service 'vm:123' on node 'node3' +info 20 node1/crm: adding new service 'vm:124' on node 'node3' +info 20 node1/crm: adding new service 'vm:125' on node 'node3' +info 20 node1/crm: adding new service 'vm:126' on node 'node3' +info 20 node1/crm: adding new service 'vm:127' on node 'node3' +info 20 node1/crm: adding new service 'vm:128' on node 'node3' +info 20 node1/crm: adding new service 'vm:129' on node 'node3' +info 20 node1/crm: adding new service 'vm:130' on node 'node3' +info 20 node1/crm: adding new service 'vm:131' on node 'node3' +info 20 node1/crm: adding new service 'vm:132' on node 'node3' +info 20 node1/crm: adding new service 'vm:133' on node 'node3' +info 20 node1/crm: adding new service 'vm:134' on node 'node3' +info 20 node1/crm: adding new service 'vm:135' on node 'node3' +info 20 node1/crm: adding new service 'vm:136' on node 'node3' +info 20 node1/crm: adding new service 'vm:137' on node 'node3' +info 20 node1/crm: adding new service 'vm:138' on node 'node3' +info 20 node1/crm: adding new service 'vm:139' on node 'node3' +info 20 node1/crm: adding new service 'vm:140' on node 'node3' +info 20 node1/crm: adding new service 'vm:141' on node 'node3' +info 20 node1/crm: adding new service 'vm:142' on node 'node3' +info 20 node1/crm: adding new service 'vm:143' on node 'node3' +info 20 node1/crm: adding new service 'vm:144' on node 'node3' +info 20 node1/crm: adding new service 'vm:145' on node 'node3' +info 20 node1/crm: adding new service 'vm:146' on node 'node3' +info 20 node1/crm: adding new service 'vm:147' on node 'node3' +info 20 node1/crm: adding new service 'vm:148' on node 'node3' +info 20 node1/crm: adding new service 'vm:149' on node 'node3' +info 20 node1/crm: adding new service 'vm:150' on node 'node3' +info 20 node1/crm: adding new service 'vm:151' on node 'node3' +info 20 node1/crm: service vm:101: re-balance selected current node node1 for startup +info 20 node1/crm: service 'vm:101': state changed from 'request_start' to 'started' (node = node1) +info 20 node1/crm: service vm:103: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:103': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:104: re-balance selected new node node1 for startup +info 20 node1/crm: service 'vm:104': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node1) +info 20 node1/crm: service vm:105: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:105': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:106: re-balance selected new node node2 for startup +info 20 node1/crm: service 'vm:106': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node2) +info 20 node1/crm: service vm:107: re-balance selected new node node1 for startup +info 20 node1/crm: service 'vm:107': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node1) +info 20 node1/crm: service vm:108: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:108': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:109: re-balance selected new node node2 for startup +info 20 node1/crm: service 'vm:109': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node2) +info 20 node1/crm: service vm:110: re-balance selected new node node1 for startup +info 20 node1/crm: service 'vm:110': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node1) +info 20 node1/crm: service vm:111: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:111': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:112: re-balance selected new node node2 for startup +info 20 node1/crm: service 'vm:112': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node2) +info 20 node1/crm: service vm:113: re-balance selected new node node1 for startup +info 20 node1/crm: service 'vm:113': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node1) +info 20 node1/crm: service vm:114: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:114': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:115: re-balance selected new node node2 for startup +info 20 node1/crm: service 'vm:115': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node2) +info 20 node1/crm: service vm:116: re-balance selected new node node1 for startup +info 20 node1/crm: service 'vm:116': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node1) +info 20 node1/crm: service vm:117: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:117': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:118: re-balance selected new node node2 for startup +info 20 node1/crm: service 'vm:118': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node2) +info 20 node1/crm: service vm:119: re-balance selected new node node1 for startup +info 20 node1/crm: service 'vm:119': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node1) +info 20 node1/crm: service vm:120: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:120': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:121: re-balance selected new node node2 for startup +info 20 node1/crm: service 'vm:121': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node2) +info 20 node1/crm: service vm:122: re-balance selected new node node1 for startup +info 20 node1/crm: service 'vm:122': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node1) +info 20 node1/crm: service vm:123: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:123': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:124: re-balance selected new node node2 for startup +info 20 node1/crm: service 'vm:124': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node2) +info 20 node1/crm: service vm:125: re-balance selected new node node1 for startup +info 20 node1/crm: service 'vm:125': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node1) +info 20 node1/crm: service vm:126: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:126': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:127: re-balance selected new node node2 for startup +info 20 node1/crm: service 'vm:127': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node2) +info 20 node1/crm: service vm:128: re-balance selected new node node1 for startup +info 20 node1/crm: service 'vm:128': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node1) +info 20 node1/crm: service vm:129: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:129': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:130: re-balance selected new node node2 for startup +info 20 node1/crm: service 'vm:130': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node2) +info 20 node1/crm: service vm:131: re-balance selected new node node1 for startup +info 20 node1/crm: service 'vm:131': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node1) +info 20 node1/crm: service vm:132: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:132': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:133: re-balance selected new node node2 for startup +info 20 node1/crm: service 'vm:133': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node2) +info 20 node1/crm: service vm:134: re-balance selected new node node1 for startup +info 20 node1/crm: service 'vm:134': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node1) +info 20 node1/crm: service vm:135: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:135': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:136: re-balance selected new node node2 for startup +info 20 node1/crm: service 'vm:136': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node2) +info 20 node1/crm: service vm:137: re-balance selected new node node1 for startup +info 20 node1/crm: service 'vm:137': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node1) +info 20 node1/crm: service vm:138: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:138': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:139: re-balance selected new node node2 for startup +info 20 node1/crm: service 'vm:139': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node2) +info 20 node1/crm: service vm:140: re-balance selected new node node1 for startup +info 20 node1/crm: service 'vm:140': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node1) +info 20 node1/crm: service vm:141: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:141': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:142: re-balance selected new node node2 for startup +info 20 node1/crm: service 'vm:142': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node2) +info 20 node1/crm: service vm:143: re-balance selected new node node1 for startup +info 20 node1/crm: service 'vm:143': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node1) +info 20 node1/crm: service vm:144: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:144': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:145: re-balance selected new node node2 for startup +info 20 node1/crm: service 'vm:145': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node2) +info 20 node1/crm: service vm:146: re-balance selected new node node1 for startup +info 20 node1/crm: service 'vm:146': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node1) +info 20 node1/crm: service vm:147: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:147': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:148: re-balance selected new node node2 for startup +info 20 node1/crm: service 'vm:148': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node2) +info 20 node1/crm: service vm:149: re-balance selected new node node1 for startup +info 20 node1/crm: service 'vm:149': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node1) +info 20 node1/crm: service vm:150: re-balance selected current node node3 for startup +info 20 node1/crm: service 'vm:150': state changed from 'request_start' to 'started' (node = node3) +info 20 node1/crm: service vm:151: re-balance selected new node node2 for startup +info 20 node1/crm: service 'vm:151': state changed from 'request_start' to 'request_start_balance' (node = node3, target = node2) +info 21 node1/lrm: got lock 'ha_agent_node1_lock' +info 21 node1/lrm: status change wait_for_agent_lock => active +info 21 node1/lrm: starting service vm:101 +info 21 node1/lrm: service status vm:101 started +info 22 node2/crm: status change wait_for_quorum => slave +info 23 node2/lrm: got lock 'ha_agent_node2_lock' +info 23 node2/lrm: status change wait_for_agent_lock => active +info 24 node3/crm: status change wait_for_quorum => slave +info 25 node3/lrm: got lock 'ha_agent_node3_lock' +info 25 node3/lrm: status change wait_for_agent_lock => active +info 25 node3/lrm: starting service vm:103 +info 25 node3/lrm: service status vm:103 started +info 25 node3/lrm: service vm:104 - start relocate to node 'node1' +info 25 node3/lrm: service vm:104 - end relocate to node 'node1' +info 25 node3/lrm: starting service vm:105 +info 25 node3/lrm: service status vm:105 started +info 25 node3/lrm: service vm:106 - start relocate to node 'node2' +info 25 node3/lrm: service vm:106 - end relocate to node 'node2' +info 25 node3/lrm: service vm:107 - start relocate to node 'node1' +info 25 node3/lrm: service vm:107 - end relocate to node 'node1' +info 25 node3/lrm: starting service vm:108 +info 25 node3/lrm: service status vm:108 started +info 25 node3/lrm: service vm:109 - start relocate to node 'node2' +info 25 node3/lrm: service vm:109 - end relocate to node 'node2' +info 25 node3/lrm: service vm:110 - start relocate to node 'node1' +info 25 node3/lrm: service vm:110 - end relocate to node 'node1' +info 25 node3/lrm: starting service vm:111 +info 25 node3/lrm: service status vm:111 started +info 25 node3/lrm: service vm:112 - start relocate to node 'node2' +info 25 node3/lrm: service vm:112 - end relocate to node 'node2' +info 25 node3/lrm: service vm:113 - start relocate to node 'node1' +info 25 node3/lrm: service vm:113 - end relocate to node 'node1' +info 25 node3/lrm: starting service vm:114 +info 25 node3/lrm: service status vm:114 started +info 25 node3/lrm: service vm:115 - start relocate to node 'node2' +info 25 node3/lrm: service vm:115 - end relocate to node 'node2' +info 25 node3/lrm: service vm:116 - start relocate to node 'node1' +info 25 node3/lrm: service vm:116 - end relocate to node 'node1' +info 25 node3/lrm: starting service vm:117 +info 25 node3/lrm: service status vm:117 started +info 25 node3/lrm: service vm:118 - start relocate to node 'node2' +info 25 node3/lrm: service vm:118 - end relocate to node 'node2' +info 25 node3/lrm: service vm:119 - start relocate to node 'node1' +info 25 node3/lrm: service vm:119 - end relocate to node 'node1' +info 25 node3/lrm: starting service vm:120 +info 25 node3/lrm: service status vm:120 started +info 25 node3/lrm: service vm:121 - start relocate to node 'node2' +info 25 node3/lrm: service vm:121 - end relocate to node 'node2' +info 25 node3/lrm: service vm:122 - start relocate to node 'node1' +info 25 node3/lrm: service vm:122 - end relocate to node 'node1' +info 25 node3/lrm: starting service vm:123 +info 25 node3/lrm: service status vm:123 started +info 25 node3/lrm: service vm:124 - start relocate to node 'node2' +info 25 node3/lrm: service vm:124 - end relocate to node 'node2' +info 25 node3/lrm: service vm:125 - start relocate to node 'node1' +info 25 node3/lrm: service vm:125 - end relocate to node 'node1' +info 25 node3/lrm: starting service vm:126 +info 25 node3/lrm: service status vm:126 started +info 25 node3/lrm: service vm:127 - start relocate to node 'node2' +info 25 node3/lrm: service vm:127 - end relocate to node 'node2' +info 25 node3/lrm: service vm:128 - start relocate to node 'node1' +info 25 node3/lrm: service vm:128 - end relocate to node 'node1' +info 25 node3/lrm: starting service vm:129 +info 25 node3/lrm: service status vm:129 started +info 25 node3/lrm: service vm:130 - start relocate to node 'node2' +info 25 node3/lrm: service vm:130 - end relocate to node 'node2' +info 25 node3/lrm: service vm:131 - start relocate to node 'node1' +info 25 node3/lrm: service vm:131 - end relocate to node 'node1' +info 25 node3/lrm: starting service vm:132 +info 25 node3/lrm: service status vm:132 started +info 25 node3/lrm: service vm:133 - start relocate to node 'node2' +info 25 node3/lrm: service vm:133 - end relocate to node 'node2' +info 25 node3/lrm: service vm:134 - start relocate to node 'node1' +info 25 node3/lrm: service vm:134 - end relocate to node 'node1' +info 25 node3/lrm: starting service vm:135 +info 25 node3/lrm: service status vm:135 started +info 25 node3/lrm: service vm:136 - start relocate to node 'node2' +info 25 node3/lrm: service vm:136 - end relocate to node 'node2' +info 25 node3/lrm: service vm:137 - start relocate to node 'node1' +info 25 node3/lrm: service vm:137 - end relocate to node 'node1' +info 25 node3/lrm: starting service vm:138 +info 25 node3/lrm: service status vm:138 started +info 25 node3/lrm: service vm:139 - start relocate to node 'node2' +info 25 node3/lrm: service vm:139 - end relocate to node 'node2' +info 25 node3/lrm: service vm:140 - start relocate to node 'node1' +info 25 node3/lrm: service vm:140 - end relocate to node 'node1' +info 25 node3/lrm: starting service vm:141 +info 25 node3/lrm: service status vm:141 started +info 25 node3/lrm: service vm:142 - start relocate to node 'node2' +info 25 node3/lrm: service vm:142 - end relocate to node 'node2' +info 25 node3/lrm: service vm:143 - start relocate to node 'node1' +info 25 node3/lrm: service vm:143 - end relocate to node 'node1' +info 25 node3/lrm: starting service vm:144 +info 25 node3/lrm: service status vm:144 started +info 25 node3/lrm: service vm:145 - start relocate to node 'node2' +info 25 node3/lrm: service vm:145 - end relocate to node 'node2' +info 25 node3/lrm: service vm:146 - start relocate to node 'node1' +info 25 node3/lrm: service vm:146 - end relocate to node 'node1' +info 25 node3/lrm: starting service vm:147 +info 25 node3/lrm: service status vm:147 started +info 25 node3/lrm: service vm:148 - start relocate to node 'node2' +info 25 node3/lrm: service vm:148 - end relocate to node 'node2' +info 25 node3/lrm: service vm:149 - start relocate to node 'node1' +info 25 node3/lrm: service vm:149 - end relocate to node 'node1' +info 25 node3/lrm: starting service vm:150 +info 25 node3/lrm: service status vm:150 started +info 25 node3/lrm: service vm:151 - start relocate to node 'node2' +info 25 node3/lrm: service vm:151 - end relocate to node 'node2' +info 40 node1/crm: service 'vm:102': state changed from 'request_stop' to 'stopped' +info 40 node1/crm: service 'vm:104': state changed from 'request_start_balance' to 'started' (node = node1) +info 40 node1/crm: service 'vm:106': state changed from 'request_start_balance' to 'started' (node = node2) +info 40 node1/crm: service 'vm:107': state changed from 'request_start_balance' to 'started' (node = node1) +info 40 node1/crm: service 'vm:109': state changed from 'request_start_balance' to 'started' (node = node2) +info 40 node1/crm: service 'vm:110': state changed from 'request_start_balance' to 'started' (node = node1) +info 40 node1/crm: service 'vm:112': state changed from 'request_start_balance' to 'started' (node = node2) +info 40 node1/crm: service 'vm:113': state changed from 'request_start_balance' to 'started' (node = node1) +info 40 node1/crm: service 'vm:115': state changed from 'request_start_balance' to 'started' (node = node2) +info 40 node1/crm: service 'vm:116': state changed from 'request_start_balance' to 'started' (node = node1) +info 40 node1/crm: service 'vm:118': state changed from 'request_start_balance' to 'started' (node = node2) +info 40 node1/crm: service 'vm:119': state changed from 'request_start_balance' to 'started' (node = node1) +info 40 node1/crm: service 'vm:121': state changed from 'request_start_balance' to 'started' (node = node2) +info 40 node1/crm: service 'vm:122': state changed from 'request_start_balance' to 'started' (node = node1) +info 40 node1/crm: service 'vm:124': state changed from 'request_start_balance' to 'started' (node = node2) +info 40 node1/crm: service 'vm:125': state changed from 'request_start_balance' to 'started' (node = node1) +info 40 node1/crm: service 'vm:127': state changed from 'request_start_balance' to 'started' (node = node2) +info 40 node1/crm: service 'vm:128': state changed from 'request_start_balance' to 'started' (node = node1) +info 40 node1/crm: service 'vm:130': state changed from 'request_start_balance' to 'started' (node = node2) +info 40 node1/crm: service 'vm:131': state changed from 'request_start_balance' to 'started' (node = node1) +info 40 node1/crm: service 'vm:133': state changed from 'request_start_balance' to 'started' (node = node2) +info 40 node1/crm: service 'vm:134': state changed from 'request_start_balance' to 'started' (node = node1) +info 40 node1/crm: service 'vm:136': state changed from 'request_start_balance' to 'started' (node = node2) +info 40 node1/crm: service 'vm:137': state changed from 'request_start_balance' to 'started' (node = node1) +info 40 node1/crm: service 'vm:139': state changed from 'request_start_balance' to 'started' (node = node2) +info 40 node1/crm: service 'vm:140': state changed from 'request_start_balance' to 'started' (node = node1) +info 40 node1/crm: service 'vm:142': state changed from 'request_start_balance' to 'started' (node = node2) +info 40 node1/crm: service 'vm:143': state changed from 'request_start_balance' to 'started' (node = node1) +info 40 node1/crm: service 'vm:145': state changed from 'request_start_balance' to 'started' (node = node2) +info 40 node1/crm: service 'vm:146': state changed from 'request_start_balance' to 'started' (node = node1) +info 40 node1/crm: service 'vm:148': state changed from 'request_start_balance' to 'started' (node = node2) +info 40 node1/crm: service 'vm:149': state changed from 'request_start_balance' to 'started' (node = node1) +info 40 node1/crm: service 'vm:151': state changed from 'request_start_balance' to 'started' (node = node2) +info 41 node1/lrm: starting service vm:104 +info 41 node1/lrm: service status vm:104 started +info 41 node1/lrm: starting service vm:107 +info 41 node1/lrm: service status vm:107 started +info 41 node1/lrm: starting service vm:110 +info 41 node1/lrm: service status vm:110 started +info 41 node1/lrm: starting service vm:113 +info 41 node1/lrm: service status vm:113 started +info 41 node1/lrm: starting service vm:116 +info 41 node1/lrm: service status vm:116 started +info 41 node1/lrm: starting service vm:119 +info 41 node1/lrm: service status vm:119 started +info 41 node1/lrm: starting service vm:122 +info 41 node1/lrm: service status vm:122 started +info 41 node1/lrm: starting service vm:125 +info 41 node1/lrm: service status vm:125 started +info 41 node1/lrm: starting service vm:128 +info 41 node1/lrm: service status vm:128 started +info 41 node1/lrm: starting service vm:131 +info 41 node1/lrm: service status vm:131 started +info 41 node1/lrm: starting service vm:134 +info 41 node1/lrm: service status vm:134 started +info 41 node1/lrm: starting service vm:137 +info 41 node1/lrm: service status vm:137 started +info 41 node1/lrm: starting service vm:140 +info 41 node1/lrm: service status vm:140 started +info 41 node1/lrm: starting service vm:143 +info 41 node1/lrm: service status vm:143 started +info 41 node1/lrm: starting service vm:146 +info 41 node1/lrm: service status vm:146 started +info 41 node1/lrm: starting service vm:149 +info 41 node1/lrm: service status vm:149 started +info 43 node2/lrm: starting service vm:106 +info 43 node2/lrm: service status vm:106 started +info 43 node2/lrm: starting service vm:109 +info 43 node2/lrm: service status vm:109 started +info 43 node2/lrm: starting service vm:112 +info 43 node2/lrm: service status vm:112 started +info 43 node2/lrm: starting service vm:115 +info 43 node2/lrm: service status vm:115 started +info 43 node2/lrm: starting service vm:118 +info 43 node2/lrm: service status vm:118 started +info 43 node2/lrm: starting service vm:121 +info 43 node2/lrm: service status vm:121 started +info 43 node2/lrm: starting service vm:124 +info 43 node2/lrm: service status vm:124 started +info 43 node2/lrm: starting service vm:127 +info 43 node2/lrm: service status vm:127 started +info 43 node2/lrm: starting service vm:130 +info 43 node2/lrm: service status vm:130 started +info 43 node2/lrm: starting service vm:133 +info 43 node2/lrm: service status vm:133 started +info 43 node2/lrm: starting service vm:136 +info 43 node2/lrm: service status vm:136 started +info 43 node2/lrm: starting service vm:139 +info 43 node2/lrm: service status vm:139 started +info 43 node2/lrm: starting service vm:142 +info 43 node2/lrm: service status vm:142 started +info 43 node2/lrm: starting service vm:145 +info 43 node2/lrm: service status vm:145 started +info 43 node2/lrm: starting service vm:148 +info 43 node2/lrm: service status vm:148 started +info 43 node2/lrm: starting service vm:151 +info 43 node2/lrm: service status vm:151 started +info 120 cmdlist: execute network node3 off +info 120 node1/crm: node 'node3': state changed from 'online' => 'unknown' +info 124 node3/crm: status change slave => wait_for_quorum +info 125 node3/lrm: status change active => lost_agent_lock +info 160 node1/crm: service 'vm:103': state changed from 'started' to 'fence' +info 160 node1/crm: service 'vm:105': state changed from 'started' to 'fence' +info 160 node1/crm: service 'vm:108': state changed from 'started' to 'fence' +info 160 node1/crm: service 'vm:111': state changed from 'started' to 'fence' +info 160 node1/crm: service 'vm:114': state changed from 'started' to 'fence' +info 160 node1/crm: service 'vm:117': state changed from 'started' to 'fence' +info 160 node1/crm: service 'vm:120': state changed from 'started' to 'fence' +info 160 node1/crm: service 'vm:123': state changed from 'started' to 'fence' +info 160 node1/crm: service 'vm:126': state changed from 'started' to 'fence' +info 160 node1/crm: service 'vm:129': state changed from 'started' to 'fence' +info 160 node1/crm: service 'vm:132': state changed from 'started' to 'fence' +info 160 node1/crm: service 'vm:135': state changed from 'started' to 'fence' +info 160 node1/crm: service 'vm:138': state changed from 'started' to 'fence' +info 160 node1/crm: service 'vm:141': state changed from 'started' to 'fence' +info 160 node1/crm: service 'vm:144': state changed from 'started' to 'fence' +info 160 node1/crm: service 'vm:147': state changed from 'started' to 'fence' +info 160 node1/crm: service 'vm:150': state changed from 'started' to 'fence' +info 160 node1/crm: node 'node3': state changed from 'unknown' => 'fence' +emai 160 node1/crm: FENCE: Try to fence node 'node3' +info 166 watchdog: execute power node3 off +info 165 node3/crm: killed by poweroff +info 166 node3/lrm: killed by poweroff +info 166 hardware: server 'node3' stopped by poweroff (watchdog) +info 240 node1/crm: got lock 'ha_agent_node3_lock' +info 240 node1/crm: fencing: acknowledged - got agent lock for node 'node3' +info 240 node1/crm: node 'node3': state changed from 'fence' => 'unknown' +emai 240 node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3' +info 240 node1/crm: service 'vm:103': state changed from 'fence' to 'recovery' +info 240 node1/crm: service 'vm:105': state changed from 'fence' to 'recovery' +info 240 node1/crm: service 'vm:108': state changed from 'fence' to 'recovery' +info 240 node1/crm: service 'vm:111': state changed from 'fence' to 'recovery' +info 240 node1/crm: service 'vm:114': state changed from 'fence' to 'recovery' +info 240 node1/crm: service 'vm:117': state changed from 'fence' to 'recovery' +info 240 node1/crm: service 'vm:120': state changed from 'fence' to 'recovery' +info 240 node1/crm: service 'vm:123': state changed from 'fence' to 'recovery' +info 240 node1/crm: service 'vm:126': state changed from 'fence' to 'recovery' +info 240 node1/crm: service 'vm:129': state changed from 'fence' to 'recovery' +info 240 node1/crm: service 'vm:132': state changed from 'fence' to 'recovery' +info 240 node1/crm: service 'vm:135': state changed from 'fence' to 'recovery' +info 240 node1/crm: service 'vm:138': state changed from 'fence' to 'recovery' +info 240 node1/crm: service 'vm:141': state changed from 'fence' to 'recovery' +info 240 node1/crm: service 'vm:144': state changed from 'fence' to 'recovery' +info 240 node1/crm: service 'vm:147': state changed from 'fence' to 'recovery' +info 240 node1/crm: service 'vm:150': state changed from 'fence' to 'recovery' +info 240 node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node2' +info 240 node1/crm: service 'vm:103': state changed from 'recovery' to 'started' (node = node2) +info 240 node1/crm: recover service 'vm:105' from fenced node 'node3' to node 'node1' +info 240 node1/crm: service 'vm:105': state changed from 'recovery' to 'started' (node = node1) +info 240 node1/crm: recover service 'vm:108' from fenced node 'node3' to node 'node1' +info 240 node1/crm: service 'vm:108': state changed from 'recovery' to 'started' (node = node1) +info 240 node1/crm: recover service 'vm:111' from fenced node 'node3' to node 'node1' +info 240 node1/crm: service 'vm:111': state changed from 'recovery' to 'started' (node = node1) +info 240 node1/crm: recover service 'vm:114' from fenced node 'node3' to node 'node1' +info 240 node1/crm: service 'vm:114': state changed from 'recovery' to 'started' (node = node1) +info 240 node1/crm: recover service 'vm:117' from fenced node 'node3' to node 'node2' +info 240 node1/crm: service 'vm:117': state changed from 'recovery' to 'started' (node = node2) +info 240 node1/crm: recover service 'vm:120' from fenced node 'node3' to node 'node1' +info 240 node1/crm: service 'vm:120': state changed from 'recovery' to 'started' (node = node1) +info 240 node1/crm: recover service 'vm:123' from fenced node 'node3' to node 'node2' +info 240 node1/crm: service 'vm:123': state changed from 'recovery' to 'started' (node = node2) +info 240 node1/crm: recover service 'vm:126' from fenced node 'node3' to node 'node1' +info 240 node1/crm: service 'vm:126': state changed from 'recovery' to 'started' (node = node1) +info 240 node1/crm: recover service 'vm:129' from fenced node 'node3' to node 'node2' +info 240 node1/crm: service 'vm:129': state changed from 'recovery' to 'started' (node = node2) +info 240 node1/crm: recover service 'vm:132' from fenced node 'node3' to node 'node1' +info 240 node1/crm: service 'vm:132': state changed from 'recovery' to 'started' (node = node1) +info 240 node1/crm: recover service 'vm:135' from fenced node 'node3' to node 'node2' +info 240 node1/crm: service 'vm:135': state changed from 'recovery' to 'started' (node = node2) +info 240 node1/crm: recover service 'vm:138' from fenced node 'node3' to node 'node1' +info 240 node1/crm: service 'vm:138': state changed from 'recovery' to 'started' (node = node1) +info 240 node1/crm: recover service 'vm:141' from fenced node 'node3' to node 'node2' +info 240 node1/crm: service 'vm:141': state changed from 'recovery' to 'started' (node = node2) +info 240 node1/crm: recover service 'vm:144' from fenced node 'node3' to node 'node1' +info 240 node1/crm: service 'vm:144': state changed from 'recovery' to 'started' (node = node1) +info 240 node1/crm: recover service 'vm:147' from fenced node 'node3' to node 'node2' +info 240 node1/crm: service 'vm:147': state changed from 'recovery' to 'started' (node = node2) +info 240 node1/crm: recover service 'vm:150' from fenced node 'node3' to node 'node1' +info 240 node1/crm: service 'vm:150': state changed from 'recovery' to 'started' (node = node1) +info 241 node1/lrm: starting service vm:105 +info 241 node1/lrm: service status vm:105 started +info 241 node1/lrm: starting service vm:108 +info 241 node1/lrm: service status vm:108 started +info 241 node1/lrm: starting service vm:111 +info 241 node1/lrm: service status vm:111 started +info 241 node1/lrm: starting service vm:114 +info 241 node1/lrm: service status vm:114 started +info 241 node1/lrm: starting service vm:120 +info 241 node1/lrm: service status vm:120 started +info 241 node1/lrm: starting service vm:126 +info 241 node1/lrm: service status vm:126 started +info 241 node1/lrm: starting service vm:132 +info 241 node1/lrm: service status vm:132 started +info 241 node1/lrm: starting service vm:138 +info 241 node1/lrm: service status vm:138 started +info 241 node1/lrm: starting service vm:144 +info 241 node1/lrm: service status vm:144 started +info 241 node1/lrm: starting service vm:150 +info 241 node1/lrm: service status vm:150 started +info 243 node2/lrm: starting service vm:103 +info 243 node2/lrm: service status vm:103 started +info 243 node2/lrm: starting service vm:117 +info 243 node2/lrm: service status vm:117 started +info 243 node2/lrm: starting service vm:123 +info 243 node2/lrm: service status vm:123 started +info 243 node2/lrm: starting service vm:129 +info 243 node2/lrm: service status vm:129 started +info 243 node2/lrm: starting service vm:135 +info 243 node2/lrm: service status vm:135 started +info 243 node2/lrm: starting service vm:141 +info 243 node2/lrm: service status vm:141 started +info 243 node2/lrm: starting service vm:147 +info 243 node2/lrm: service status vm:147 started +info 720 hardware: exit simulation - done diff --git a/src/test/test-crs-static-rebalance1/manager_status b/src/test/test-crs-static-rebalance1/manager_status new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/src/test/test-crs-static-rebalance1/manager_status @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/test/test-crs-static-rebalance1/service_config b/src/test/test-crs-static-rebalance1/service_config new file mode 100644 index 0000000..be098fb --- /dev/null +++ b/src/test/test-crs-static-rebalance1/service_config @@ -0,0 +1,53 @@ +{ + "vm:101": { "node": "node1", "state": "enabled" }, + "vm:102": { "node": "node2" }, + "vm:103": { "node": "node3", "state": "enabled" }, + "vm:104": { "node": "node3", "state": "enabled" }, + "vm:105": { "node": "node3", "state": "enabled" }, + "vm:106": { "node": "node3", "state": "enabled" }, + "vm:107": { "node": "node3", "state": "enabled" }, + "vm:108": { "node": "node3", "state": "enabled" }, + "vm:109": { "node": "node3", "state": "enabled" }, + "vm:110": { "node": "node3", "state": "enabled" }, + "vm:111": { "node": "node3", "state": "enabled" }, + "vm:112": { "node": "node3", "state": "enabled" }, + "vm:113": { "node": "node3", "state": "enabled" }, + "vm:114": { "node": "node3", "state": "enabled" }, + "vm:115": { "node": "node3", "state": "enabled" }, + "vm:116": { "node": "node3", "state": "enabled" }, + "vm:117": { "node": "node3", "state": "enabled" }, + "vm:118": { "node": "node3", "state": "enabled" }, + "vm:119": { "node": "node3", "state": "enabled" }, + "vm:120": { "node": "node3", "state": "enabled" }, + "vm:121": { "node": "node3", "state": "enabled" }, + "vm:122": { "node": "node3", "state": "enabled" }, + "vm:123": { "node": "node3", "state": "enabled" }, + "vm:124": { "node": "node3", "state": "enabled" }, + "vm:125": { "node": "node3", "state": "enabled" }, + "vm:126": { "node": "node3", "state": "enabled" }, + "vm:127": { "node": "node3", "state": "enabled" }, + "vm:128": { "node": "node3", "state": "enabled" }, + "vm:129": { "node": "node3", "state": "enabled" }, + "vm:130": { "node": "node3", "state": "enabled" }, + "vm:131": { "node": "node3", "state": "enabled" }, + "vm:132": { "node": "node3", "state": "enabled" }, + "vm:133": { "node": "node3", "state": "enabled" }, + "vm:134": { "node": "node3", "state": "enabled" }, + "vm:135": { "node": "node3", "state": "enabled" }, + "vm:136": { "node": "node3", "state": "enabled" }, + "vm:137": { "node": "node3", "state": "enabled" }, + "vm:138": { "node": "node3", "state": "enabled" }, + "vm:139": { "node": "node3", "state": "enabled" }, + "vm:140": { "node": "node3", "state": "enabled" }, + "vm:141": { "node": "node3", "state": "enabled" }, + "vm:142": { "node": "node3", "state": "enabled" }, + "vm:143": { "node": "node3", "state": "enabled" }, + "vm:144": { "node": "node3", "state": "enabled" }, + "vm:145": { "node": "node3", "state": "enabled" }, + "vm:146": { "node": "node3", "state": "enabled" }, + "vm:147": { "node": "node3", "state": "enabled" }, + "vm:148": { "node": "node3", "state": "enabled" }, + "vm:149": { "node": "node3", "state": "enabled" }, + "vm:150": { "node": "node3", "state": "enabled" }, + "vm:151": { "node": "node3", "state": "enabled" } +} diff --git a/src/test/test-crs-static-rebalance1/static_service_stats b/src/test/test-crs-static-rebalance1/static_service_stats new file mode 100644 index 0000000..7fb992d --- /dev/null +++ b/src/test/test-crs-static-rebalance1/static_service_stats @@ -0,0 +1,3 @@ +{ + "vm:102": { "maxcpu": 2, "maxmem": 4000000000 } +} -- 2.39.2