request start: allow to auto-rebalance on a new start request

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

diff --git a/src/PVE/HA/LRM.pm b/src/PVE/HA/LRM.pm
index 828ba97bff1d97ad65ddcc3cea705f6a2f9711bb..55ef250db2ba13e60c332597ad834cfa21d09074 100644 (file)
--- a/src/PVE/HA/LRM.pm
+++ b/src/PVE/HA/LRM.pm
@@ -906,7 +906,7 @@ sub exec_resource_agent {
            return ERROR;
        }
 
-    } elsif ($cmd eq 'migrate' || $cmd eq 'relocate') {
+    } elsif ($cmd eq 'migrate' || $cmd eq 'relocate' || $cmd eq 'request_start_balance') {
 
        my $target = $params->{target};
        if (!defined($target)) {

diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm
index e6c4215133a467a8c40608bd0877642f89e64dc8..d325f06848ec3b6ca4a15c85b092c53aeabafcbe 100644 (file)
--- a/src/PVE/HA/Manager.pm
+++ b/src/PVE/HA/Manager.pm
@@ -64,6 +64,8 @@ sub update_crs_scheduler_mode {
     my $haenv = $self->{haenv};
     my $dc_cfg = $haenv->get_datacenter_settings();
 
+    $self->{crs}->{rebalance_on_request_start} = !!$dc_cfg->{crs}->{'ha-rebalance-on-start'};
+
     my $old_mode = $self->{crs}->{scheduler};
     my $new_mode = $dc_cfg->{crs}->{ha} || 'basic';
 
@@ -210,6 +212,7 @@ my $valid_service_states = {
     stopped => 1,
     request_stop => 1,
     request_start => 1,
+    request_start_balance => 1,
     started => 1,
     fence => 1,
     recovery => 1,
@@ -262,10 +265,11 @@ sub recompute_online_node_usage {
                || $state eq 'freeze' || $state eq 'error' || $state eq 'recovery'
            ) {
                $online_node_usage->add_service_usage_to_node($sd->{node}, $sid, $sd->{node});
-           } elsif (($state eq 'migrate') || ($state eq 'relocate')) {
+           } elsif ($state eq 'migrate' || $state eq 'relocate' || $state eq 'request_start_balance') {
                my $source = $sd->{node};
                # count it for both, source and target as load is put on both
-               $online_node_usage->add_service_usage_to_node($source, $sid, $source, $target);
+               $online_node_usage->add_service_usage_to_node($source, $sid, $source, $target)
+                   if $state ne 'request_start_balance';
                $online_node_usage->add_service_usage_to_node($target, $sid, $source, $target);
            } elsif ($state eq 'stopped' || $state eq 'request_start') {
                # do nothing
@@ -495,7 +499,7 @@ sub manage {
 
                $self->next_state_request_start($sid, $cd, $sd, $lrm_res);
 
-           } elsif ($last_state eq 'migrate' || $last_state eq 'relocate') {
+           } elsif ($last_state eq 'migrate' || $last_state eq 'relocate' || $last_state eq 'request_start_balance') {
 
                $self->next_state_migrate_relocate($sid, $cd, $sd, $lrm_res);
 
@@ -696,7 +700,31 @@ sub next_state_stopped {
 sub next_state_request_start {
     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 
-    $change_service_state->($self, $sid, 'started', node => $sd->{node});
+    my $haenv = $self->{haenv};
+    my $current_node = $sd->{node};
+
+    if ($self->{crs}->{rebalance_on_request_start}) {
+       my $selected_node = select_service_node(
+           $self->{groups},
+           $self->{online_node_usage},
+           $sid,
+           $cd,
+           $sd->{node},
+           0, # try_next
+           $sd->{failed_nodes},
+           $sd->{maintenance_node},
+           1, # best_score
+       );
+       my $select_text = $selected_node ne $current_node ? 'new' : 'current';
+       $haenv->log('info', "service $sid: re-balance selected $select_text node $selected_node for startup");
+
+       if ($selected_node ne $current_node) {
+           $change_service_state->($self, $sid, 'request_start_balance', node => $current_node, target => $selected_node);
+           return;
+       }
+    }
+
+    $change_service_state->($self, $sid, 'started', node => $current_node);
 }
 
 sub record_service_failed_on_node {

diff --git a/src/test/test-crs-static-rebalance1/README b/src/test/test-crs-static-rebalance1/README
new file mode 100644 (file)
index 0000000..de6a72f
--- /dev/null
+++ b/src/test/test-crs-static-rebalance1/README
@@ -0,0 +1 @@
+Test failover and how recovery gets balanced out after single node network failure.

diff --git a/src/test/test-crs-static-rebalance1/cmdlist b/src/test/test-crs-static-rebalance1/cmdlist
new file mode 100644 (file)
index 0000000..eee0e40
--- /dev/null
+++ b/src/test/test-crs-static-rebalance1/cmdlist
@@ -0,0 +1,4 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "network node3 off" ]
+]

diff --git a/src/test/test-crs-static-rebalance1/datacenter.cfg b/src/test/test-crs-static-rebalance1/datacenter.cfg
new file mode 100644 (file)
index 0000000..9f5137b
--- /dev/null
+++ b/src/test/test-crs-static-rebalance1/datacenter.cfg
@@ -0,0 +1,7 @@
+{
+    "crs": {
+        "ha": "static",
+        "ha-rebalance-on-start": 1
+    }
+}
+

diff --git a/src/test/test-crs-static-rebalance1/hardware_status b/src/test/test-crs-static-rebalance1/hardware_status
new file mode 100644 (file)
index 0000000..651ad79
--- /dev/null
+++ b/src/test/test-crs-static-rebalance1/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 256000000000 },
+  "node2": { "power": "off", "network": "off", "cpus": 32, "memory": 256000000000 },
+  "node3": { "power": "off", "network": "off", "cpus": 32, "memory": 256000000000 }
+}

diff --git a/src/test/test-crs-static-rebalance1/log.expect b/src/test/test-crs-static-rebalance1/log.expect
new file mode 100644 (file)
index 0000000..da3ed5b
--- /dev/null
+++ b/src/test/test-crs-static-rebalance1/log.expect
@@ -0,0 +1,489 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: using scheduler mode 'static'
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     20    node1/crm: adding new service 'vm:104' on node 'node3'
+info     20    node1/crm: adding new service 'vm:105' on node 'node3'
+info     20    node1/crm: adding new service 'vm:106' on node 'node3'
+info     20    node1/crm: adding new service 'vm:107' on node 'node3'
+info     20    node1/crm: adding new service 'vm:108' on node 'node3'
+info     20    node1/crm: adding new service 'vm:109' on node 'node3'
+info     20    node1/crm: adding new service 'vm:110' on node 'node3'
+info     20    node1/crm: adding new service 'vm:111' on node 'node3'
+info     20    node1/crm: adding new service 'vm:112' on node 'node3'
+info     20    node1/crm: adding new service 'vm:113' on node 'node3'
+info     20    node1/crm: adding new service 'vm:114' on node 'node3'
+info     20    node1/crm: adding new service 'vm:115' on node 'node3'
+info     20    node1/crm: adding new service 'vm:116' on node 'node3'
+info     20    node1/crm: adding new service 'vm:117' on node 'node3'
+info     20    node1/crm: adding new service 'vm:118' on node 'node3'
+info     20    node1/crm: adding new service 'vm:119' on node 'node3'
+info     20    node1/crm: adding new service 'vm:120' on node 'node3'
+info     20    node1/crm: adding new service 'vm:121' on node 'node3'
+info     20    node1/crm: adding new service 'vm:122' on node 'node3'
+info     20    node1/crm: adding new service 'vm:123' on node 'node3'
+info     20    node1/crm: adding new service 'vm:124' on node 'node3'
+info     20    node1/crm: adding new service 'vm:125' on node 'node3'
+info     20    node1/crm: adding new service 'vm:126' on node 'node3'
+info     20    node1/crm: adding new service 'vm:127' on node 'node3'
+info     20    node1/crm: adding new service 'vm:128' on node 'node3'
+info     20    node1/crm: adding new service 'vm:129' on node 'node3'
+info     20    node1/crm: adding new service 'vm:130' on node 'node3'
+info     20    node1/crm: adding new service 'vm:131' on node 'node3'
+info     20    node1/crm: adding new service 'vm:132' on node 'node3'
+info     20    node1/crm: adding new service 'vm:133' on node 'node3'
+info     20    node1/crm: adding new service 'vm:134' on node 'node3'
+info     20    node1/crm: adding new service 'vm:135' on node 'node3'
+info     20    node1/crm: adding new service 'vm:136' on node 'node3'
+info     20    node1/crm: adding new service 'vm:137' on node 'node3'
+info     20    node1/crm: adding new service 'vm:138' on node 'node3'
+info     20    node1/crm: adding new service 'vm:139' on node 'node3'
+info     20    node1/crm: adding new service 'vm:140' on node 'node3'
+info     20    node1/crm: adding new service 'vm:141' on node 'node3'
+info     20    node1/crm: adding new service 'vm:142' on node 'node3'
+info     20    node1/crm: adding new service 'vm:143' on node 'node3'
+info     20    node1/crm: adding new service 'vm:144' on node 'node3'
+info     20    node1/crm: adding new service 'vm:145' on node 'node3'
+info     20    node1/crm: adding new service 'vm:146' on node 'node3'
+info     20    node1/crm: adding new service 'vm:147' on node 'node3'
+info     20    node1/crm: adding new service 'vm:148' on node 'node3'
+info     20    node1/crm: adding new service 'vm:149' on node 'node3'
+info     20    node1/crm: adding new service 'vm:150' on node 'node3'
+info     20    node1/crm: adding new service 'vm:151' on node 'node3'
+info     20    node1/crm: service vm:101: re-balance selected current node node1 for startup
+info     20    node1/crm: service 'vm:101': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service vm:103: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:103': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:104: re-balance selected new node node1 for startup
+info     20    node1/crm: service 'vm:104': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node1)
+info     20    node1/crm: service vm:105: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:105': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:106: re-balance selected new node node2 for startup
+info     20    node1/crm: service 'vm:106': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node2)
+info     20    node1/crm: service vm:107: re-balance selected new node node1 for startup
+info     20    node1/crm: service 'vm:107': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node1)
+info     20    node1/crm: service vm:108: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:108': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:109: re-balance selected new node node2 for startup
+info     20    node1/crm: service 'vm:109': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node2)
+info     20    node1/crm: service vm:110: re-balance selected new node node1 for startup
+info     20    node1/crm: service 'vm:110': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node1)
+info     20    node1/crm: service vm:111: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:111': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:112: re-balance selected new node node2 for startup
+info     20    node1/crm: service 'vm:112': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node2)
+info     20    node1/crm: service vm:113: re-balance selected new node node1 for startup
+info     20    node1/crm: service 'vm:113': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node1)
+info     20    node1/crm: service vm:114: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:114': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:115: re-balance selected new node node2 for startup
+info     20    node1/crm: service 'vm:115': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node2)
+info     20    node1/crm: service vm:116: re-balance selected new node node1 for startup
+info     20    node1/crm: service 'vm:116': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node1)
+info     20    node1/crm: service vm:117: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:117': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:118: re-balance selected new node node2 for startup
+info     20    node1/crm: service 'vm:118': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node2)
+info     20    node1/crm: service vm:119: re-balance selected new node node1 for startup
+info     20    node1/crm: service 'vm:119': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node1)
+info     20    node1/crm: service vm:120: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:120': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:121: re-balance selected new node node2 for startup
+info     20    node1/crm: service 'vm:121': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node2)
+info     20    node1/crm: service vm:122: re-balance selected new node node1 for startup
+info     20    node1/crm: service 'vm:122': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node1)
+info     20    node1/crm: service vm:123: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:123': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:124: re-balance selected new node node2 for startup
+info     20    node1/crm: service 'vm:124': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node2)
+info     20    node1/crm: service vm:125: re-balance selected new node node1 for startup
+info     20    node1/crm: service 'vm:125': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node1)
+info     20    node1/crm: service vm:126: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:126': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:127: re-balance selected new node node2 for startup
+info     20    node1/crm: service 'vm:127': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node2)
+info     20    node1/crm: service vm:128: re-balance selected new node node1 for startup
+info     20    node1/crm: service 'vm:128': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node1)
+info     20    node1/crm: service vm:129: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:129': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:130: re-balance selected new node node2 for startup
+info     20    node1/crm: service 'vm:130': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node2)
+info     20    node1/crm: service vm:131: re-balance selected new node node1 for startup
+info     20    node1/crm: service 'vm:131': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node1)
+info     20    node1/crm: service vm:132: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:132': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:133: re-balance selected new node node2 for startup
+info     20    node1/crm: service 'vm:133': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node2)
+info     20    node1/crm: service vm:134: re-balance selected new node node1 for startup
+info     20    node1/crm: service 'vm:134': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node1)
+info     20    node1/crm: service vm:135: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:135': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:136: re-balance selected new node node2 for startup
+info     20    node1/crm: service 'vm:136': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node2)
+info     20    node1/crm: service vm:137: re-balance selected new node node1 for startup
+info     20    node1/crm: service 'vm:137': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node1)
+info     20    node1/crm: service vm:138: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:138': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:139: re-balance selected new node node2 for startup
+info     20    node1/crm: service 'vm:139': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node2)
+info     20    node1/crm: service vm:140: re-balance selected new node node1 for startup
+info     20    node1/crm: service 'vm:140': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node1)
+info     20    node1/crm: service vm:141: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:141': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:142: re-balance selected new node node2 for startup
+info     20    node1/crm: service 'vm:142': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node2)
+info     20    node1/crm: service vm:143: re-balance selected new node node1 for startup
+info     20    node1/crm: service 'vm:143': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node1)
+info     20    node1/crm: service vm:144: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:144': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:145: re-balance selected new node node2 for startup
+info     20    node1/crm: service 'vm:145': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node2)
+info     20    node1/crm: service vm:146: re-balance selected new node node1 for startup
+info     20    node1/crm: service 'vm:146': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node1)
+info     20    node1/crm: service vm:147: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:147': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:148: re-balance selected new node node2 for startup
+info     20    node1/crm: service 'vm:148': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node2)
+info     20    node1/crm: service vm:149: re-balance selected new node node1 for startup
+info     20    node1/crm: service 'vm:149': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node1)
+info     20    node1/crm: service vm:150: re-balance selected current node node3 for startup
+info     20    node1/crm: service 'vm:150': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service vm:151: re-balance selected new node node2 for startup
+info     20    node1/crm: service 'vm:151': state changed from 'request_start' to 'request_start_balance'  (node = node3, target = node2)
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+info     25    node3/lrm: service vm:104 - start relocate to node 'node1'
+info     25    node3/lrm: service vm:104 - end relocate to node 'node1'
+info     25    node3/lrm: starting service vm:105
+info     25    node3/lrm: service status vm:105 started
+info     25    node3/lrm: service vm:106 - start relocate to node 'node2'
+info     25    node3/lrm: service vm:106 - end relocate to node 'node2'
+info     25    node3/lrm: service vm:107 - start relocate to node 'node1'
+info     25    node3/lrm: service vm:107 - end relocate to node 'node1'
+info     25    node3/lrm: starting service vm:108
+info     25    node3/lrm: service status vm:108 started
+info     25    node3/lrm: service vm:109 - start relocate to node 'node2'
+info     25    node3/lrm: service vm:109 - end relocate to node 'node2'
+info     25    node3/lrm: service vm:110 - start relocate to node 'node1'
+info     25    node3/lrm: service vm:110 - end relocate to node 'node1'
+info     25    node3/lrm: starting service vm:111
+info     25    node3/lrm: service status vm:111 started
+info     25    node3/lrm: service vm:112 - start relocate to node 'node2'
+info     25    node3/lrm: service vm:112 - end relocate to node 'node2'
+info     25    node3/lrm: service vm:113 - start relocate to node 'node1'
+info     25    node3/lrm: service vm:113 - end relocate to node 'node1'
+info     25    node3/lrm: starting service vm:114
+info     25    node3/lrm: service status vm:114 started
+info     25    node3/lrm: service vm:115 - start relocate to node 'node2'
+info     25    node3/lrm: service vm:115 - end relocate to node 'node2'
+info     25    node3/lrm: service vm:116 - start relocate to node 'node1'
+info     25    node3/lrm: service vm:116 - end relocate to node 'node1'
+info     25    node3/lrm: starting service vm:117
+info     25    node3/lrm: service status vm:117 started
+info     25    node3/lrm: service vm:118 - start relocate to node 'node2'
+info     25    node3/lrm: service vm:118 - end relocate to node 'node2'
+info     25    node3/lrm: service vm:119 - start relocate to node 'node1'
+info     25    node3/lrm: service vm:119 - end relocate to node 'node1'
+info     25    node3/lrm: starting service vm:120
+info     25    node3/lrm: service status vm:120 started
+info     25    node3/lrm: service vm:121 - start relocate to node 'node2'
+info     25    node3/lrm: service vm:121 - end relocate to node 'node2'
+info     25    node3/lrm: service vm:122 - start relocate to node 'node1'
+info     25    node3/lrm: service vm:122 - end relocate to node 'node1'
+info     25    node3/lrm: starting service vm:123
+info     25    node3/lrm: service status vm:123 started
+info     25    node3/lrm: service vm:124 - start relocate to node 'node2'
+info     25    node3/lrm: service vm:124 - end relocate to node 'node2'
+info     25    node3/lrm: service vm:125 - start relocate to node 'node1'
+info     25    node3/lrm: service vm:125 - end relocate to node 'node1'
+info     25    node3/lrm: starting service vm:126
+info     25    node3/lrm: service status vm:126 started
+info     25    node3/lrm: service vm:127 - start relocate to node 'node2'
+info     25    node3/lrm: service vm:127 - end relocate to node 'node2'
+info     25    node3/lrm: service vm:128 - start relocate to node 'node1'
+info     25    node3/lrm: service vm:128 - end relocate to node 'node1'
+info     25    node3/lrm: starting service vm:129
+info     25    node3/lrm: service status vm:129 started
+info     25    node3/lrm: service vm:130 - start relocate to node 'node2'
+info     25    node3/lrm: service vm:130 - end relocate to node 'node2'
+info     25    node3/lrm: service vm:131 - start relocate to node 'node1'
+info     25    node3/lrm: service vm:131 - end relocate to node 'node1'
+info     25    node3/lrm: starting service vm:132
+info     25    node3/lrm: service status vm:132 started
+info     25    node3/lrm: service vm:133 - start relocate to node 'node2'
+info     25    node3/lrm: service vm:133 - end relocate to node 'node2'
+info     25    node3/lrm: service vm:134 - start relocate to node 'node1'
+info     25    node3/lrm: service vm:134 - end relocate to node 'node1'
+info     25    node3/lrm: starting service vm:135
+info     25    node3/lrm: service status vm:135 started
+info     25    node3/lrm: service vm:136 - start relocate to node 'node2'
+info     25    node3/lrm: service vm:136 - end relocate to node 'node2'
+info     25    node3/lrm: service vm:137 - start relocate to node 'node1'
+info     25    node3/lrm: service vm:137 - end relocate to node 'node1'
+info     25    node3/lrm: starting service vm:138
+info     25    node3/lrm: service status vm:138 started
+info     25    node3/lrm: service vm:139 - start relocate to node 'node2'
+info     25    node3/lrm: service vm:139 - end relocate to node 'node2'
+info     25    node3/lrm: service vm:140 - start relocate to node 'node1'
+info     25    node3/lrm: service vm:140 - end relocate to node 'node1'
+info     25    node3/lrm: starting service vm:141
+info     25    node3/lrm: service status vm:141 started
+info     25    node3/lrm: service vm:142 - start relocate to node 'node2'
+info     25    node3/lrm: service vm:142 - end relocate to node 'node2'
+info     25    node3/lrm: service vm:143 - start relocate to node 'node1'
+info     25    node3/lrm: service vm:143 - end relocate to node 'node1'
+info     25    node3/lrm: starting service vm:144
+info     25    node3/lrm: service status vm:144 started
+info     25    node3/lrm: service vm:145 - start relocate to node 'node2'
+info     25    node3/lrm: service vm:145 - end relocate to node 'node2'
+info     25    node3/lrm: service vm:146 - start relocate to node 'node1'
+info     25    node3/lrm: service vm:146 - end relocate to node 'node1'
+info     25    node3/lrm: starting service vm:147
+info     25    node3/lrm: service status vm:147 started
+info     25    node3/lrm: service vm:148 - start relocate to node 'node2'
+info     25    node3/lrm: service vm:148 - end relocate to node 'node2'
+info     25    node3/lrm: service vm:149 - start relocate to node 'node1'
+info     25    node3/lrm: service vm:149 - end relocate to node 'node1'
+info     25    node3/lrm: starting service vm:150
+info     25    node3/lrm: service status vm:150 started
+info     25    node3/lrm: service vm:151 - start relocate to node 'node2'
+info     25    node3/lrm: service vm:151 - end relocate to node 'node2'
+info     40    node1/crm: service 'vm:102': state changed from 'request_stop' to 'stopped'
+info     40    node1/crm: service 'vm:104': state changed from 'request_start_balance' to 'started'  (node = node1)
+info     40    node1/crm: service 'vm:106': state changed from 'request_start_balance' to 'started'  (node = node2)
+info     40    node1/crm: service 'vm:107': state changed from 'request_start_balance' to 'started'  (node = node1)
+info     40    node1/crm: service 'vm:109': state changed from 'request_start_balance' to 'started'  (node = node2)
+info     40    node1/crm: service 'vm:110': state changed from 'request_start_balance' to 'started'  (node = node1)
+info     40    node1/crm: service 'vm:112': state changed from 'request_start_balance' to 'started'  (node = node2)
+info     40    node1/crm: service 'vm:113': state changed from 'request_start_balance' to 'started'  (node = node1)
+info     40    node1/crm: service 'vm:115': state changed from 'request_start_balance' to 'started'  (node = node2)
+info     40    node1/crm: service 'vm:116': state changed from 'request_start_balance' to 'started'  (node = node1)
+info     40    node1/crm: service 'vm:118': state changed from 'request_start_balance' to 'started'  (node = node2)
+info     40    node1/crm: service 'vm:119': state changed from 'request_start_balance' to 'started'  (node = node1)
+info     40    node1/crm: service 'vm:121': state changed from 'request_start_balance' to 'started'  (node = node2)
+info     40    node1/crm: service 'vm:122': state changed from 'request_start_balance' to 'started'  (node = node1)
+info     40    node1/crm: service 'vm:124': state changed from 'request_start_balance' to 'started'  (node = node2)
+info     40    node1/crm: service 'vm:125': state changed from 'request_start_balance' to 'started'  (node = node1)
+info     40    node1/crm: service 'vm:127': state changed from 'request_start_balance' to 'started'  (node = node2)
+info     40    node1/crm: service 'vm:128': state changed from 'request_start_balance' to 'started'  (node = node1)
+info     40    node1/crm: service 'vm:130': state changed from 'request_start_balance' to 'started'  (node = node2)
+info     40    node1/crm: service 'vm:131': state changed from 'request_start_balance' to 'started'  (node = node1)
+info     40    node1/crm: service 'vm:133': state changed from 'request_start_balance' to 'started'  (node = node2)
+info     40    node1/crm: service 'vm:134': state changed from 'request_start_balance' to 'started'  (node = node1)
+info     40    node1/crm: service 'vm:136': state changed from 'request_start_balance' to 'started'  (node = node2)
+info     40    node1/crm: service 'vm:137': state changed from 'request_start_balance' to 'started'  (node = node1)
+info     40    node1/crm: service 'vm:139': state changed from 'request_start_balance' to 'started'  (node = node2)
+info     40    node1/crm: service 'vm:140': state changed from 'request_start_balance' to 'started'  (node = node1)
+info     40    node1/crm: service 'vm:142': state changed from 'request_start_balance' to 'started'  (node = node2)
+info     40    node1/crm: service 'vm:143': state changed from 'request_start_balance' to 'started'  (node = node1)
+info     40    node1/crm: service 'vm:145': state changed from 'request_start_balance' to 'started'  (node = node2)
+info     40    node1/crm: service 'vm:146': state changed from 'request_start_balance' to 'started'  (node = node1)
+info     40    node1/crm: service 'vm:148': state changed from 'request_start_balance' to 'started'  (node = node2)
+info     40    node1/crm: service 'vm:149': state changed from 'request_start_balance' to 'started'  (node = node1)
+info     40    node1/crm: service 'vm:151': state changed from 'request_start_balance' to 'started'  (node = node2)
+info     41    node1/lrm: starting service vm:104
+info     41    node1/lrm: service status vm:104 started
+info     41    node1/lrm: starting service vm:107
+info     41    node1/lrm: service status vm:107 started
+info     41    node1/lrm: starting service vm:110
+info     41    node1/lrm: service status vm:110 started
+info     41    node1/lrm: starting service vm:113
+info     41    node1/lrm: service status vm:113 started
+info     41    node1/lrm: starting service vm:116
+info     41    node1/lrm: service status vm:116 started
+info     41    node1/lrm: starting service vm:119
+info     41    node1/lrm: service status vm:119 started
+info     41    node1/lrm: starting service vm:122
+info     41    node1/lrm: service status vm:122 started
+info     41    node1/lrm: starting service vm:125
+info     41    node1/lrm: service status vm:125 started
+info     41    node1/lrm: starting service vm:128
+info     41    node1/lrm: service status vm:128 started
+info     41    node1/lrm: starting service vm:131
+info     41    node1/lrm: service status vm:131 started
+info     41    node1/lrm: starting service vm:134
+info     41    node1/lrm: service status vm:134 started
+info     41    node1/lrm: starting service vm:137
+info     41    node1/lrm: service status vm:137 started
+info     41    node1/lrm: starting service vm:140
+info     41    node1/lrm: service status vm:140 started
+info     41    node1/lrm: starting service vm:143
+info     41    node1/lrm: service status vm:143 started
+info     41    node1/lrm: starting service vm:146
+info     41    node1/lrm: service status vm:146 started
+info     41    node1/lrm: starting service vm:149
+info     41    node1/lrm: service status vm:149 started
+info     43    node2/lrm: starting service vm:106
+info     43    node2/lrm: service status vm:106 started
+info     43    node2/lrm: starting service vm:109
+info     43    node2/lrm: service status vm:109 started
+info     43    node2/lrm: starting service vm:112
+info     43    node2/lrm: service status vm:112 started
+info     43    node2/lrm: starting service vm:115
+info     43    node2/lrm: service status vm:115 started
+info     43    node2/lrm: starting service vm:118
+info     43    node2/lrm: service status vm:118 started
+info     43    node2/lrm: starting service vm:121
+info     43    node2/lrm: service status vm:121 started
+info     43    node2/lrm: starting service vm:124
+info     43    node2/lrm: service status vm:124 started
+info     43    node2/lrm: starting service vm:127
+info     43    node2/lrm: service status vm:127 started
+info     43    node2/lrm: starting service vm:130
+info     43    node2/lrm: service status vm:130 started
+info     43    node2/lrm: starting service vm:133
+info     43    node2/lrm: service status vm:133 started
+info     43    node2/lrm: starting service vm:136
+info     43    node2/lrm: service status vm:136 started
+info     43    node2/lrm: starting service vm:139
+info     43    node2/lrm: service status vm:139 started
+info     43    node2/lrm: starting service vm:142
+info     43    node2/lrm: service status vm:142 started
+info     43    node2/lrm: starting service vm:145
+info     43    node2/lrm: service status vm:145 started
+info     43    node2/lrm: starting service vm:148
+info     43    node2/lrm: service status vm:148 started
+info     43    node2/lrm: starting service vm:151
+info     43    node2/lrm: service status vm:151 started
+info    120      cmdlist: execute network node3 off
+info    120    node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info    124    node3/crm: status change slave => wait_for_quorum
+info    125    node3/lrm: status change active => lost_agent_lock
+info    160    node1/crm: service 'vm:103': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:105': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:108': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:111': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:114': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:117': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:120': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:123': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:126': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:129': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:132': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:135': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:138': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:141': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:144': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:147': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:150': state changed from 'started' to 'fence'
+info    160    node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+emai    160    node1/crm: FENCE: Try to fence node 'node3'
+info    166     watchdog: execute power node3 off
+info    165    node3/crm: killed by poweroff
+info    166    node3/lrm: killed by poweroff
+info    166     hardware: server 'node3' stopped by poweroff (watchdog)
+info    240    node1/crm: got lock 'ha_agent_node3_lock'
+info    240    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
+info    240    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai    240    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info    240    node1/crm: service 'vm:103': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:105': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:108': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:111': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:114': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:117': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:120': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:123': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:126': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:129': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:132': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:135': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:138': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:141': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:144': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:147': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:150': state changed from 'fence' to 'recovery'
+info    240    node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node2'
+info    240    node1/crm: service 'vm:103': state changed from 'recovery' to 'started'  (node = node2)
+info    240    node1/crm: recover service 'vm:105' from fenced node 'node3' to node 'node1'
+info    240    node1/crm: service 'vm:105': state changed from 'recovery' to 'started'  (node = node1)
+info    240    node1/crm: recover service 'vm:108' from fenced node 'node3' to node 'node1'
+info    240    node1/crm: service 'vm:108': state changed from 'recovery' to 'started'  (node = node1)
+info    240    node1/crm: recover service 'vm:111' from fenced node 'node3' to node 'node1'
+info    240    node1/crm: service 'vm:111': state changed from 'recovery' to 'started'  (node = node1)
+info    240    node1/crm: recover service 'vm:114' from fenced node 'node3' to node 'node1'
+info    240    node1/crm: service 'vm:114': state changed from 'recovery' to 'started'  (node = node1)
+info    240    node1/crm: recover service 'vm:117' from fenced node 'node3' to node 'node2'
+info    240    node1/crm: service 'vm:117': state changed from 'recovery' to 'started'  (node = node2)
+info    240    node1/crm: recover service 'vm:120' from fenced node 'node3' to node 'node1'
+info    240    node1/crm: service 'vm:120': state changed from 'recovery' to 'started'  (node = node1)
+info    240    node1/crm: recover service 'vm:123' from fenced node 'node3' to node 'node2'
+info    240    node1/crm: service 'vm:123': state changed from 'recovery' to 'started'  (node = node2)
+info    240    node1/crm: recover service 'vm:126' from fenced node 'node3' to node 'node1'
+info    240    node1/crm: service 'vm:126': state changed from 'recovery' to 'started'  (node = node1)
+info    240    node1/crm: recover service 'vm:129' from fenced node 'node3' to node 'node2'
+info    240    node1/crm: service 'vm:129': state changed from 'recovery' to 'started'  (node = node2)
+info    240    node1/crm: recover service 'vm:132' from fenced node 'node3' to node 'node1'
+info    240    node1/crm: service 'vm:132': state changed from 'recovery' to 'started'  (node = node1)
+info    240    node1/crm: recover service 'vm:135' from fenced node 'node3' to node 'node2'
+info    240    node1/crm: service 'vm:135': state changed from 'recovery' to 'started'  (node = node2)
+info    240    node1/crm: recover service 'vm:138' from fenced node 'node3' to node 'node1'
+info    240    node1/crm: service 'vm:138': state changed from 'recovery' to 'started'  (node = node1)
+info    240    node1/crm: recover service 'vm:141' from fenced node 'node3' to node 'node2'
+info    240    node1/crm: service 'vm:141': state changed from 'recovery' to 'started'  (node = node2)
+info    240    node1/crm: recover service 'vm:144' from fenced node 'node3' to node 'node1'
+info    240    node1/crm: service 'vm:144': state changed from 'recovery' to 'started'  (node = node1)
+info    240    node1/crm: recover service 'vm:147' from fenced node 'node3' to node 'node2'
+info    240    node1/crm: service 'vm:147': state changed from 'recovery' to 'started'  (node = node2)
+info    240    node1/crm: recover service 'vm:150' from fenced node 'node3' to node 'node1'
+info    240    node1/crm: service 'vm:150': state changed from 'recovery' to 'started'  (node = node1)
+info    241    node1/lrm: starting service vm:105
+info    241    node1/lrm: service status vm:105 started
+info    241    node1/lrm: starting service vm:108
+info    241    node1/lrm: service status vm:108 started
+info    241    node1/lrm: starting service vm:111
+info    241    node1/lrm: service status vm:111 started
+info    241    node1/lrm: starting service vm:114
+info    241    node1/lrm: service status vm:114 started
+info    241    node1/lrm: starting service vm:120
+info    241    node1/lrm: service status vm:120 started
+info    241    node1/lrm: starting service vm:126
+info    241    node1/lrm: service status vm:126 started
+info    241    node1/lrm: starting service vm:132
+info    241    node1/lrm: service status vm:132 started
+info    241    node1/lrm: starting service vm:138
+info    241    node1/lrm: service status vm:138 started
+info    241    node1/lrm: starting service vm:144
+info    241    node1/lrm: service status vm:144 started
+info    241    node1/lrm: starting service vm:150
+info    241    node1/lrm: service status vm:150 started
+info    243    node2/lrm: starting service vm:103
+info    243    node2/lrm: service status vm:103 started
+info    243    node2/lrm: starting service vm:117
+info    243    node2/lrm: service status vm:117 started
+info    243    node2/lrm: starting service vm:123
+info    243    node2/lrm: service status vm:123 started
+info    243    node2/lrm: starting service vm:129
+info    243    node2/lrm: service status vm:129 started
+info    243    node2/lrm: starting service vm:135
+info    243    node2/lrm: service status vm:135 started
+info    243    node2/lrm: starting service vm:141
+info    243    node2/lrm: service status vm:141 started
+info    243    node2/lrm: starting service vm:147
+info    243    node2/lrm: service status vm:147 started
+info    720     hardware: exit simulation - done

diff --git a/src/test/test-crs-static-rebalance1/manager_status b/src/test/test-crs-static-rebalance1/manager_status
new file mode 100644 (file)
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-crs-static-rebalance1/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file

diff --git a/src/test/test-crs-static-rebalance1/service_config b/src/test/test-crs-static-rebalance1/service_config
new file mode 100644 (file)
index 0000000..be098fb
--- /dev/null
+++ b/src/test/test-crs-static-rebalance1/service_config
@@ -0,0 +1,53 @@
+{
+    "vm:101": { "node": "node1", "state": "enabled" },
+    "vm:102": { "node": "node2" },
+    "vm:103": { "node": "node3", "state": "enabled" },
+    "vm:104": { "node": "node3", "state": "enabled" },
+    "vm:105": { "node": "node3", "state": "enabled" },
+    "vm:106": { "node": "node3", "state": "enabled" },
+    "vm:107": { "node": "node3", "state": "enabled" },
+    "vm:108": { "node": "node3", "state": "enabled" },
+    "vm:109": { "node": "node3", "state": "enabled" },
+    "vm:110": { "node": "node3", "state": "enabled" },
+    "vm:111": { "node": "node3", "state": "enabled" },
+    "vm:112": { "node": "node3", "state": "enabled" },
+    "vm:113": { "node": "node3", "state": "enabled" },
+    "vm:114": { "node": "node3", "state": "enabled" },
+    "vm:115": { "node": "node3", "state": "enabled" },
+    "vm:116": { "node": "node3", "state": "enabled" },
+    "vm:117": { "node": "node3", "state": "enabled" },
+    "vm:118": { "node": "node3", "state": "enabled" },
+    "vm:119": { "node": "node3", "state": "enabled" },
+    "vm:120": { "node": "node3", "state": "enabled" },
+    "vm:121": { "node": "node3", "state": "enabled" },
+    "vm:122": { "node": "node3", "state": "enabled" },
+    "vm:123": { "node": "node3", "state": "enabled" },
+    "vm:124": { "node": "node3", "state": "enabled" },
+    "vm:125": { "node": "node3", "state": "enabled" },
+    "vm:126": { "node": "node3", "state": "enabled" },
+    "vm:127": { "node": "node3", "state": "enabled" },
+    "vm:128": { "node": "node3", "state": "enabled" },
+    "vm:129": { "node": "node3", "state": "enabled" },
+    "vm:130": { "node": "node3", "state": "enabled" },
+    "vm:131": { "node": "node3", "state": "enabled" },
+    "vm:132": { "node": "node3", "state": "enabled" },
+    "vm:133": { "node": "node3", "state": "enabled" },
+    "vm:134": { "node": "node3", "state": "enabled" },
+    "vm:135": { "node": "node3", "state": "enabled" },
+    "vm:136": { "node": "node3", "state": "enabled" },
+    "vm:137": { "node": "node3", "state": "enabled" },
+    "vm:138": { "node": "node3", "state": "enabled" },
+    "vm:139": { "node": "node3", "state": "enabled" },
+    "vm:140": { "node": "node3", "state": "enabled" },
+    "vm:141": { "node": "node3", "state": "enabled" },
+    "vm:142": { "node": "node3", "state": "enabled" },
+    "vm:143": { "node": "node3", "state": "enabled" },
+    "vm:144": { "node": "node3", "state": "enabled" },
+    "vm:145": { "node": "node3", "state": "enabled" },
+    "vm:146": { "node": "node3", "state": "enabled" },
+    "vm:147": { "node": "node3", "state": "enabled" },
+    "vm:148": { "node": "node3", "state": "enabled" },
+    "vm:149": { "node": "node3", "state": "enabled" },
+    "vm:150": { "node": "node3", "state": "enabled" },
+    "vm:151": { "node": "node3", "state": "enabled" }
+}

diff --git a/src/test/test-crs-static-rebalance1/static_service_stats b/src/test/test-crs-static-rebalance1/static_service_stats
new file mode 100644 (file)
index 0000000..7fb992d
--- /dev/null
+++ b/src/test/test-crs-static-rebalance1/static_service_stats
@@ -0,0 +1,3 @@
+{
+    "vm:102": { "maxcpu": 2, "maxmem": 4000000000 }
+}