use command list to simulate membership

diff --git a/PVE/HA/NodeStatus.pm b/PVE/HA/NodeStatus.pm
index cef804f88ae4ef59953a793fc48cb5967904b2b6..3ab6f290d00708083bb6d1889f5b1dd2b96f2d88 100644 (file)
--- a/PVE/HA/NodeStatus.pm
+++ b/PVE/HA/NodeStatus.pm
@@ -125,7 +125,14 @@ sub fence_node {
        &$set_node_state($self, $node, 'fence');
     }
 
-    return $haenv->test_ha_agent_lock($node)
+    my $success = $haenv->test_ha_agent_lock($node);
+
+    if ($success) {
+       $haenv->log("info", "fencing: acknowleged - got agent_lock");
+       die "implement me";
+    }
+
+    return $success;
 }
 
 1;

diff --git a/PVE/HA/SimEnv.pm b/PVE/HA/SimEnv.pm
index dfee9d261a35556893c8340b0e33b50f3f03cdb8..0375b1d8e5d750396780312a5218774d33f2eff1 100644 (file)
--- a/PVE/HA/SimEnv.pm
+++ b/PVE/HA/SimEnv.pm
@@ -16,50 +16,53 @@ my $cur_time = 0;
 
 my $max_sim_time = 1000;
 
+my $read_cluster_status = sub {
+    my ($self) = @_;
 
+    my $filename = "$self->{statusdir}/cluster_status";
 
-# time => quorate nodes (first node gets manager lock)
-my $quorum_setup = [];
+    my $raw = PVE::Tools::file_get_contents($filename);
+    my $cstatus = decode_json($raw);
 
-my $compute_node_info = sub {
+    return $cstatus;
+};
 
-    my $last_node_info = {};
+my $write_cluster_status = sub {
+    my ($self, $cstatus) = @_;
 
-    foreach my $entry (@$quorum_setup) {
-       my ($time, $members) = @$entry;
+    my $filename = "$self->{statusdir}/cluster_status";
 
-       $max_sim_time = $time + 1000;
+    PVE::Tools::file_set_contents($filename, encode_json($cstatus));
+};
 
-       my $node_info = {};
+my $compute_node_info = sub {
+    my ($self, $cstatus) = @_;
 
-       foreach my $node (@$members) {
-           $node_info->{$node}->{online} = 1;
-           if (!$last_node_info->{$node}) {
-               $node_info->{$node}->{join_time} = $time;
-           } else {
-               $node_info->{$node}->{join_time} =
-                   $last_node_info->{$node}->{join_time};
-           }
-       }
+    my $node_info = {};
 
-       push @$entry, $node_info;
+    my $node_count = 0;
+    my $online_count = 0;
 
-       $last_node_info = $node_info;
-    }
-};
+    foreach my $node (keys %$cstatus) {
+       my $d = $cstatus->{$node};
 
-my $lookup_quorum_info = sub {
-    my ($self) = @_;
+       my $online = ($d->{power} eq 'on' && $d->{network} eq 'on') ? 1 : 0;
+       $node_info->{$node}->{online} = $online;
 
-    foreach my $entry (reverse @$quorum_setup) {
-       my ($time, $members) = @$entry;
+       $node_count++;
+       $online_count++ if $online;
+    }
 
-       if ($cur_time >= $time) {
-           return $members;
+    my $quorate = ($online_count > int($node_count/2)) ? 1 : 0;
+                  
+    if (!$quorate) {
+       foreach my $node (keys %$cstatus) {
+           my $d = $cstatus->{$node};
+           $node_info->{$node}->{online} = 0;
        }
     }
-    
-    return undef;
+
+    return ($node_info, $quorate);
 };
 
 sub new {
@@ -72,16 +75,18 @@ sub new {
        $nodename = PVE::Tools::file_read_firstline("$testdir/hostname");
     }
 
-    if (-f "$testdir/membership") {
-       my $raw = PVE::Tools::file_get_contents("$testdir/membership");
-       $quorum_setup = decode_json($raw);
-    }
-
     my $statusdir = "$testdir/status";
 
     my $self = $class->SUPER::new($statusdir, $nodename);
 
-    &$compute_node_info();
+    if (-f "$testdir/cmdlist") {
+       my $raw = PVE::Tools::file_get_contents("$testdir/cmdlist");
+       $self->{cmdlist} = decode_json($raw);
+    } else {
+       $self->{cmdlist} = [];
+    }
+
+    $self->{loop_count} = 0;
 
     return $self;
 }
@@ -181,7 +186,7 @@ sub sim_get_lock {
        return $res;
     };
 
-    $self->sim_cluster_lock($code);
+    return $self->sim_cluster_lock($code);
 }
 
 sub read_manager_status {
@@ -315,27 +320,23 @@ sub test_ha_agent_lock {
 sub quorate {
     my ($self) = @_;
 
-    if (my $members = &$lookup_quorum_info($self)) {
-       foreach my $node (@$members) {
-           return 1 if $node eq $self->{nodename};
-       }
-    }
-
-    return 0;
+    my $code = sub { 
+       my $cstatus = &$read_cluster_status($self);
+       my ($node_info, $quorate) = &$compute_node_info($self, $cstatus); 
+       return $quorate;
+    };
+    return $self->sim_cluster_lock($code);
 }
 
 sub get_node_info {
     my ($self) = @_;
 
-    foreach my $entry (reverse @$quorum_setup) {
-       my ($time, $members, $node_info) = @$entry;
-
-       if ($cur_time >= $time) {
-           return $node_info;
-       }
-    }
-
-    die "unbale to get node info";
+    my $code = sub { 
+       my $cstatus = &$read_cluster_status($self);
+       my ($node_info, $quorate) = &$compute_node_info($self, $cstatus); 
+       return $node_info;
+    };
+    return $self->sim_cluster_lock($code);
 }
 
 sub loop_start_hook {
@@ -343,12 +344,25 @@ sub loop_start_hook {
 
     $self->{loop_start_time} = $cur_time;
 
+    # apply new comand after 5 loop iterations
+
+    if (($self->{loop_count} % 5) == 0) {
+       my $list = shift $self->{cmdlist};
+       return if !$list;
+
+       foreach my $cmd (@$list) {
+           $self->sim_cluster_cmd($cmd);
+       }
+    }
+
     # do nothing
 }
 
 sub loop_end_hook {
     my ($self) = @_;
 
+    ++$self->{loop_count};
+
     my $delay = $cur_time - $self->{loop_start_time};
 
     die "loop take too long ($delay seconds)\n" if $delay > 30;
@@ -356,4 +370,38 @@ sub loop_end_hook {
     die "simulation end\n" if $cur_time > $max_sim_time;
 }
 
+# simulate cluster commands
+# power <node> <on|off>
+# network <node> <on|off>
+
+sub sim_cluster_cmd {
+    my ($self, $cmdstr) = @_;
+
+    my $code = sub {
+
+       my $cstatus = &$read_cluster_status($self);
+
+       my ($cmd, $node, $action) = split(/\s+/, $cmdstr);
+
+       die "sim_cluster_cmd: no node specified" if !$node;
+       die "sim_cluster_cmd: unknown action '$action'" if $action !~ m/^(on|off)$/;
+
+       if ($cmd eq 'power') {
+               $cstatus->{$node}->{power} = $action;
+               $cstatus->{$node}->{network} = $action;
+       } elsif ($cmd eq 'network') {
+               $cstatus->{$node}->{network} = $action;
+       } else {
+           die "sim_cluster_cmd: unknown command '$cmd'\n";
+       }
+
+       $self->log('info', "execute $cmdstr");
+
+       &$write_cluster_status($self, $cstatus);
+    };
+
+    return $self->sim_cluster_lock($code);
+}
+
+
 1;

diff --git a/test/ha-tester.pl b/test/ha-tester.pl
index 6eae43f8e47c8362dc52102e814dad1594cb804d..d6eee93668f2c05f597e503f6ae72ea7fcafcd53 100755 (executable)
--- a/test/ha-tester.pl
+++ b/test/ha-tester.pl
@@ -32,10 +32,12 @@ sub run_test {
     if (-f "$dir/manager_status") {
        system("cp $dir/manager_status $statusdir/manager_status");
     }
-   if (-f "$dir/service_status") {
+    if (-f "$dir/service_status") {
        system("cp $dir/service_status $statusdir/service_status");
     }
 
+    system("cp $dir/cluster_status $statusdir/cluster_status");
+
     my $logfile = "$dir/log";
     my $logexpect = "$logfile.expect";