test/sim: allow to simulate cfs failures

Add simulated hardware commands for the cluster file system.

This allows to tell the regression test or simulator system that a
certain nodes calls to methods accessing the CFS should fail, i.e.,
die.
With this we can cover a situation which mainly happen during a
cluster file system update.

For now allow to define if the CFS is read-/writeable (state rw) and
if updates of the CFS (state update) should work or fail.

Add 'can read/write' assertions all over the relevant methods.

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
Reviewed-by: Dominik Csapak <d.csapak@proxmox.com>
Tested-by: Dominik Csapak <d.csapak@proxmox.com>

diff --git a/src/PVE/HA/Sim/Env.pm b/src/PVE/HA/Sim/Env.pm
index e00b2b1fc2fd6e7adb249d9e4a6f58aebf8f0677..92c9394e66d5de6b7d1520a78b5fe246191734a6 100644 (file)
--- a/src/PVE/HA/Sim/Env.pm
+++ b/src/PVE/HA/Sim/Env.pm
@@ -53,6 +53,15 @@ sub hardware {
     return $self->{hardware};
 }
 
+my $assert_cfs_can_rw = sub {
+    my ($self, $emsg) = @_;
+
+    $emsg //= 'cfs connection refused - not mounted?';
+
+    die "$emsg\n"
+       if !$self->{hardware}->get_cfs_state($self->{nodename}, 'rw');
+};
+
 sub sim_get_lock {
     my ($self, $lock_name, $unlock) = @_;
 
@@ -125,6 +134,8 @@ sub sim_get_lock {
 sub read_manager_status {
     my ($self) = @_;
 
+    $assert_cfs_can_rw->($self);
+
     my $filename = "$self->{statusdir}/manager_status";
 
     return PVE::HA::Tools::read_json_from_file($filename, {});
@@ -133,6 +144,8 @@ sub read_manager_status {
 sub write_manager_status {
     my ($self, $status_obj) = @_;
 
+    $assert_cfs_can_rw->($self);
+
     my $filename = "$self->{statusdir}/manager_status";
 
     PVE::HA::Tools::write_json_to_file($filename, $status_obj);
@@ -143,6 +156,8 @@ sub read_lrm_status {
 
     $node = $self->{nodename} if !defined($node);
 
+    $assert_cfs_can_rw->($self);
+
     return $self->{hardware}->read_lrm_status($node);
 }
 
@@ -151,6 +166,8 @@ sub write_lrm_status {
 
     my $node = $self->{nodename};
 
+    $assert_cfs_can_rw->($self);
+
     return $self->{hardware}->write_lrm_status($node, $status_obj);
 }
 
@@ -181,12 +198,16 @@ sub is_node_shutdown {
 sub read_service_config {
     my ($self) = @_;
 
+    $assert_cfs_can_rw->($self);
+
     return $self->{hardware}->read_service_config();
 }
 
 sub read_fence_config {
     my ($self) = @_;
 
+    $assert_cfs_can_rw->($self);
+
     return $self->{hardware}->read_fence_config();
 }
 
@@ -209,6 +230,8 @@ sub exec_fence_agent {
 sub read_group_config {
     my ($self) = @_;
 
+    $assert_cfs_can_rw->($self);
+
     return $self->{hardware}->read_group_config();
 }
 
@@ -216,18 +239,24 @@ sub read_group_config {
 sub steal_service {
     my ($self, $sid, $current_node, $new_node) = @_;
 
+    $assert_cfs_can_rw->($self);
+
     return $self->{hardware}->change_service_location($sid, $current_node, $new_node);
 }
 
 sub queue_crm_commands {
     my ($self, $cmd) = @_;
 
+    $assert_cfs_can_rw->($self);
+
     return $self->{hardware}->queue_crm_commands($cmd);
 }
 
 sub read_crm_commands {
     my ($self) = @_;
 
+    $assert_cfs_can_rw->($self);
+
     return $self->{hardware}->read_crm_commands();
 }
 

diff --git a/src/PVE/HA/Sim/Hardware.pm b/src/PVE/HA/Sim/Hardware.pm
index 6ba2210d6d8a77f5c5f08cb0173a5e6fbf40e71a..2019d8f158e16bbb8f0b862ed529200fc33846f4 100644 (file)
--- a/src/PVE/HA/Sim/Hardware.pm
+++ b/src/PVE/HA/Sim/Hardware.pm
@@ -494,9 +494,22 @@ sub get_node_info {
     return ($node_info, $quorate);
 }
 
+# helper for Sim/ only
+sub get_cfs_state {
+    my ($self, $node, $state) = @_;
+
+    # TODO: ensure nolock is OK when adding this to RTSim
+    my $cstatus = $self->read_hardware_status_nolock();
+    my $res = $cstatus->{$node}->{cfs}->{$state};
+
+    # we assume default true if not defined
+    return !defined($res) || $res;
+}
+
 # simulate hardware commands
 # power <node> <on|off>
 # network <node> <on|off>
+# cfs <node> <rw|update> <work|fail>
 # reboot <node>
 # shutdown <node>
 # restart-lrm <node>
@@ -539,6 +552,7 @@ sub sim_hardware_cmd {
                    $d->{crm} = $self->crm_control('start', $d, $lock_fh) if !defined($d->{crm});
                    $d->{lrm} = $self->lrm_control('start', $d, $lock_fh) if !defined($d->{lrm});
                    $d->{lrm_restart} = undef;
+                   $cstatus->{$node}->{cfs} = {};
 
                } else {
 
@@ -572,6 +586,15 @@ sub sim_hardware_cmd {
 
            $self->write_hardware_status_nolock($cstatus);
 
+       } elsif ($cmd eq 'cfs') {
+           die "sim_hardware_cmd: unknown cfs action '$action' for node '$node'"
+               if $action !~ m/^(rw|update)$/;
+           die "sim_hardware_cmd: unknown cfs command '$target' for '$action' on node '$node'"
+               if $target !~ m/^(work|fail)$/;
+
+           $cstatus->{$node}->{cfs}->{$action} = $target eq 'work';
+           $self->write_hardware_status_nolock($cstatus);
+
        } elsif ($cmd eq 'reboot' || $cmd eq 'shutdown') {
            $cstatus->{$node}->{shutdown} = $cmd;
 

diff --git a/src/test/test-cfs-unavailable1/README b/src/test/test-cfs-unavailable1/README
new file mode 100644 (file)
index 0000000..ffd526c
--- /dev/null
+++ b/src/test/test-cfs-unavailable1/README
@@ -0,0 +1 @@
+Test a cfs update behavior, e.g., cfs is (temporarily) not read and writeable.

diff --git a/src/test/test-cfs-unavailable1/cmdlist b/src/test/test-cfs-unavailable1/cmdlist
new file mode 100644 (file)
index 0000000..34596f1
--- /dev/null
+++ b/src/test/test-cfs-unavailable1/cmdlist
@@ -0,0 +1,5 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "cfs node1 rw fail", "service vm:101 stopped" ],
+    [ "cfs node1 rw work" ]
+]

diff --git a/src/test/test-cfs-unavailable1/hardware_status b/src/test/test-cfs-unavailable1/hardware_status
new file mode 100644 (file)
index 0000000..451beb1
--- /dev/null
+++ b/src/test/test-cfs-unavailable1/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}

diff --git a/src/test/test-cfs-unavailable1/manager_status b/src/test/test-cfs-unavailable1/manager_status
new file mode 100644 (file)
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-cfs-unavailable1/manager_status
@@ -0,0 +1 @@
+{}

diff --git a/src/test/test-cfs-unavailable1/service_config b/src/test/test-cfs-unavailable1/service_config
new file mode 100644 (file)
index 0000000..70f11d6
--- /dev/null
+++ b/src/test/test-cfs-unavailable1/service_config
@@ -0,0 +1,5 @@
+{
+    "vm:101": { "node": "node1", "state": "enabled" },
+    "vm:102": { "node": "node2" },
+    "vm:103": { "node": "node3", "state": "enabled" }
+}