simulate CRM and LRM - cleanup logs

diff --git a/PVE/HA/CRM.pm b/PVE/HA/CRM.pm
index 8c4c90f36621b5ff8d500ce8f79a81bf93e9711a..ef18d55d43dbfc8bf6e9ea64df74c2d95ef90663 100644 (file)
--- a/PVE/HA/CRM.pm
+++ b/PVE/HA/CRM.pm
@@ -60,7 +60,7 @@ sub set_local_status {
     # important: only update if if really changed 
     return if $old->{state} eq $new->{state};
 
-    $haenv->log('info', "CRM status change $old->{state} => $new->{state}");
+    $haenv->log('info', "status change $old->{state} => $new->{state}");
 
     $new->{state_change_time} = $haenv->get_time();
 

diff --git a/PVE/HA/LRM.pm b/PVE/HA/LRM.pm
index 8d5c646d211026a8167916ec38da055b3600383e..78da10bfdec3925ca5cc8d9964e8efc05cec89d7 100644 (file)
--- a/PVE/HA/LRM.pm
+++ b/PVE/HA/LRM.pm
@@ -13,7 +13,7 @@ use PVE::HA::Tools;
 
 my $valid_states = {
     wait_for_agent_lock => "waiting for agnet lock",
-    locked => "got agent_lock",
+    active => "got agent_lock",
     lost_agent_lock => "lost agent_lock",
 };
 
@@ -56,7 +56,7 @@ sub set_local_status {
     # important: only update if if really changed 
     return if $old->{state} eq $new->{state};
 
-    $haenv->log('info', "LRM status change $old->{state} => $new->{state}");
+    $haenv->log('info', "status change $old->{state} => $new->{state}");
 
     $new->{state_change_time} = $haenv->get_time();
 
@@ -111,22 +111,22 @@ sub do_one_iteration {
        my $service_count = 1; # todo: correctly compute
 
        if ($service_count && $haenv->quorate()) {
-           if ($self->get_protectedt_ha_agent_lock()) {
-               $self->set_local_status({ state => 'locked' });
+           if ($self->get_protected_ha_agent_lock()) {
+               $self->set_local_status({ state => 'active' });
            }
        }
        
     } elsif ($state eq 'lost_agent_lock') {
 
        if ($haenv->quorate()) {
-           if ($self->get_protectedt_ha_agent_lock()) {
-               $self->set_local_status({ state => 'locked' });
+           if ($self->get_protected_ha_agent_lock()) {
+               $self->set_local_status({ state => 'active' });
            }
        }
 
-    } elsif ($state eq 'locked') {
+    } elsif ($state eq 'active') {
 
-       if (!$self->get_protectedt_ha_agent_lock()) {
+       if (!$self->get_protected_ha_agent_lock()) {
            $self->set_local_status({ state => 'lost_agent_lock'});
        }
     }
@@ -142,7 +142,7 @@ sub do_one_iteration {
 
        $haenv->sleep(5);
           
-    } elsif ($state eq 'locked') {
+    } elsif ($state eq 'active') {
 
        my $startime = $haenv->get_time();
 

diff --git a/PVE/HA/Sim/Env.pm b/PVE/HA/Sim/Env.pm
index 46b5b3f5ec758cffe53aabe01b2a1d2cce34c5aa..011f343694370cfc292e374e3d338c90594b6c9f 100644 (file)
--- a/PVE/HA/Sim/Env.pm
+++ b/PVE/HA/Sim/Env.pm
@@ -12,10 +12,12 @@ use PVE::HA::Tools;
 use PVE::HA::Env;
 
 sub new {
-    my ($this, $nodename, $hardware) = @_;
+    my ($this, $nodename, $hardware, $log_id, $lock_timeout) = @_;
 
     die "missing nodename" if !$nodename;
-
+    die "missing log_id" if !$log_id;
+    die "missing lock_timeout" if !$lock_timeout;
+    
     my $class = ref($this) || $this;
 
     my $self = bless {}, $class;
@@ -27,6 +29,10 @@ sub new {
     $self->{cur_time} = 0;
     $self->{loop_delay} = 0;
 
+    $self->{lock_timeout} = $lock_timeout;
+
+    $self->{log_id} = $log_id;
+
     return $self;
 }
 
@@ -57,9 +63,9 @@ sub sim_get_lock {
            if (my $d = $data->{$lock_name}) {
                my $tdiff = $ctime - $d->{time};
            
-               if ($tdiff > 120) {
+               if ($tdiff > $self->{lock_timeout}) {
                    $res = 1;
-               } elsif (($tdiff <= 120) && ($d->{node} eq $nodename)) {
+               } elsif (($tdiff <= $self->{lock_timeout}) && ($d->{node} eq $nodename)) {
                    delete $data->{$lock_name};
                    $res = 1;
                } else {
@@ -73,7 +79,7 @@ sub sim_get_lock {
            
                my $tdiff = $ctime - $d->{time};
            
-               if ($tdiff <= 120) {
+               if ($tdiff <= $self->{lock_timeout}) {
                    if ($d->{node} eq $nodename) {
                        $d->{time} = $ctime;
                        $res = 1;
@@ -156,7 +162,7 @@ sub log {
 
     my $time = $self->get_time();
 
-    printf("%-5s %5d %10s: $msg\n", $level, $time, $self->{nodename});
+    printf("%-5s %5d %12s: $msg\n", $level, $time, "$self->{nodename}/$self->{log_id}");
 }
 
 sub get_time {

diff --git a/PVE/HA/Sim/Hardware.pm b/PVE/HA/Sim/Hardware.pm
index 652b1679f9f876e967a0edb1dfbcf11b49531a11..037982f020b42395de958a1f62f9dde76987229a 100644 (file)
--- a/PVE/HA/Sim/Hardware.pm
+++ b/PVE/HA/Sim/Hardware.pm
@@ -16,10 +16,14 @@ use Fcntl qw(:DEFAULT :flock);
 use File::Copy;
 use File::Path qw(make_path remove_tree);
 
+my $watchdog_timeout = 180;
+my $lock_timeout = 200;
+
 my $max_sim_time = 10000;
 
 use PVE::HA::Sim::Env;
 use PVE::HA::CRM;
+use PVE::HA::LRM;
 
 # Status directory layout
 #
@@ -98,14 +102,16 @@ sub new {
 
     foreach my $node (sort keys %$cstatus) {
 
-       my $haenv = PVE::HA::Env->new('PVE::HA::Sim::Env', $node, $self);
+       $self->{nodes}->{$node} = {};
 
-       die "HA is not enabled\n" if !$haenv->manager_status_exists();
+       $self->{nodes}->{$node}->{crm_env} = 
+           PVE::HA::Env->new('PVE::HA::Sim::Env', $node, $self, 'crm', $lock_timeout);
 
-       $haenv->log('info', "starting server");
+       $self->{nodes}->{$node}->{lrm_env} = 
+           PVE::HA::Env->new('PVE::HA::Sim::Env', $node, $self, 'lrm', $lock_timeout);
 
-       $self->{nodes}->{$node}->{haenv} = $haenv;
        $self->{nodes}->{$node}->{crm} = undef; # create on power on
+       $self->{nodes}->{$node}->{lrm} = undef; # create on power on
     }
 
     return $self;
@@ -126,7 +132,7 @@ sub log {
 
     $id = 'hardware' if !$id;
 
-    printf("%-5s %5d %10s: $msg\n", $level, $time, $id);
+    printf("%-5s %5d %12s: $msg\n", $level, $time, $id);
 }
 
 sub statusdir {
@@ -226,17 +232,25 @@ sub sim_hardware_cmd {
        die "sim_hardware_cmd: no node specified" if !$node;
        die "sim_hardware_cmd: unknown action '$action'" if $action !~ m/^(on|off)$/;
 
-       my $haenv = $self->{nodes}->{$node}->{haenv};
-       die "sim_hardware_cmd: no such node '$node'\n" if !$haenv;
+       my $d = $self->{nodes}->{$node};
+       die "sim_hardware_cmd: no such node '$node'\n" if !$d;
+
+       $self->log('info', "execute $cmdstr", $logid);
        
        if ($cmd eq 'power') {
            if ($cstatus->{$node}->{power} ne $action) {
-               if ($action eq 'on') {
-                   my $crm = $self->{nodes}->{$node}->{crm} = 
-                       PVE::HA::CRM->new($haenv);
-               } elsif ($self->{nodes}->{$node}->{crm}) {
-                   $haenv->log('info', "server killed by poweroff", $logid);
-                   $self->{nodes}->{$node}->{crm} = undef;
+               if ($action eq 'on') {        
+                   $d->{crm} = PVE::HA::CRM->new($d->{crm_env}) if !$d->{crm};
+                   $d->{lrm} = PVE::HA::LRM->new($d->{lrm_env}) if !$d->{lrm};
+               } else {
+                   if ($d->{crm}) {
+                       $d->{crm_env}->log('info', "killed by poweroff");
+                       $d->{crm} = undef;
+                   }
+                   if ($d->{lrm}) {
+                       $d->{lrm_env}->log('info', "killed by poweroff");
+                       $d->{lrm} = undef;
+                   }
                }
            }
 
@@ -249,8 +263,6 @@ sub sim_hardware_cmd {
            die "sim_hardware_cmd: unknown command '$cmd'\n";
        }
 
-       $self->log('info', "execute $cmdstr", $logid);
-
        $self->write_hardware_status_nolock($cstatus);
     };
 
@@ -269,25 +281,39 @@ sub run {
        my @nodes = sort keys %{$self->{nodes}};
 
        foreach my $node (@nodes) {
-           my $haenv = $self->{nodes}->{$node}->{haenv};
-           my $crm = $self->{nodes}->{$node}->{crm};
 
-           next if !$crm;
+           my $d = $self->{nodes}->{$node};
+           
+           if (my $crm = $d->{crm}) {
 
-           $haenv->loop_start_hook($self->get_time());
+               $d->{crm_env}->loop_start_hook($self->get_time());
 
-           die "implement me" if !$crm->do_one_iteration();
+               die "implement me (CRM exit)" if !$crm->do_one_iteration();
 
-           $haenv->loop_end_hook();
+               $d->{crm_env}->loop_end_hook();
+
+               my $nodetime = $d->{crm_env}->get_time();
+               $self->{cur_time} = $nodetime if $nodetime > $self->{cur_time};
+           }
+
+           if (my $lrm = $d->{lrm}) {
+
+               $d->{lrm_env}->loop_start_hook($self->get_time());
+
+               die "implement me (LRM exit)" if !$lrm->do_one_iteration();
+
+               $d->{lrm_env}->loop_end_hook();
+
+               my $nodetime = $d->{lrm_env}->get_time();
+               $self->{cur_time} = $nodetime if $nodetime > $self->{cur_time};
+           }
 
-           my $nodetime = $haenv->get_time();
-           $self->{cur_time} = $nodetime if $nodetime > $self->{cur_time};
-           
            foreach my $n (@nodes) {
                if (!$self->watchdog_check($n)) {
-                   $self->sim_hardware_cmd("power $n off", 'fencedev');
-                   $self->log('info', "server '$n' killed by poweroff (fencing)");
+                   $self->sim_hardware_cmd("power $n off", 'watchdog');
+                   $self->log('info', "server '$n' stopped by poweroff (watchdog)");
                    $self->{nodes}->{$n}->{crm} = undef;
+                   $self->{nodes}->{$n}->{lrm} = undef;
                }
            }
        }
@@ -356,7 +382,7 @@ sub watchdog_check {
            my $ctime = $self->get_time();
            my $tdiff = $ctime - $wd->{update_time};
 
-           if ($tdiff > 60) { # expired
+           if ($tdiff > $watchdog_timeout) { # expired
                $res = 0;
                delete $wdstatus->{$wfh};
            }
@@ -403,7 +429,7 @@ sub watchdog_close {
        die "no such watchdog handle '$wfh'\n" if !defined($wd);
 
        my $tdiff = $self->get_time() - $wd->{update_time};
-       die "watchdog expired" if $tdiff > 60;
+       die "watchdog expired" if $tdiff > $watchdog_timeout;
 
        delete $wdstatus->{$wfh};
 
@@ -426,7 +452,7 @@ sub watchdog_update {
        my $ctime = $self->get_time();
        my $tdiff = $ctime - $wd->{update_time};
 
-       die "watchdog expired" if $tdiff > 60;
+       die "watchdog expired" if $tdiff > $watchdog_timeout;
        
        $wd->{update_time} = $ctime;