treewide trailing whitespace cleanup

[pve-ha-manager.git] / src / PVE / HA / CRM.pm

diff --git a/src/PVE/HA/CRM.pm b/src/PVE/HA/CRM.pm
index 3dac3acf5cf3da4560406276ef5fed8f82282a0e..3c8ced53466d26029dee213812581121aa2be218 100644 (file)
--- a/src/PVE/HA/CRM.pm
+++ b/src/PVE/HA/CRM.pm
@@ -28,10 +28,11 @@ sub new {
        haenv => $haenv,
        manager => undef,
        status => { state => 'startup' },
+       cluster_state_update => 0,
     }, $class;
 
     $self->set_local_status({ state => 'wait_for_quorum' });
-    
+
     return $self;
 }
 
@@ -59,7 +60,7 @@ sub set_local_status {
 
     my $old = $self->{status};
 
-    # important: only update if if really changed 
+    # important: only update if if really changed
     return if $old->{state} eq $new->{state};
 
     $haenv->log('info', "status change $old->{state} => $new->{state}");
@@ -89,7 +90,7 @@ sub get_protected_ha_manager_lock {
     my $starttime = $haenv->get_time();
 
     for (;;) {
-       
+
        if ($haenv->get_ha_manager_lock()) {
            if ($self->{ha_manager_wd}) {
                $haenv->watchdog_update($self->{ha_manager_wd});
@@ -99,7 +100,7 @@ sub get_protected_ha_manager_lock {
            }
            return 1;
        }
-           
+
        last if ++$count > 5; # try max 5 time
 
        my $delay = $haenv->get_time() - $starttime;
@@ -107,18 +108,39 @@ sub get_protected_ha_manager_lock {
 
        $haenv->sleep(1);
     }
-    
+
     return 0;
 }
 
-sub check_pending_fencing {
-    my ($manager_status, $node) = @_;
+# checks quorum, for no active pending fence jobs and if services are configured
+sub can_get_active {
+    my ($self, $allow_no_service) = @_;
+
+    my $haenv = $self->{haenv};
+
+    return 0 if !$haenv->quorate();
+
+    # we may not do any active work with an incosistent cluster state
+    return 0 if !$self->{cluster_state_update};
 
+    my $manager_status = eval { $haenv->read_manager_status() };
+    if (my $err = $@) {
+       $haenv->log('err', "could not read manager status: $err");
+       return 0;
+    }
     my $ss = $manager_status->{service_status};
+    return 0 if PVE::HA::Tools::count_fenced_services($ss, $haenv->nodename());
 
-    return 1 if PVE::HA::Tools::count_fenced_services($ss, $node);
+    if (!$allow_no_service) {
+       my $conf = eval { $haenv->read_service_config() };
+       if (my $err = $@) {
+           $haenv->log('err', "could not read service config: $err");
+           return undef;
+       }
+       return 0 if !scalar(%{$conf});
+    }
 
-    return 0;
+    return 1;
 }
 
 sub do_one_iteration {
@@ -126,18 +148,30 @@ sub do_one_iteration {
 
     my $haenv = $self->{haenv};
 
+    $haenv->loop_start_hook();
+
+    $self->{cluster_state_update} = $haenv->cluster_state_update();
+
+    my $res = $self->work();
+
+    $haenv->loop_end_hook();
+
+    return $res;
+}
+
+sub work {
+    my ($self) = @_;
+
+    my $haenv = $self->{haenv};
+
     my $status = $self->get_local_status();
     my $state = $status->{state};
 
-    my $manager_status = $haenv->read_manager_status();
-    my $pending_fencing = check_pending_fencing($manager_status, $haenv->nodename());
-    
-    # do state changes first 
+    # do state changes first
 
     if ($state eq 'wait_for_quorum') {
 
-       if (!$pending_fencing && $haenv->quorate() &&
-           PVE::HA::Tools::has_services($haenv)) {
+       if ($self->can_get_active()) {
            if ($self->get_protected_ha_manager_lock()) {
                $self->set_local_status({ state => 'master' });
            } else {
@@ -147,8 +181,7 @@ sub do_one_iteration {
 
     } elsif ($state eq 'slave') {
 
-       if (!$pending_fencing && $haenv->quorate() &&
-           PVE::HA::Tools::has_services($haenv)) {
+       if ($self->can_get_active()) {
            if ($self->get_protected_ha_manager_lock()) {
                $self->set_local_status({ state => 'master' });
            }
@@ -158,7 +191,7 @@ sub do_one_iteration {
 
     } elsif ($state eq 'lost_manager_lock') {
 
-       if (!$pending_fencing && $haenv->quorate()) {
+       if ($self->can_get_active(1)) {
            if ($self->get_protected_ha_manager_lock()) {
                $self->set_local_status({ state => 'master' });
            }
@@ -170,7 +203,7 @@ sub do_one_iteration {
            $self->set_local_status({ state => 'lost_manager_lock'});
        }
     }
-   
+
     $status = $self->get_local_status();
     $state = $status->{state};
 
@@ -181,7 +214,7 @@ sub do_one_iteration {
        return 0 if $self->{shutdown_request};
 
        $haenv->sleep(5);
-          
+
     } elsif ($state eq 'master') {
 
        my $manager = $self->{manager};
@@ -216,6 +249,14 @@ sub do_one_iteration {
                $shutdown = 1;
 
            } else {
+               if (!$self->{cluster_state_update}) {
+                   # update failed but we could still renew our lock (cfs restart?),
+                   # safely skip manage and expect to update just fine next round
+                   $haenv->log('notice', "temporary inconsistent cluster state " .
+                               "(cfs restart?), skip round");
+                   return;
+               }
+
                $manager->manage();
            }
        };
@@ -228,7 +269,7 @@ sub do_one_iteration {
        $haenv->sleep_until($startime + $max_time);
 
     } elsif ($state eq 'lost_manager_lock') {
-       
+
        if ($self->{ha_manager_wd}) {
            $haenv->watchdog_close($self->{ha_manager_wd});
            delete $self->{ha_manager_wd};