improve logging

[pve-ha-manager.git] / src / PVE / HA / Manager.pm

diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm
index a32c22bdf3318284b37b4af4bda560316a34dc31..6be22d835fdb8ccd016d83da94845dbb7e01a96d 100644 (file)
--- a/src/PVE/HA/Manager.pm
+++ b/src/PVE/HA/Manager.pm
@@ -5,9 +5,11 @@ use warnings;
 use Digest::MD5 qw(md5_base64);
 
 use Data::Dumper;
-
+use PVE::Tools;
 use PVE::HA::NodeStatus;
 
+my $fence_delay = 30;
+
 sub new {
     my ($this, $haenv) = @_;
 
@@ -92,7 +94,9 @@ sub select_service_node {
 
     my $top_pri = $pri_list[0];
 
-    my @nodes = sort { $online_node_usage->{$a} <=> $online_node_usage->{$b} } keys %{$pri_groups->{$top_pri}};
+    my @nodes = sort { 
+       $online_node_usage->{$a} <=> $online_node_usage->{$b} || $a cmp $b
+    } keys %{$pri_groups->{$top_pri}};
 
     my $found;
     for (my $i = scalar(@nodes) - 1; $i >= 0; $i--) {
@@ -122,6 +126,13 @@ sub select_service_node {
 
 my $uid_counter = 0;
 
+sub compute_new_uuid {
+    my ($state) = @_;
+    
+    $uid_counter++;
+    return md5_base64($state . $$ . time() . $uid_counter);
+}
+
 my $valid_service_states = {
     stopped => 1,
     request_stop => 1,
@@ -192,11 +203,11 @@ my $change_service_state = sub {
 
     $self->recompute_online_node_usage();
 
-    $uid_counter++;
-    $sd->{uid} = md5_base64($new_state . $$ . time() . $uid_counter);
+    $sd->{uid} = compute_new_uuid($new_state);
+    
 
     $text_state = " ($text_state)" if $text_state;
-    $haenv->log('info', "service '$sid': state changed from '${old_state}' to '${new_state}' $text_state\n");
+    $haenv->log('info', "service '$sid': state changed from '${old_state}' to '${new_state}' $text_state");
 };
 
 # read LRM status for all active nodes 
@@ -262,7 +273,7 @@ sub manage {
     $ns->update($haenv->get_node_info());
 
     if (!$ns->node_is_online($haenv->nodename())) {
-       $haenv->log('info', "master seems offline\n");
+       $haenv->log('info', "master seems offline");
        return;
     }
 
@@ -275,13 +286,21 @@ sub manage {
     # compute new service status
 
     # add new service
-    foreach my $sid (keys %$sc) {
+    foreach my $sid (sort keys %$sc) {
        next if $ss->{$sid}; # already there
-       $haenv->log('info', "Adding new service '$sid'\n");
+       $haenv->log('info', "adding new service '$sid' on node '$sc->{$sid}->{node}'");
        # assume we are running to avoid relocate running service at add
-       $ss->{$sid} = { state => 'started', node => $sc->{$sid}->{node}};
+       $ss->{$sid} = { state => 'started', node => $sc->{$sid}->{node},
+                       uid => compute_new_uuid('started') };
     }
 
+    # remove stale service from manager state
+    foreach my $sid (keys %$ss) {
+       next if $sc->{$sid};
+       $haenv->log('info', "removing stale service '$sid' (no config)");
+       delete $ss->{$sid};
+    }
+    
     $self->update_crm_commands();
 
     for (;;) {
@@ -348,9 +367,6 @@ sub manage {
        last if !$repeat;
     }
 
-    # remove stale services
-    # fixme:
-
     $self->flush_master_status();
 }
 
@@ -379,7 +395,7 @@ sub next_state_request_stop {
        }
     }
 
-    if (!$ns->node_is_online($sd->{node})) {
+    if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
        &$change_service_state($self, $sid, 'fence');
        return;
     }
@@ -404,7 +420,7 @@ sub next_state_migrate_relocate {
        }
     }
 
-    if (!$ns->node_is_online($sd->{node})) {
+    if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
        &$change_service_state($self, $sid, 'fence');
        return;
     }
@@ -471,8 +487,9 @@ sub next_state_started {
     my $ns = $self->{ns};
 
     if (!$ns->node_is_online($sd->{node})) {
-
-       &$change_service_state($self, $sid, 'fence');
+       if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
+           &$change_service_state($self, $sid, 'fence');
+       }
        return;
     }