src/PVE/HA/NodeStatus.pm

   1 package PVE::HA::NodeStatus;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Data::Dumper;
   7
   8 sub new {
   9     my ($this, $haenv, $status) = @_;
  10
  11     my $class = ref($this) || $this;
  12
  13     my $self = bless {
  14         haenv => $haenv,
  15         status => $status,
  16         last_online => {},
  17     }, $class;
  18
  19     return $self;
  20 }
  21
  22 # possible node state:
  23 my $valid_node_states = {
  24     online => "node online and member of quorate partition",
  25     unknown => "not member of quorate partition, but possibly still running",
  26     fence => "node needs to be fenced",
  27     gone => "node vanished from cluster members list, possibly deleted"
  28 };
  29
  30 sub get_node_state {
  31     my ($self, $node) = @_;
  32
  33     $self->{status}->{$node} = 'unknown'
  34         if !$self->{status}->{$node};
  35
  36     return $self->{status}->{$node};
  37 }
  38
  39 sub node_is_online {
  40     my ($self, $node) = @_;
  41
  42     return $self->get_node_state($node) eq 'online';
  43 }
  44
  45 sub node_is_offline_delayed {
  46     my ($self, $node, $delay) = @_;
  47
  48     my $haenv = $self->{haenv};
  49
  50     return undef if $self->get_node_state($node) eq 'online';
  51
  52     my $last_online = $self->{last_online}->{$node};
  53
  54     my $ctime = $haenv->get_time();
  55
  56     if (!defined($last_online)) {
  57         $self->{last_online}->{$node} = $ctime;
  58         return undef;
  59     }
  60
  61     return ($ctime - $last_online) >= $delay;
  62 }
  63
  64 sub list_nodes {
  65     my ($self) = @_;
  66
  67     return [sort keys %{$self->{status}}];
  68 }
  69
  70 sub list_online_nodes {
  71     my ($self) = @_;
  72
  73     my $res = [];
  74
  75     foreach my $node (sort keys %{$self->{status}}) {
  76         next if $self->{status}->{$node} ne 'online';
  77         push @$res, $node;
  78     }
  79
  80     return $res;
  81 }
  82
  83 my $delete_node = sub {
  84     my ($self, $node) = @_;
  85
  86     return undef if $self->get_node_state($node) ne 'gone';
  87
  88     my $haenv = $self->{haenv};
  89
  90     delete $self->{last_online}->{$node};
  91     delete $self->{status}->{$node};
  92
  93     $haenv->log('notice', "deleting gone node '$node', not a cluster member".
  94                 " anymore.");
  95 };
  96
  97 my $set_node_state = sub {
  98     my ($self, $node, $state) = @_;
  99
 100     my $haenv = $self->{haenv};
 101
 102     die "unknown node state '$state'\n"
 103         if !defined($valid_node_states->{$state});
 104
 105     my $last_state = $self->get_node_state($node);
 106
 107     return if $state eq $last_state;
 108
 109     $self->{status}->{$node} = $state;
 110
 111     $haenv->log('info', "node '$node': state changed from " .
 112                 "'$last_state' => '$state'\n");
 113 };
 114
 115 sub update {
 116     my ($self, $node_info) = @_;
 117
 118     my $haenv = $self->{haenv};
 119
 120     foreach my $node (sort keys %$node_info) {
 121         my $d = $node_info->{$node};
 122         next if !$d->{online};
 123
 124         # record last time the node was online (required to implement fence delay)
 125         $self->{last_online}->{$node} = $haenv->get_time();
 126
 127         my $state = $self->get_node_state($node);
 128
 129         if ($state eq 'online') {
 130             # &$set_node_state($self, $node, 'online');
 131         } elsif ($state eq 'unknown' || $state eq 'gone') {
 132             &$set_node_state($self, $node, 'online');
 133         } elsif ($state eq 'fence') {
 134             # do nothing, wait until fenced
 135         } else {
 136             die "detected unknown node state '$state";
 137         }
 138     }
 139
 140     foreach my $node (keys %{$self->{status}}) {
 141         my $d = $node_info->{$node};
 142         next if $d && $d->{online};
 143
 144         my $state = $self->get_node_state($node);
 145
 146         # node is not inside quorate partition, possibly not active
 147
 148         if ($state eq 'online') {
 149             &$set_node_state($self, $node, 'unknown');
 150         } elsif ($state eq 'unknown') {
 151
 152             # node isn't in the member list anymore, deleted from the cluster?
 153             &$set_node_state($self, $node, 'gone') if(!defined($d));
 154
 155         } elsif ($state eq 'fence') {
 156             # do nothing, wait until fenced
 157         } elsif($state eq 'gone') {
 158             if($self->node_is_offline_delayed($node, 3600)) {
 159                 &$delete_node($self, $node);
 160             }
 161         } else {
 162             die "detected unknown node state '$state";
 163         }
 164
 165    }
 166 }
 167
 168 # start fencing
 169 sub fence_node {
 170     my ($self, $node) = @_;
 171
 172     my $haenv = $self->{haenv};
 173
 174     my $state = $self->get_node_state($node);
 175
 176     if ($state ne 'fence') {
 177         &$set_node_state($self, $node, 'fence');
 178     }
 179
 180     my $success = $haenv->get_ha_agent_lock($node);
 181
 182     if ($success) {
 183         $haenv->log("info", "fencing: acknowleged - got agent lock for node '$node'");
 184         &$set_node_state($self, $node, 'unknown');
 185     }
 186
 187     return $success;
 188 }
 189
 190 1;