src/PVE/HA/NodeStatus.pm

   1 package PVE::HA::NodeStatus;
   2
   3 use strict;
   4 use warnings;
   5
   6 use JSON;
   7
   8 my $fence_delay = 60;
   9
  10 sub new {
  11     my ($this, $haenv, $status) = @_;
  12
  13     my $class = ref($this) || $this;
  14
  15     my $self = bless {
  16         haenv => $haenv,
  17         status => $status,
  18         last_online => {},
  19     }, $class;
  20
  21     return $self;
  22 }
  23
  24 # possible node state:
  25 my $valid_node_states = {
  26     online => "node online and member of quorate partition",
  27     maintenance => "node is a member of quorate partition but currently not able to do work",
  28     unknown => "not member of quorate partition, but possibly still running",
  29     fence => "node needs to be fenced",
  30     gone => "node vanished from cluster members list, possibly deleted"
  31 };
  32
  33 sub get_node_state {
  34     my ($self, $node) = @_;
  35
  36     $self->{status}->{$node} = 'unknown'
  37         if !$self->{status}->{$node};
  38
  39     return $self->{status}->{$node};
  40 }
  41
  42 sub node_is_operational {
  43     my ($self, $node) = @_;
  44     return $self->node_is_online($node) || $self->get_node_state($node) eq 'maintenance';
  45 }
  46
  47 sub node_is_online {
  48     my ($self, $node) = @_;
  49
  50     return $self->get_node_state($node) eq 'online';
  51 }
  52
  53 sub node_is_offline_delayed {
  54     my ($self, $node, $delay) = @_;
  55
  56     $delay = $fence_delay if !defined($delay);
  57
  58     my $haenv = $self->{haenv};
  59
  60     return undef if $self->get_node_state($node) eq 'online';
  61
  62     my $last_online = $self->{last_online}->{$node};
  63
  64     my $ctime = $haenv->get_time();
  65
  66     if (!defined($last_online)) {
  67         $self->{last_online}->{$node} = $ctime;
  68         return undef;
  69     }
  70
  71     return ($ctime - $last_online) >= $delay;
  72 }
  73
  74 sub list_nodes {
  75     my ($self) = @_;
  76
  77     return [sort keys %{$self->{status}}];
  78 }
  79
  80 sub list_online_nodes {
  81     my ($self) = @_;
  82
  83     my $res = [];
  84
  85     foreach my $node (sort keys %{$self->{status}}) {
  86         next if $self->{status}->{$node} ne 'online';
  87         push @$res, $node;
  88     }
  89
  90     return $res;
  91 }
  92
  93 my $delete_node = sub {
  94     my ($self, $node) = @_;
  95
  96     return undef if $self->get_node_state($node) ne 'gone';
  97
  98     my $haenv = $self->{haenv};
  99
 100     delete $self->{last_online}->{$node};
 101     delete $self->{status}->{$node};
 102
 103     $haenv->log('notice', "deleting gone node '$node', not a cluster member".
 104                 " anymore.");
 105 };
 106
 107 my $set_node_state = sub {
 108     my ($self, $node, $state) = @_;
 109
 110     my $haenv = $self->{haenv};
 111
 112     die "unknown node state '$state'\n"
 113         if !defined($valid_node_states->{$state});
 114
 115     my $last_state = $self->get_node_state($node);
 116
 117     return if $state eq $last_state;
 118
 119     $self->{status}->{$node} = $state;
 120
 121     $haenv->log('info', "node '$node': state changed from " .
 122                 "'$last_state' => '$state'\n");
 123 };
 124
 125 sub update {
 126     my ($self, $node_info, $lrm_modes) = @_;
 127
 128     my $haenv = $self->{haenv};
 129
 130     foreach my $node (sort keys %$node_info) {
 131         my $d = $node_info->{$node};
 132         my $lrm_mode = $lrm_modes->{$node} // 'unkown';
 133         next if !$d->{online};
 134
 135         # record last time the node was online (required to implement fence delay)
 136         $self->{last_online}->{$node} = $haenv->get_time();
 137
 138         my $state = $self->get_node_state($node);
 139
 140         if ($state eq 'online') {
 141             if ($lrm_mode eq 'maintenance') {
 142                 #$haenv->log('info', "update node state maintance");
 143                 $set_node_state->($self, $node, 'maintenance');
 144             }
 145             # &$set_node_state($self, $node, 'online');
 146         } elsif ($state eq 'unknown' || $state eq 'gone') {
 147             &$set_node_state($self, $node, 'online');
 148         } elsif ($state eq 'fence') {
 149             # do nothing, wait until fenced
 150         } elsif ($state eq 'maintenance') {
 151             if ($lrm_mode ne 'maintenance') {
 152                 $set_node_state->($self, $node, 'online');
 153             }
 154         } else {
 155             die "detected unknown node state '$state";
 156         }
 157     }
 158
 159     foreach my $node (sort keys %{$self->{status}}) {
 160         my $d = $node_info->{$node};
 161         next if $d && $d->{online};
 162
 163         my $state = $self->get_node_state($node);
 164
 165         # node is not inside quorate partition, possibly not active
 166
 167         if ($state eq 'online' || $state eq 'maintenance') {
 168             &$set_node_state($self, $node, 'unknown');
 169         } elsif ($state eq 'unknown') {
 170
 171             # node isn't in the member list anymore, deleted from the cluster?
 172             &$set_node_state($self, $node, 'gone') if(!defined($d));
 173
 174         } elsif ($state eq 'fence') {
 175             # do nothing, wait until fenced
 176         } elsif($state eq 'gone') {
 177             if ($self->node_is_offline_delayed($node, 3600)) {
 178                 &$delete_node($self, $node);
 179             }
 180         } else {
 181             die "detected unknown node state '$state";
 182         }
 183
 184    }
 185 }
 186
 187 # assembles a commont text for fence emails
 188 my $send_fence_state_email = sub {
 189     my ($self, $subject_prefix, $subject, $node) = @_;
 190
 191     my $haenv = $self->{haenv};
 192
 193     my $mail_text = <<EOF
 194 The node '$node' failed and needs manual intervention.
 195
 196 The PVE HA manager tries  to fence it and recover the
 197 configured HA resources to a healthy node if possible.
 198
 199 Current fence status:  $subject_prefix
 200 $subject
 201
 202
 203 Overall Cluster status:
 204 -----------------------
 205
 206 EOF
 207 ;
 208     my $mail_subject = $subject_prefix . ': ' . $subject;
 209
 210     my $status = $haenv->read_manager_status();
 211     my $data = { manager_status => $status, node_status => $self->{status} };
 212
 213     $mail_text .= to_json($data, { pretty => 1, canonical => 1});
 214
 215     $haenv->sendmail($mail_subject, $mail_text);
 216 };
 217
 218
 219 # start fencing
 220 sub fence_node {
 221     my ($self, $node) = @_;
 222
 223     my $haenv = $self->{haenv};
 224
 225     my $state = $self->get_node_state($node);
 226
 227     if ($state ne 'fence') {
 228         &$set_node_state($self, $node, 'fence');
 229         my $msg = "Try to fence node '$node'";
 230         &$send_fence_state_email($self, 'FENCE', $msg, $node);
 231     }
 232
 233     my $success = $haenv->get_ha_agent_lock($node);
 234
 235     if ($success) {
 236         my $msg = "fencing: acknowledged - got agent lock for node '$node'";
 237         $haenv->log("info", $msg);
 238         &$set_node_state($self, $node, 'unknown');
 239         &$send_fence_state_email($self, 'SUCCEED', $msg, $node);
 240     }
 241
 242     return $success;
 243 }
 244
 245 1;