src/PVE/HA/CRM.pm

   1 package PVE::HA::CRM;
   2
   3 # Cluster Resource Manager
   4
   5 use strict;
   6 use warnings;
   7
   8 use PVE::SafeSyslog;
   9 use PVE::Tools;
  10 use PVE::HA::Tools;
  11
  12 use PVE::HA::Manager;
  13
  14 # Server can have several state:
  15
  16 my $valid_states = {
  17     wait_for_quorum => "cluster is not quorate, waiting",
  18     master => "quorate, and we got the ha_manager lock",
  19     lost_manager_lock => "we lost the ha_manager lock (watchgog active)",
  20     slave => "quorate, but we do not own the ha_manager lock",
  21 };
  22
  23 sub new {
  24     my ($this, $haenv) = @_;
  25
  26     my $class = ref($this) || $this;
  27
  28     my $self = bless {
  29         haenv => $haenv,
  30         manager => undef,
  31         status => { state => 'startup' },
  32     }, $class;
  33
  34     $self->set_local_status({ state => 'wait_for_quorum' });
  35
  36     return $self;
  37 }
  38
  39 sub shutdown_request {
  40     my ($self) = @_;
  41
  42     syslog('info' , "server received shutdown request")
  43         if !$self->{shutdown_request};
  44
  45     $self->{shutdown_request} = 1;
  46 }
  47
  48 sub get_local_status {
  49     my ($self) = @_;
  50
  51     return $self->{status};
  52 }
  53
  54 sub set_local_status {
  55     my ($self, $new) = @_;
  56
  57     die "invalid state '$new->{state}'" if !$valid_states->{$new->{state}};
  58
  59     my $haenv = $self->{haenv};
  60
  61     my $old = $self->{status};
  62
  63     # important: only update if if really changed
  64     return if $old->{state} eq $new->{state};
  65
  66     $haenv->log('info', "status change $old->{state} => $new->{state}");
  67
  68     $new->{state_change_time} = $haenv->get_time();
  69
  70     $self->{status} = $new;
  71
  72     # fixme: do not use extra class
  73     if ($new->{state} eq 'master') {
  74         mkdir("/etc/pve/ha");
  75         $self->{manager} = PVE::HA::Manager->new($haenv);
  76     } else {
  77         if ($self->{manager}) {
  78             # fixme: what should we do here?
  79             $self->{manager}->cleanup();
  80             $self->{manager} = undef;
  81         }
  82     }
  83 }
  84
  85 sub get_protected_ha_manager_lock {
  86     my ($self) = @_;
  87
  88     my $haenv = $self->{haenv};
  89
  90     my $count = 0;
  91     my $starttime = $haenv->get_time();
  92
  93     for (;;) {
  94
  95         if ($haenv->get_ha_manager_lock()) {
  96             if ($self->{ha_manager_wd}) {
  97                 $haenv->watchdog_update($self->{ha_manager_wd});
  98             } else {
  99                 my $wfh = $haenv->watchdog_open();
 100                 $self->{ha_manager_wd} = $wfh;
 101             }
 102             return 1;
 103         }
 104
 105         last if ++$count > 5; # try max 5 time
 106
 107         my $delay = $haenv->get_time() - $starttime;
 108         last if $delay > 5; # for max 5 seconds
 109
 110         $haenv->sleep(1);
 111     }
 112
 113     return 0;
 114 }
 115
 116 sub check_pending_fencing {
 117     my ($manager_status, $node) = @_;
 118
 119     my $ss = $manager_status->{service_status};
 120
 121     return 1 if PVE::HA::Tools::count_fenced_services($ss, $node);
 122
 123     return 0;
 124 }
 125
 126 sub do_one_iteration {
 127     my ($self) = @_;
 128
 129     my $haenv = $self->{haenv};
 130
 131     my $status = $self->get_local_status();
 132     my $state = $status->{state};
 133
 134     my $manager_status = $haenv->read_manager_status();
 135     my $pending_fencing = check_pending_fencing($manager_status, $haenv->nodename());
 136
 137     # do state changes first
 138
 139     if ($state eq 'wait_for_quorum') {
 140
 141         if (!$pending_fencing && $haenv->quorate()) {
 142             if ($self->get_protected_ha_manager_lock()) {
 143                 $self->set_local_status({ state => 'master' });
 144             } else {
 145                 $self->set_local_status({ state => 'slave' });
 146             }
 147         }
 148
 149     } elsif ($state eq 'slave') {
 150
 151         if (!$pending_fencing && $haenv->quorate()) {
 152             if ($self->get_protected_ha_manager_lock()) {
 153                 $self->set_local_status({ state => 'master' });
 154             }
 155         } else {
 156             $self->set_local_status({ state => 'wait_for_quorum' });
 157         }
 158
 159     } elsif ($state eq 'lost_manager_lock') {
 160
 161         if (!$pending_fencing && $haenv->quorate()) {
 162             if ($self->get_protected_ha_manager_lock()) {
 163                 $self->set_local_status({ state => 'master' });
 164             }
 165         }
 166
 167     } elsif ($state eq 'master') {
 168
 169         if (!$self->get_protected_ha_manager_lock()) {
 170             $self->set_local_status({ state => 'lost_manager_lock'});
 171         }
 172     }
 173
 174     $status = $self->get_local_status();
 175     $state = $status->{state};
 176
 177     # do work
 178
 179     if ($state eq 'wait_for_quorum') {
 180
 181         return 0 if $self->{shutdown_request};
 182
 183         $haenv->sleep(5);
 184
 185     } elsif ($state eq 'master') {
 186
 187         my $manager = $self->{manager};
 188
 189         die "no manager" if !defined($manager);
 190
 191         my $startime = $haenv->get_time();
 192
 193         my $max_time = 10;
 194
 195         my $shutdown = 0;
 196
 197         # do work (max_time seconds)
 198         eval {
 199             # fixme: set alert timer
 200
 201             if ($self->{shutdown_request}) {
 202
 203                 if ($self->{ha_manager_wd}) {
 204                     $haenv->watchdog_close($self->{ha_manager_wd});
 205                     delete $self->{ha_manager_wd};
 206                 }
 207
 208                 $shutdown = 1;
 209
 210             } else {
 211                 $manager->manage();
 212             }
 213         };
 214         if (my $err = $@) {
 215             $haenv->log('err', "got unexpected error - $err");
 216         }
 217
 218         return 0 if $shutdown;
 219
 220         $haenv->sleep_until($startime + $max_time);
 221
 222     } elsif ($state eq 'lost_manager_lock') {
 223
 224         if ($self->{ha_manager_wd}) {
 225             $haenv->watchdog_close($self->{ha_manager_wd});
 226             delete $self->{ha_manager_wd};
 227         }
 228
 229         return 0 if $self->{shutdown_request};
 230
 231         $self->set_local_status({ state => 'wait_for_quorum' });
 232
 233     } elsif ($state eq 'slave') {
 234
 235         return 0 if $self->{shutdown_request};
 236
 237         # wait until we get master
 238
 239     } else {
 240
 241         die "got unexpected status '$state'\n";
 242     }
 243
 244     return 1;
 245 }
 246
 247 1;