PVE/HA/Manager.pm

   1 package PVE::HA::Manager;
   2
   3 use strict;
   4 use warnings;
   5 use Digest::MD5 qw(md5_base64);
   6
   7 use Data::Dumper;
   8
   9 use PVE::HA::NodeStatus;
  10
  11 sub new {
  12     my ($this, $haenv) = @_;
  13
  14     my $class = ref($this) || $this;
  15
  16     my $ms = $haenv->read_manager_status();
  17
  18     $ms->{master_node} = $haenv->nodename();
  19
  20     my $ns = PVE::HA::NodeStatus->new($haenv, $ms->{node_status} || {});
  21
  22     # fixme: use separate class  PVE::HA::ServiceStatus
  23     my $ss = $ms->{service_status} || {};
  24
  25     my $self = bless {
  26         haenv => $haenv,
  27         ms => $ms, # master status
  28         ns => $ns, # PVE::HA::NodeStatus
  29         ss => $ss, # service status
  30     }, $class;
  31
  32     return $self;
  33 }
  34
  35 sub cleanup {
  36     my ($self) = @_;
  37
  38     # todo: ?
  39 }
  40
  41 sub flush_master_status {
  42     my ($self) = @_;
  43
  44     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
  45
  46     $ms->{node_status} = $ns->{status};
  47     $ms->{service_status} = $ss;
  48
  49     $haenv->write_manager_status($ms);
  50 }
  51
  52 # Attention: must be idempotent (alway return the same result for same input!)
  53 sub select_service_node {
  54     my ($self, $service_conf) = @_;
  55
  56     my $ns = $self->{ns};
  57
  58     my $pref_node = $service_conf->{node};
  59
  60     return $pref_node if $ns->node_is_online($pref_node);
  61
  62     my $online_nodes = $ns->list_online_nodes();
  63
  64     return shift @$online_nodes;
  65 }
  66
  67 my $uid_counter = 0;
  68
  69 my $change_service_state = sub {
  70     my ($self, $sid, $new_state, %params) = @_;
  71
  72     my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
  73
  74     my $sd = $ss->{$sid} || die "no such service '$sid";
  75
  76     my $old_state = $sd->{state};
  77
  78     die "no state change" if $old_state eq $new_state; # just to be sure
  79
  80     my $changes = '';
  81     foreach my $k (keys %params) {
  82         my $v = $params{$k};
  83         next if defined($sd->{$k}) && $sd->{$k} eq $v;
  84         $changes .= ", " if $changes;
  85         $changes .= "$k = $v";
  86         $sd->{$k} = $v;
  87     }
  88
  89     $sd->{state} = $new_state;
  90     $uid_counter++;
  91     $sd->{uid} = md5_base64($new_state . $$ . time() . $uid_counter);
  92
  93     # fixme: cleanup state (remove unused values)
  94
  95     $changes = " ($changes)" if $changes;
  96     $haenv->log('info', "service '$sid': state changed to '$new_state' $changes\n");
  97 };
  98
  99 # read LRM status for all nodes (even for offline nodes)
 100 sub read_lrm_status {
 101     my ($self, $node_info) = @_;
 102
 103     my $haenv = $self->{haenv};
 104
 105     my $res = {};
 106
 107     foreach my $node (keys %$node_info) {
 108         my $ls = $haenv->read_lrm_status($node);
 109         foreach my $uid (keys %$ls) {
 110             next if $res->{$uid}; # should not happen
 111             $res->{$uid} = $ls->{$uid};
 112         }
 113     }
 114
 115     return $res;
 116 }
 117
 118 sub manage {
 119     my ($self) = @_;
 120
 121     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 122
 123     my ($node_info, $quorate) = $haenv->get_node_info();
 124     $ns->update($node_info);
 125
 126     # fixme: what if $quorate is 0??
 127
 128     if (!$ns->node_is_online($haenv->nodename())) {
 129         $haenv->log('info', "master seems offline\n");
 130         return;
 131     }
 132
 133     my $lrm_status = $self->read_lrm_status($node_info);
 134
 135     my $sc = $haenv->read_service_config();
 136
 137     # compute new service status
 138
 139     # add new service
 140     foreach my $sid (keys %$sc) {
 141         next if $ss->{$sid}; # already there
 142         $haenv->log('info', "Adding new service '$sid'\n");
 143         # assume we are running to avoid relocate running service at add
 144         $ss->{$sid} = { state => 'started', node => $sc->{$sid}->{current_node}};
 145     }
 146
 147     for (;;) {
 148         my $repeat = 0;
 149
 150         foreach my $sid (keys %$ss) {
 151             my $sd = $ss->{$sid};
 152             my $cd = $sc->{$sid} || { state => 'disabled' };
 153
 154             my $last_state = $sd->{state};
 155
 156             if ($last_state eq 'stopped') {
 157
 158                 if ($cd->{state} eq 'disabled') {
 159                     # do nothing
 160                 } elsif ($cd->{state} eq 'enabled') {
 161                     if (my $node = $self->select_service_node($cd)) {
 162                         &$change_service_state($self, $sid, 'started', node => $node);
 163                     } else {
 164                         # fixme: warn
 165                     }
 166                 } else {
 167                     # do nothing - todo: log something?
 168                 }
 169
 170             } elsif ($last_state eq 'started') {
 171
 172                 if (!$ns->node_is_online($sd->{node})) {
 173
 174                     &$change_service_state($self, $sid, 'fence');
 175
 176                 } else {
 177
 178                     if ($cd->{state} eq 'disabled') {
 179                         &$change_service_state($self, $sid, 'request_stop');
 180                     } elsif ($cd->{state} eq 'enabled') {
 181                         my $node = $self->select_service_node($cd);
 182                         if ($node && ($sd->{node} ne $node)) {
 183                             &$change_service_state($self, $sid, 'migrate');
 184                         } else {
 185                             # do nothing
 186                         }
 187                     } else {
 188                         # do nothing - todo: log something?
 189                     }
 190                 }
 191
 192             } elsif ($last_state eq 'migrate') {
 193
 194                 die "implement me";
 195
 196             } elsif ($last_state eq 'fence') {
 197
 198                 # do nothing here - wait until fenced
 199
 200             } elsif ($last_state eq 'request_stop') {
 201
 202 #fixme:         die "implement me";
 203
 204             } else {
 205
 206                 die "unknown service state '$last_state'";
 207             }
 208
 209
 210             $repeat = 1 if $sd->{state} ne $last_state;
 211         }
 212
 213         # handle fencing
 214         my $fenced_nodes = {};
 215         foreach my $sid (keys %$ss) {
 216             my $sd = $ss->{$sid};
 217             next if $sd->{state} ne 'fence';
 218
 219             if (!defined($fenced_nodes->{$sd->{node}})) {
 220                 $fenced_nodes->{$sd->{node}} = $ns->fence_node($sd->{node}) || 0;
 221             }
 222
 223             next if !$fenced_nodes->{$sd->{node}};
 224
 225             # node fence was sucessful - mark service as stopped
 226             &$change_service_state($self, $sid, 'stopped');
 227         }
 228
 229         last if !$repeat;
 230     }
 231
 232     # remove stale services
 233     # fixme:
 234
 235     $self->flush_master_status();
 236 }
 237
 238
 239 1;