PVE/HA/Manager.pm

   1 package PVE::HA::Manager;
   2
   3 use strict;
   4 use warnings;
   5 use Digest::MD5 qw(md5_base64);
   6
   7 use Data::Dumper;
   8
   9 use PVE::HA::NodeStatus;
  10
  11 sub new {
  12     my ($this, $haenv) = @_;
  13
  14     my $class = ref($this) || $this;
  15
  16     my $ms = $haenv->read_manager_status();
  17
  18     $ms->{master_node} = $haenv->nodename();
  19
  20     my $ns = PVE::HA::NodeStatus->new($haenv, $ms->{node_status} || {});
  21
  22     # fixme: use separate class  PVE::HA::ServiceStatus
  23     my $ss = $ms->{service_status} || {};
  24
  25     my $self = bless {
  26         haenv => $haenv,
  27         ms => $ms, # master status
  28         ns => $ns, # PVE::HA::NodeStatus
  29         ss => $ss, # service status
  30     }, $class;
  31
  32     return $self;
  33 }
  34
  35 sub cleanup {
  36     my ($self) = @_;
  37
  38     # todo: ?
  39 }
  40
  41 sub flush_master_status {
  42     my ($self) = @_;
  43
  44     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
  45
  46     $ms->{node_status} = $ns->{status};
  47     $ms->{service_status} = $ss;
  48
  49     $haenv->write_manager_status($ms);
  50 }
  51
  52 # Attention: must be idempotent (alway return the same result for same input!)
  53 sub select_service_node {
  54     my ($self, $service_conf) = @_;
  55
  56     my $ns = $self->{ns};
  57
  58     my $pref_node = $service_conf->{node};
  59
  60     return $pref_node if $ns->node_is_online($pref_node);
  61
  62     my $online_nodes = $ns->list_online_nodes();
  63
  64     return shift @$online_nodes;
  65 }
  66
  67 my $uid_counter = 0;
  68
  69 my $valid_service_states = {
  70     stopped => 1,
  71     request_stop => 1,
  72     started => 1,
  73     fence => 1,
  74     migrate => 1,
  75     relocate => 1,
  76     error => 1,
  77 };
  78
  79 my $change_service_state = sub {
  80     my ($self, $sid, $new_state, %params) = @_;
  81
  82     my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
  83
  84     my $sd = $ss->{$sid} || die "no such service '$sid";
  85
  86     my $old_state = $sd->{state};
  87     my $old_node = $sd->{node};
  88
  89     die "no state change" if $old_state eq $new_state; # just to be sure
  90
  91     die "invalid CRM service state '$new_state'\n" if !$valid_service_states->{$new_state};
  92
  93     foreach my $k (keys %$sd) { delete $sd->{$k}; };
  94
  95     $sd->{state} = $new_state;
  96     $sd->{node} = $old_node;
  97
  98     my $text_state = '';
  99     foreach my $k (keys %params) {
 100         my $v = $params{$k};
 101         $text_state .= ", " if $text_state;
 102         $text_state .= "$k = $v";
 103         $sd->{$k} = $v;
 104     }
 105
 106     $uid_counter++;
 107     $sd->{uid} = md5_base64($new_state . $$ . time() . $uid_counter);
 108
 109     $text_state = " ($text_state)" if $text_state;
 110     $haenv->log('info', "service '$sid': state changed from '${old_state}' to '${new_state}' $text_state\n");
 111 };
 112
 113 # read LRM status for all active nodes
 114 sub read_lrm_status {
 115     my ($self) = @_;
 116
 117     my $nodes = $self->{ns}->list_online_nodes();
 118     my $haenv = $self->{haenv};
 119
 120     my $res = {};
 121
 122     foreach my $node (@$nodes) {
 123         my $ls = $haenv->read_lrm_status($node);
 124         foreach my $uid (keys %$ls) {
 125             next if $res->{$uid}; # should not happen
 126             $res->{$uid} = $ls->{$uid};
 127         }
 128     }
 129
 130     return $res;
 131 }
 132
 133 # read new crm commands and save them into crm master status
 134 sub update_crm_commands {
 135     my ($self) = @_;
 136
 137     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 138
 139     my $cmdlist = $haenv->read_crm_commands();
 140
 141     foreach my $cmd (split(/\n/, $cmdlist)) {
 142         chomp $cmd;
 143
 144         if ($cmd =~ m/^(migrate|relocate)\s+(\S+)\s+(\S+)$/) {
 145             my ($task, $sid, $node) = ($1, $2, $3);
 146             if (my $sd = $ss->{$sid}) {
 147                 if (!$ns->node_is_online($node)) {
 148                     $haenv->log('err', "crm command error - node not online: $cmd");
 149                 } else {
 150                     if ($node eq $sd->{node}) {
 151                         $haenv->log('info', "ignore crm command - service already on target node: $cmd");
 152                     } else {
 153                         $haenv->log('info', "got crm command: $cmd");
 154                         $ss->{$sid}->{cmd} = [ $task, $node];
 155                     }
 156                 }
 157             } else {
 158                 $haenv->log('err', "crm command error - no such service: $cmd");
 159             }
 160
 161         } else {
 162             $haenv->log('err', "unable to parse crm command: $cmd");
 163         }
 164     }
 165
 166 }
 167
 168 sub manage {
 169     my ($self) = @_;
 170
 171     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 172
 173     $ns->update($haenv->get_node_info());
 174
 175     if (!$ns->node_is_online($haenv->nodename())) {
 176         $haenv->log('info', "master seems offline\n");
 177         return;
 178     }
 179
 180     my $lrm_status = $self->read_lrm_status();
 181
 182     my $sc = $haenv->read_service_config();
 183
 184     # compute new service status
 185
 186     # add new service
 187     foreach my $sid (keys %$sc) {
 188         next if $ss->{$sid}; # already there
 189         $haenv->log('info', "Adding new service '$sid'\n");
 190         # assume we are running to avoid relocate running service at add
 191         $ss->{$sid} = { state => 'started', node => $sc->{$sid}->{node}};
 192     }
 193
 194     $self->update_crm_commands();
 195
 196     for (;;) {
 197         my $repeat = 0;
 198
 199         foreach my $sid (keys %$ss) {
 200             my $sd = $ss->{$sid};
 201             my $cd = $sc->{$sid} || { state => 'disabled' };
 202
 203             my $lrm_res = $sd->{uid} ? $lrm_status->{$sd->{uid}} : undef;
 204
 205             my $last_state = $sd->{state};
 206
 207             if ($last_state eq 'stopped') {
 208
 209                 $self->next_state_stopped($sid, $cd, $sd);
 210
 211             } elsif ($last_state eq 'started') {
 212
 213                 $self->next_state_started($sid, $cd, $sd);
 214
 215             } elsif ($last_state eq 'migrate' || $last_state eq 'relocate') {
 216
 217                 # check result from LRM daemon
 218                 if ($lrm_res) {
 219                     my $exit_code = $lrm_res->{exit_code};
 220                     if ($exit_code == 0) {
 221                         &$change_service_state($self, $sid, 'started', node => $sd->{target});
 222                     } else {
 223                         $haenv->log('err', "service '$sid' - migration failed (exit code $exit_code)");
 224                         &$change_service_state($self, $sid, 'started', node => $sd->{node});
 225                     }
 226                 }
 227
 228             } elsif ($last_state eq 'fence') {
 229
 230                 # do nothing here - wait until fenced
 231
 232             } elsif ($last_state eq 'request_stop') {
 233
 234                 # check result from LRM daemon
 235                 if ($lrm_res) {
 236                     my $exit_code = $lrm_res->{exit_code};
 237                     if ($exit_code == 0) {
 238                         &$change_service_state($self, $sid, 'stopped');
 239                     } else {
 240                         &$change_service_state($self, $sid, 'error'); # fixme: what state?
 241                     }
 242                 }
 243
 244             } elsif ($last_state eq 'error') {
 245
 246                 # fixme:
 247
 248             } else {
 249
 250                 die "unknown service state '$last_state'";
 251             }
 252
 253             $repeat = 1 if $sd->{state} ne $last_state;
 254         }
 255
 256         # handle fencing
 257         my $fenced_nodes = {};
 258         foreach my $sid (keys %$ss) {
 259             my $sd = $ss->{$sid};
 260             next if $sd->{state} ne 'fence';
 261
 262             if (!defined($fenced_nodes->{$sd->{node}})) {
 263                 $fenced_nodes->{$sd->{node}} = $ns->fence_node($sd->{node}) || 0;
 264             }
 265
 266             next if !$fenced_nodes->{$sd->{node}};
 267
 268             # node fence was sucessful - mark service as stopped
 269             &$change_service_state($self, $sid, 'stopped');
 270         }
 271
 272         last if !$repeat;
 273     }
 274
 275     # remove stale services
 276     # fixme:
 277
 278     $self->flush_master_status();
 279 }
 280
 281 # functions to compute next service states
 282 # $cd: service configuration data (read only)
 283 # $sd: service status data (read only)
 284 #
 285 # Note: use change_service_state() to alter state
 286 #
 287
 288 sub next_state_stopped {
 289     my ($self, $sid, $cd, $sd) = @_;
 290
 291     my $haenv = $self->{haenv};
 292     my $ns = $self->{ns};
 293
 294     if ($sd->{node} ne $cd->{node}) {
 295         # this can happen if we fence a node with active migrations
 296         # hack: modify $sd (normally this should be considered read-only)
 297         $haenv->log('info', "fixup service '$sid' location ($sd->{node} => $cd->{node}");
 298         $sd->{node} = $cd->{node};
 299     }
 300
 301     if ($sd->{cmd}) {
 302         my ($cmd, $target) = @{$sd->{cmd}};
 303         delete $sd->{cmd};
 304
 305         if ($cmd eq 'migrate' || $cmd eq 'relocate') {
 306             if (!$ns->node_is_online($target)) {
 307                 $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 308             } elsif ($sd->{node} eq $target) {
 309                 $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
 310             } else {
 311                 $haenv->change_service_location($sid, $target);
 312                 $cd->{node} = $sd->{node} = $target; # fixme: $sd is read-only??!!
 313                 $haenv->log('info', "$cmd service '$sid' to node '$target' (stopped)");
 314             }
 315         } else {
 316             $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 317         }
 318     }
 319
 320     if ($cd->{state} eq 'disabled') {
 321         # do nothing
 322         return;
 323     }
 324
 325     if ($cd->{state} eq 'enabled') {
 326         if (my $node = $self->select_service_node($cd)) {
 327             if ($node && ($sd->{node} ne $node)) {
 328                 $haenv->change_service_location($sid, $node);
 329             }
 330             &$change_service_state($self, $sid, 'started', node => $node);
 331         } else {
 332             # fixme: warn
 333         }
 334
 335         return;
 336     }
 337
 338     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 339 }
 340
 341 sub next_state_started {
 342     my ($self, $sid, $cd, $sd) = @_;
 343
 344     my $haenv = $self->{haenv};
 345     my $ns = $self->{ns};
 346
 347     if (!$ns->node_is_online($sd->{node})) {
 348
 349         &$change_service_state($self, $sid, 'fence');
 350         return;
 351     }
 352
 353     if ($cd->{state} eq 'disabled') {
 354         &$change_service_state($self, $sid, 'request_stop');
 355         return;
 356     }
 357
 358     if ($cd->{state} eq 'enabled') {
 359
 360         if ($sd->{cmd}) {
 361             my ($cmd, $target) = @{$sd->{cmd}};
 362             delete $sd->{cmd};
 363
 364             if ($cmd eq 'migrate' || $cmd eq 'relocate') {
 365                 if (!$ns->node_is_online($target)) {
 366                     $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 367                 } elsif ($sd->{node} eq $target) {
 368                     $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
 369                 } else {
 370                     $haenv->log('info', "$cmd service '$sid' to node '$target' (running)");
 371                     &$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
 372                 }
 373             } else {
 374                 $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 375             }
 376         } else {
 377
 378             my $node = $self->select_service_node($cd);
 379
 380             if ($node && ($sd->{node} ne $node)) {
 381                 $haenv->log('info', "migrate service '$sid' to node '$node' (running)");
 382                 &$change_service_state($self, $sid, 'migrate', node => $sd->{node}, target => $node);
 383             } else {
 384                 # do nothing
 385             }
 386         }
 387
 388         return;
 389     }
 390
 391     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 392 }
 393
 394 1;