src/PVE/HA/Manager.pm

   1 package PVE::HA::Manager;
   2
   3 use strict;
   4 use warnings;
   5 use Digest::MD5 qw(md5_base64);
   6
   7 use PVE::Tools;
   8 use PVE::HA::Tools ':exit_codes';
   9 use PVE::HA::NodeStatus;
  10 use PVE::HA::Usage::Basic;
  11
  12 ## Variable Name & Abbreviations Convention
  13 #
  14 # The HA stack has some variables it uses frequently and thus abbreviates it such that it may be
  15 # confusing for new readers. Here's a short list of the most common used.
  16 #
  17 # NOTE: variables should be assumed to be read only if not otherwise stated, only use the specific
  18 # methods to re-compute/read/alter them.
  19 #
  20 # - $haenv -> HA environment, the main interface to the simulator/test/real world
  21 # - $sid -> Service ID, unique identifier for a service, `type:vmid` is common
  22 #
  23 # - $ms -> Master/Manager Status, contains runtime info from the current active manager
  24 # - $ns -> Node Status, hash holding online/offline status about all nodes
  25 #
  26 # - $ss -> Service Status, hash holding the current state (last LRM cmd result, failed starts
  27 #          or migrates, maintenance fallback node, for *all* services ...
  28 # - $sd -> Service Data, the service status of a *single* service, iow. $ss->{$sid}
  29 #
  30 # - $sc -> Service Configuration, hash for all services including target state, group, ...
  31 # - $cd -> Configuration Data, the service config of a *single* service, iow. $sc->{$sid}
  32 #
  33 # Try to avoid adding new two letter (or similar over abbreviated) names, but also don't send
  34 # patches for changing above, as that set is mostly sensible and should be easy to remember once
  35 # spending a bit time in the HA code base.
  36
  37 sub new {
  38     my ($this, $haenv) = @_;
  39
  40     my $class = ref($this) || $this;
  41
  42     my $self = bless { haenv => $haenv }, $class;
  43
  44     my $old_ms = $haenv->read_manager_status();
  45
  46     # we only copy the state part of the manager which cannot be auto generated
  47
  48     $self->{ns} = PVE::HA::NodeStatus->new($haenv, $old_ms->{node_status} || {});
  49
  50     # fixme: use separate class  PVE::HA::ServiceStatus
  51     $self->{ss} = $old_ms->{service_status} || {};
  52
  53     $self->{ms} = { master_node => $haenv->nodename() };
  54
  55     return $self;
  56 }
  57
  58 sub cleanup {
  59     my ($self) = @_;
  60
  61     # todo: ?
  62 }
  63
  64 sub flush_master_status {
  65     my ($self) = @_;
  66
  67     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
  68
  69     $ms->{node_status} = $ns->{status};
  70     $ms->{service_status} = $ss;
  71     $ms->{timestamp} = $haenv->get_time();
  72
  73     $haenv->write_manager_status($ms);
  74 }
  75
  76 sub get_service_group {
  77     my ($groups, $online_node_usage, $service_conf) = @_;
  78
  79     my $group = {};
  80     # add all online nodes to default group to allow try_next when no group set
  81     $group->{nodes}->{$_} = 1 for $online_node_usage->list_nodes();
  82
  83     # overwrite default if service is bound to a specific group
  84     if (my $group_id = $service_conf->{group}) {
  85         $group = $groups->{ids}->{$group_id} if $groups->{ids}->{$group_id};
  86     }
  87
  88     return $group;
  89 }
  90
  91 # groups available nodes with their priority as group index
  92 sub get_node_priority_groups {
  93     my ($group, $online_node_usage) = @_;
  94
  95     my $pri_groups = {};
  96     my $group_members = {};
  97     foreach my $entry (keys %{$group->{nodes}}) {
  98         my ($node, $pri) = ($entry, 0);
  99         if ($entry =~ m/^(\S+):(\d+)$/) {
 100             ($node, $pri) = ($1, $2);
 101         }
 102         next if !$online_node_usage->contains_node($node); # offline
 103         $pri_groups->{$pri}->{$node} = 1;
 104         $group_members->{$node} = $pri;
 105     }
 106
 107     # add non-group members to unrestricted groups (priority -1)
 108     if (!$group->{restricted}) {
 109         my $pri = -1;
 110         for my $node ($online_node_usage->list_nodes()) {
 111             next if defined($group_members->{$node});
 112             $pri_groups->{$pri}->{$node} = 1;
 113             $group_members->{$node} = -1;
 114         }
 115     }
 116
 117     return ($pri_groups, $group_members);
 118 }
 119
 120 sub select_service_node {
 121     my ($groups, $online_node_usage, $sid, $service_conf, $current_node, $try_next, $tried_nodes, $maintenance_fallback) = @_;
 122
 123     my $group = get_service_group($groups, $online_node_usage, $service_conf);
 124
 125     my ($pri_groups, $group_members) = get_node_priority_groups($group, $online_node_usage);
 126
 127     my @pri_list = sort {$b <=> $a} keys %$pri_groups;
 128     return undef if !scalar(@pri_list);
 129
 130     # stay on current node if possible (avoids random migrations)
 131     if (!$try_next && $group->{nofailback} && defined($group_members->{$current_node})) {
 132         return $current_node;
 133     }
 134
 135     # select node from top priority node list
 136
 137     my $top_pri = $pri_list[0];
 138
 139     # try to avoid nodes where the service failed already if we want to relocate
 140     if ($try_next) {
 141         foreach my $node (@$tried_nodes) {
 142             delete $pri_groups->{$top_pri}->{$node};
 143         }
 144     }
 145
 146     my $scores = $online_node_usage->score_nodes_to_start_service($sid, $current_node);
 147     my @nodes = sort {
 148         $scores->{$a} <=> $scores->{$b} || $a cmp $b
 149     } keys %{$pri_groups->{$top_pri}};
 150
 151     my $found;
 152     my $found_maintenance_fallback;
 153     for (my $i = scalar(@nodes) - 1; $i >= 0; $i--) {
 154         my $node = $nodes[$i];
 155         if ($node eq $current_node) {
 156             $found = $i;
 157         }
 158         if (defined($maintenance_fallback) && $node eq $maintenance_fallback) {
 159             $found_maintenance_fallback = $i;
 160         }
 161     }
 162
 163     if (defined($found_maintenance_fallback)) {
 164         return $nodes[$found_maintenance_fallback];
 165     }
 166
 167     if ($try_next) {
 168         if (defined($found) && ($found < (scalar(@nodes) - 1))) {
 169             return $nodes[$found + 1];
 170         } else {
 171             return $nodes[0];
 172         }
 173     } elsif (defined($found)) {
 174         return $nodes[$found];
 175     } else {
 176         return $nodes[0];
 177     }
 178 }
 179
 180 my $uid_counter = 0;
 181
 182 sub compute_new_uuid {
 183     my ($state) = @_;
 184
 185     $uid_counter++;
 186     return md5_base64($state . $$ . time() . $uid_counter);
 187 }
 188
 189 my $valid_service_states = {
 190     stopped => 1,
 191     request_stop => 1,
 192     started => 1,
 193     fence => 1,
 194     recovery => 1,
 195     migrate => 1,
 196     relocate => 1,
 197     freeze => 1,
 198     error => 1,
 199 };
 200
 201 sub recompute_online_node_usage {
 202     my ($self) = @_;
 203
 204     my $online_node_usage = PVE::HA::Usage::Basic->new($self->{haenv});
 205
 206     my $online_nodes = $self->{ns}->list_online_nodes();
 207
 208     $online_node_usage->add_node($_) for $online_nodes->@*;
 209
 210     foreach my $sid (keys %{$self->{ss}}) {
 211         my $sd = $self->{ss}->{$sid};
 212         my $state = $sd->{state};
 213         my $target = $sd->{target}; # optional
 214         if ($online_node_usage->contains_node($sd->{node})) {
 215             if (
 216                 $state eq 'started' || $state eq 'request_stop' || $state eq 'fence' ||
 217                 $state eq 'freeze' || $state eq 'error' || $state eq 'recovery'
 218             ) {
 219                 $online_node_usage->add_service_usage_to_node($sd->{node}, $sid, $sd->{node});
 220             } elsif (($state eq 'migrate') || ($state eq 'relocate')) {
 221                 my $source = $sd->{node};
 222                 # count it for both, source and target as load is put on both
 223                 $online_node_usage->add_service_usage_to_node($source, $sid, $source, $target);
 224                 $online_node_usage->add_service_usage_to_node($target, $sid, $source, $target);
 225             } elsif ($state eq 'stopped') {
 226                 # do nothing
 227             } else {
 228                 die "should not be reached (sid = '$sid', state = '$state')";
 229             }
 230         } elsif (defined($target) && $online_node_usage->contains_node($target)) {
 231             if ($state eq 'migrate' || $state eq 'relocate') {
 232                 # to correctly track maintenance modi and also consider the target as used for the
 233                 # case a node dies, as we cannot really know if the to-be-aborted incoming migration
 234                 # has already cleaned up all used resources
 235                 $online_node_usage->add_service_usage_to_node($target, $sid, $sd->{node}, $target);
 236             }
 237         }
 238     }
 239
 240     $self->{online_node_usage} = $online_node_usage;
 241 }
 242
 243 my $change_service_state = sub {
 244     my ($self, $sid, $new_state, %params) = @_;
 245
 246     my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
 247
 248     my $sd = $ss->{$sid} || die "no such service '$sid";
 249
 250     my $old_state = $sd->{state};
 251     my $old_node = $sd->{node};
 252     my $old_failed_nodes = $sd->{failed_nodes};
 253     my $old_maintenance_node = $sd->{maintenance_node};
 254
 255     die "no state change" if $old_state eq $new_state; # just to be sure
 256
 257     die "invalid CRM service state '$new_state'\n" if !$valid_service_states->{$new_state};
 258
 259     foreach my $k (keys %$sd) { delete $sd->{$k}; };
 260
 261     $sd->{state} = $new_state;
 262     $sd->{node} = $old_node;
 263     $sd->{failed_nodes} = $old_failed_nodes if defined($old_failed_nodes);
 264     $sd->{maintenance_node} = $old_maintenance_node if defined($old_maintenance_node);
 265
 266     my $text_state = '';
 267     foreach my $k (sort keys %params) {
 268         my $v = $params{$k};
 269         $text_state .= ", " if $text_state;
 270         $text_state .= "$k = $v";
 271         $sd->{$k} = $v;
 272     }
 273
 274     $self->recompute_online_node_usage();
 275
 276     $sd->{uid} = compute_new_uuid($new_state);
 277
 278     $text_state = "  ($text_state)" if $text_state;
 279     $haenv->log('info', "service '$sid': state changed from '${old_state}'" .
 280                 " to '${new_state}'$text_state");
 281 };
 282
 283 # clean up a possible bad state from a recovered service to allow its start
 284 my $fence_recovery_cleanup = sub {
 285     my ($self, $sid, $fenced_node) = @_;
 286
 287     my $haenv = $self->{haenv};
 288
 289     my (undef, $type, $id) = $haenv->parse_sid($sid);
 290     my $plugin = PVE::HA::Resources->lookup($type);
 291
 292     # should not happen
 293     die "unknown resource type '$type'" if !$plugin;
 294
 295     # locks may block recovery, cleanup those which are safe to remove after fencing,
 296     # i.e., after the original node was reset and thus all it's state
 297     my $removable_locks = [
 298         'backup',
 299         'mounted',
 300         'migrate',
 301         'clone',
 302         'rollback',
 303         'snapshot',
 304         'snapshot-delete',
 305         'suspending',
 306         'suspended',
 307     ];
 308     if (my $removed_lock = $plugin->remove_locks($haenv, $id, $removable_locks, $fenced_node)) {
 309         $haenv->log('warning', "removed leftover lock '$removed_lock' from recovered " .
 310                     "service '$sid' to allow its start.");
 311     }
 312 };
 313
 314 # read LRM status for all nodes
 315 sub read_lrm_status {
 316     my ($self) = @_;
 317
 318     my $nodes = $self->{ns}->list_nodes();
 319     my $haenv = $self->{haenv};
 320
 321     my $results = {};
 322     my $modes = {};
 323     foreach my $node (@$nodes) {
 324         my $lrm_status = $haenv->read_lrm_status($node);
 325         $modes->{$node} = $lrm_status->{mode} || 'active';
 326         foreach my $uid (keys %{$lrm_status->{results}}) {
 327             next if $results->{$uid}; # should not happen
 328             $results->{$uid} = $lrm_status->{results}->{$uid};
 329         }
 330     }
 331
 332     return ($results, $modes);
 333 }
 334
 335 # read new crm commands and save them into crm master status
 336 sub update_crm_commands {
 337     my ($self) = @_;
 338
 339     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 340
 341     my $cmdlist = $haenv->read_crm_commands();
 342
 343     foreach my $cmd (split(/\n/, $cmdlist)) {
 344         chomp $cmd;
 345
 346         if ($cmd =~ m/^(migrate|relocate)\s+(\S+)\s+(\S+)$/) {
 347             my ($task, $sid, $node) = ($1, $2, $3);
 348             if (my $sd = $ss->{$sid}) {
 349                 if (!$ns->node_is_online($node)) {
 350                     $haenv->log('err', "crm command error - node not online: $cmd");
 351                 } else {
 352                     if ($node eq $sd->{node}) {
 353                         $haenv->log('info', "ignore crm command - service already on target node: $cmd");
 354                     } else {
 355                         $haenv->log('info', "got crm command: $cmd");
 356                         $ss->{$sid}->{cmd} = [ $task, $node ];
 357                     }
 358                 }
 359             } else {
 360                 $haenv->log('err', "crm command error - no such service: $cmd");
 361             }
 362
 363         } elsif ($cmd =~ m/^stop\s+(\S+)\s+(\S+)$/) {
 364             my ($sid, $timeout) = ($1, $2);
 365             if (my $sd = $ss->{$sid}) {
 366                 $haenv->log('info', "got crm command: $cmd");
 367                 $ss->{$sid}->{cmd} = [ 'stop', $timeout ];
 368             } else {
 369                 $haenv->log('err', "crm command error - no such service: $cmd");
 370             }
 371         } else {
 372             $haenv->log('err', "unable to parse crm command: $cmd");
 373         }
 374     }
 375
 376 }
 377
 378 sub manage {
 379     my ($self) = @_;
 380
 381     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 382
 383     my ($node_info) = $haenv->get_node_info();
 384     my ($lrm_results, $lrm_modes) = $self->read_lrm_status();
 385
 386     $ns->update($node_info, $lrm_modes);
 387
 388     if (!$ns->node_is_operational($haenv->nodename())) {
 389         $haenv->log('info', "master seems offline");
 390         return;
 391     }
 392
 393     my $sc = $haenv->read_service_config();
 394
 395     $self->{groups} = $haenv->read_group_config(); # update
 396
 397     # compute new service status
 398
 399     # add new service
 400     foreach my $sid (sort keys %$sc) {
 401         next if $ss->{$sid}; # already there
 402         my $cd = $sc->{$sid};
 403         next if $cd->{state} eq 'ignored';
 404
 405         $haenv->log('info', "adding new service '$sid' on node '$cd->{node}'");
 406         # assume we are running to avoid relocate running service at add
 407         my $state = ($cd->{state} eq 'started') ? 'started' : 'request_stop';
 408         $ss->{$sid} = { state => $state, node => $cd->{node},
 409                         uid => compute_new_uuid('started') };
 410     }
 411
 412     # remove stale or ignored services from manager state
 413     foreach my $sid (keys %$ss) {
 414         next if $sc->{$sid} && $sc->{$sid}->{state} ne 'ignored';
 415
 416         my $reason =  defined($sc->{$sid}) ? 'ignored state requested' : 'no config';
 417         $haenv->log('info', "removing stale service '$sid' ($reason)");
 418
 419         # remove all service related state information
 420         delete $ss->{$sid};
 421     }
 422
 423     $self->update_crm_commands();
 424
 425     for (;;) {
 426         my $repeat = 0;
 427
 428         $self->recompute_online_node_usage();
 429
 430         foreach my $sid (sort keys %$ss) {
 431             my $sd = $ss->{$sid};
 432             my $cd = $sc->{$sid} || { state => 'disabled' };
 433
 434             my $lrm_res = $sd->{uid} ? $lrm_results->{$sd->{uid}} : undef;
 435
 436             my $last_state = $sd->{state};
 437
 438             if ($last_state eq 'stopped') {
 439
 440                 $self->next_state_stopped($sid, $cd, $sd, $lrm_res);
 441
 442             } elsif ($last_state eq 'started') {
 443
 444                 $self->next_state_started($sid, $cd, $sd, $lrm_res);
 445
 446             } elsif ($last_state eq 'migrate' || $last_state eq 'relocate') {
 447
 448                 $self->next_state_migrate_relocate($sid, $cd, $sd, $lrm_res);
 449
 450             } elsif ($last_state eq 'fence') {
 451
 452                 # do nothing here - wait until fenced
 453
 454             } elsif ($last_state eq 'recovery') {
 455
 456                 $self->next_state_recovery($sid, $cd, $sd, $lrm_res);
 457
 458             } elsif ($last_state eq 'request_stop') {
 459
 460                 $self->next_state_request_stop($sid, $cd, $sd, $lrm_res);
 461
 462             } elsif ($last_state eq 'freeze') {
 463
 464                 my $lrm_mode = $sd->{node} ? $lrm_modes->{$sd->{node}} : undef;
 465                 # unfreeze
 466                 my $state = ($cd->{state} eq 'started') ? 'started' : 'request_stop';
 467                 &$change_service_state($self, $sid, $state)
 468                     if $lrm_mode && $lrm_mode eq 'active';
 469
 470             } elsif ($last_state eq 'error') {
 471
 472                 $self->next_state_error($sid, $cd, $sd, $lrm_res);
 473
 474             } else {
 475
 476                 die "unknown service state '$last_state'";
 477             }
 478
 479             my $lrm_mode = $sd->{node} ? $lrm_modes->{$sd->{node}} : undef;
 480             if ($lrm_mode && $lrm_mode eq 'restart') {
 481                 if (($sd->{state} eq 'started' || $sd->{state} eq 'stopped' ||
 482                      $sd->{state} eq 'request_stop')) {
 483                     &$change_service_state($self, $sid, 'freeze');
 484                 }
 485             }
 486
 487             $repeat = 1 if $sd->{state} ne $last_state;
 488         }
 489
 490         # handle fencing
 491         my $fenced_nodes = {};
 492         foreach my $sid (sort keys %$ss) {
 493             my ($service_state, $service_node) = $ss->{$sid}->@{'state', 'node'};
 494             next if $service_state ne 'fence';
 495
 496             if (!defined($fenced_nodes->{$service_node})) {
 497                 $fenced_nodes->{$service_node} = $ns->fence_node($service_node) || 0;
 498             }
 499
 500             next if !$fenced_nodes->{$service_node};
 501
 502             # node fence was successful - recover service
 503             $change_service_state->($self, $sid, 'recovery');
 504             $repeat = 1; # for faster recovery execution
 505         }
 506
 507         # Avoid that a node without services in 'fence' state (e.g., removed
 508         # manually by admin) is stuck with the 'fence' node state.
 509         for my $node (sort grep { !defined($fenced_nodes->{$_}) } keys $ns->{status}->%*) {
 510             next if $ns->get_node_state($node) ne 'fence';
 511
 512             $haenv->log('notice', "node '$node' in fence state but no services to-fence! admin interference?!");
 513             $repeat = 1 if $ns->fence_node($node);
 514         }
 515
 516         last if !$repeat;
 517     }
 518
 519     $self->flush_master_status();
 520 }
 521
 522 # functions to compute next service states
 523 # $cd: service configuration data (read only)
 524 # $sd: service status data (read only)
 525 #
 526 # Note: use change_service_state() to alter state
 527 #
 528
 529 sub next_state_request_stop {
 530     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 531
 532     my $haenv = $self->{haenv};
 533     my $ns = $self->{ns};
 534
 535     # check result from LRM daemon
 536     if ($lrm_res) {
 537         my $exit_code = $lrm_res->{exit_code};
 538         if ($exit_code == SUCCESS) {
 539             &$change_service_state($self, $sid, 'stopped');
 540             return;
 541         } else {
 542             $haenv->log('err', "service '$sid' stop failed (exit code $exit_code)");
 543             &$change_service_state($self, $sid, 'error'); # fixme: what state?
 544             return;
 545         }
 546     }
 547
 548     if ($ns->node_is_offline_delayed($sd->{node})) {
 549         &$change_service_state($self, $sid, 'fence');
 550         return;
 551     }
 552 }
 553
 554 sub next_state_migrate_relocate {
 555     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 556
 557     my $haenv = $self->{haenv};
 558     my $ns = $self->{ns};
 559
 560     # check result from LRM daemon
 561     if ($lrm_res) {
 562         my $exit_code = $lrm_res->{exit_code};
 563         my $req_state = $cd->{state} eq 'started' ? 'started' : 'request_stop';
 564         if ($exit_code == SUCCESS) {
 565             &$change_service_state($self, $sid, $req_state, node => $sd->{target});
 566             return;
 567         } elsif ($exit_code == EWRONG_NODE) {
 568             $haenv->log('err', "service '$sid' - migration failed: service" .
 569                         " registered on wrong node!");
 570             &$change_service_state($self, $sid, 'error');
 571         } else {
 572             $haenv->log('err', "service '$sid' - migration failed (exit code $exit_code)");
 573             &$change_service_state($self, $sid, $req_state, node => $sd->{node});
 574             return;
 575         }
 576     }
 577
 578     if ($ns->node_is_offline_delayed($sd->{node})) {
 579         &$change_service_state($self, $sid, 'fence');
 580         return;
 581     }
 582 }
 583
 584 sub next_state_stopped {
 585     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 586
 587     my $haenv = $self->{haenv};
 588     my $ns = $self->{ns};
 589
 590     if ($sd->{node} ne $cd->{node}) {
 591         # this can happen if we fence a node with active migrations
 592         # hack: modify $sd (normally this should be considered read-only)
 593         $haenv->log('info', "fixup service '$sid' location ($sd->{node} => $cd->{node})");
 594         $sd->{node} = $cd->{node};
 595     }
 596
 597     if ($sd->{cmd}) {
 598         my $cmd = shift @{$sd->{cmd}};
 599
 600         if ($cmd eq 'migrate' || $cmd eq 'relocate') {
 601             my $target = shift @{$sd->{cmd}};
 602             if (!$ns->node_is_online($target)) {
 603                 $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 604             } elsif ($sd->{node} eq $target) {
 605                 $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
 606             } else {
 607                 &$change_service_state($self, $sid, $cmd, node => $sd->{node},
 608                                        target => $target);
 609                 return;
 610             }
 611         } elsif ($cmd eq 'stop') {
 612                 $haenv->log('info', "ignore service '$sid' $cmd request - service already stopped");
 613         } else {
 614             $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 615         }
 616         delete $sd->{cmd};
 617     }
 618
 619     if ($cd->{state} eq 'disabled') {
 620         # NOTE: do nothing here, the stop state is an exception as we do not
 621         # process the LRM result here, thus the LRM always tries to stop the
 622         # service (protection for the case no CRM is active)
 623         return;
 624     }
 625
 626     if ($ns->node_is_offline_delayed($sd->{node}) && $ns->get_node_state($sd->{node}) ne 'maintenance') {
 627         &$change_service_state($self, $sid, 'fence');
 628         return;
 629     }
 630
 631     if ($cd->{state} eq 'stopped') {
 632         # almost the same as 'disabled' state but the service will also get recovered
 633         return;
 634     }
 635
 636     if ($cd->{state} eq 'started') {
 637         # simply mark it started, if it's on the wrong node
 638         # next_state_started will fix that for us
 639         &$change_service_state($self, $sid, 'started', node => $sd->{node});
 640         return;
 641     }
 642
 643     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 644 }
 645
 646 sub record_service_failed_on_node {
 647     my ($self, $sid, $node) = @_;
 648
 649     if (!defined($self->{ss}->{$sid}->{failed_nodes})) {
 650         $self->{ss}->{$sid}->{failed_nodes} = [];
 651     }
 652
 653     push @{$self->{ss}->{$sid}->{failed_nodes}}, $node;
 654 }
 655
 656 sub next_state_started {
 657     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 658
 659     my $haenv = $self->{haenv};
 660     my $master_status = $self->{ms};
 661     my $ns = $self->{ns};
 662
 663     if (!$ns->node_is_online($sd->{node})) {
 664         if ($ns->node_is_offline_delayed($sd->{node})) {
 665             &$change_service_state($self, $sid, 'fence');
 666         }
 667         if ($ns->get_node_state($sd->{node}) ne 'maintenance') {
 668             return;
 669         } else {
 670             # save current node as fallback for when it comes out of
 671             # maintenance
 672             $sd->{maintenance_node} = $sd->{node};
 673         }
 674     }
 675
 676     if ($cd->{state} eq 'disabled' || $cd->{state} eq 'stopped') {
 677         &$change_service_state($self, $sid, 'request_stop');
 678         return;
 679     }
 680
 681     if ($cd->{state} eq 'started') {
 682
 683         if ($sd->{cmd}) {
 684             my $cmd = shift @{$sd->{cmd}};
 685
 686             if ($cmd eq 'migrate' || $cmd eq 'relocate') {
 687                 my $target = shift @{$sd->{cmd}};
 688                 if (!$ns->node_is_online($target)) {
 689                     $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 690                 } elsif ($sd->{node} eq $target) {
 691                     $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
 692                 } else {
 693                     $haenv->log('info', "$cmd service '$sid' to node '$target'");
 694                     &$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
 695                 }
 696             } elsif ($cmd eq 'stop') {
 697                 my $timeout = shift @{$sd->{cmd}};
 698                 if ($timeout == 0) {
 699                     $haenv->log('info', "request immediate service hard-stop for service '$sid'");
 700                 } else {
 701                     $haenv->log('info', "request graceful stop with timeout '$timeout' for service '$sid'");
 702                 }
 703                 &$change_service_state($self, $sid, 'request_stop', timeout => $timeout);
 704                 $haenv->update_service_config($sid, {'state' => 'stopped'});
 705             } else {
 706                 $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 707             }
 708
 709             delete $sd->{cmd};
 710
 711         } else {
 712
 713             my $try_next = 0;
 714
 715             if ($lrm_res) {
 716
 717                 my $ec = $lrm_res->{exit_code};
 718                 if ($ec == SUCCESS) {
 719
 720                     if (defined($sd->{failed_nodes})) {
 721                         $haenv->log('info', "relocation policy successful for '$sid' on node '$sd->{node}'," .
 722                                     " failed nodes: " . join(', ', @{$sd->{failed_nodes}}) );
 723                     }
 724
 725                     delete $sd->{failed_nodes};
 726
 727                     # store flag to indicate successful start - only valid while state == 'started'
 728                     $sd->{running} = 1;
 729
 730                 } elsif ($ec == ERROR) {
 731
 732                     delete $sd->{running};
 733
 734                     # apply our relocate policy if we got ERROR from the LRM
 735                     $self->record_service_failed_on_node($sid, $sd->{node});
 736
 737                     if (scalar(@{$sd->{failed_nodes}}) <= $cd->{max_relocate}) {
 738
 739                         # tell select_service_node to relocate if possible
 740                         $try_next = 1;
 741
 742                         $haenv->log('warning', "starting service $sid on node".
 743                                    " '$sd->{node}' failed, relocating service.");
 744
 745                     } else {
 746
 747                         $haenv->log('err', "recovery policy for service $sid " .
 748                                     "failed, entering error state. Failed nodes: ".
 749                                     join(', ', @{$sd->{failed_nodes}}));
 750                         &$change_service_state($self, $sid, 'error');
 751                         return;
 752
 753                     }
 754                 } else {
 755                     $self->record_service_failed_on_node($sid, $sd->{node});
 756
 757                     $haenv->log('err', "service '$sid' got unrecoverable error" .
 758                                 " (exit code $ec))");
 759                     # we have no save way out (yet) for other errors
 760                     &$change_service_state($self, $sid, 'error');
 761                     return;
 762                 }
 763             }
 764
 765             my $node = select_service_node(
 766                 $self->{groups},
 767                 $self->{online_node_usage},
 768                 $sid,
 769                 $cd,
 770                 $sd->{node},
 771                 $try_next,
 772                 $sd->{failed_nodes},
 773                 $sd->{maintenance_node},
 774             );
 775
 776             if ($node && ($sd->{node} ne $node)) {
 777                 $self->{online_node_usage}->add_service_usage_to_node($node, $sid, $sd->{node});
 778
 779                 if (defined(my $fallback = $sd->{maintenance_node})) {
 780                     if ($node eq $fallback) {
 781                         $haenv->log('info', "moving service '$sid' back to '$fallback', node came back from maintenance.");
 782                         delete $sd->{maintenance_node};
 783                     } elsif ($sd->{node} ne $fallback) {
 784                         $haenv->log('info', "dropping maintenance fallback node '$fallback' for '$sid'");
 785                         delete $sd->{maintenance_node};
 786                     }
 787                 }
 788
 789                 if ($cd->{type} eq 'vm') {
 790                     $haenv->log('info', "migrate service '$sid' to node '$node' (running)");
 791                     &$change_service_state($self, $sid, 'migrate', node => $sd->{node}, target => $node);
 792                 } else {
 793                     $haenv->log('info', "relocate service '$sid' to node '$node'");
 794                     &$change_service_state($self, $sid, 'relocate', node => $sd->{node}, target => $node);
 795                 }
 796             } else {
 797                 if ($try_next && !defined($node)) {
 798                     $haenv->log('warning', "Start Error Recovery: Tried all available " .
 799                                 " nodes for service '$sid', retry start on current node. " .
 800                                 "Tried nodes: " . join(', ', @{$sd->{failed_nodes}}));
 801                 }
 802                 # ensure service get started again if it went unexpected down
 803                 # but ensure also no LRM result gets lost
 804                 $sd->{uid} = compute_new_uuid($sd->{state}) if defined($lrm_res);
 805             }
 806         }
 807
 808         return;
 809     }
 810
 811     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 812 }
 813
 814 sub next_state_error {
 815     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 816
 817     my $ns = $self->{ns};
 818     my $ms = $self->{ms};
 819
 820     if ($cd->{state} eq 'disabled') {
 821         # clean up on error recovery
 822         delete $sd->{failed_nodes};
 823
 824         &$change_service_state($self, $sid, 'stopped');
 825         return;
 826     }
 827
 828 }
 829
 830 # after a node was fenced this recovers the service to a new node
 831 sub next_state_recovery {
 832     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 833
 834     my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
 835     my $ns = $self->{ns};
 836     my $ms = $self->{ms};
 837
 838     if ($sd->{state} ne 'recovery') { # should not happen
 839         $haenv->log('err', "cannot recover service '$sid' from fencing, wrong state '$sd->{state}'");
 840         return;
 841     }
 842
 843     my $fenced_node = $sd->{node}; # for logging purpose
 844
 845     $self->recompute_online_node_usage(); # we want the most current node state
 846
 847     my $recovery_node = select_service_node(
 848         $self->{groups},
 849         $self->{online_node_usage},
 850         $sid,
 851         $cd,
 852         $sd->{node},
 853     );
 854
 855     if ($recovery_node) {
 856         my $msg = "recover service '$sid' from fenced node '$fenced_node' to node '$recovery_node'";
 857         if ($recovery_node eq $fenced_node) {
 858             # can happen if restriced groups and the node came up again OK
 859             $msg = "recover service '$sid' to previous failed and fenced node '$fenced_node' again";
 860         }
 861         $haenv->log('info', "$msg");
 862
 863         $fence_recovery_cleanup->($self, $sid, $fenced_node);
 864
 865         $haenv->steal_service($sid, $sd->{node}, $recovery_node);
 866         $self->{online_node_usage}->add_service_usage_to_node($recovery_node, $sid, $recovery_node);
 867
 868         # NOTE: $sd *is normally read-only*, fencing is the exception
 869         $cd->{node} = $sd->{node} = $recovery_node;
 870         my $new_state = ($cd->{state} eq 'started') ? 'started' : 'request_stop';
 871         $change_service_state->($self, $sid, $new_state, node => $recovery_node);
 872     } else {
 873         # no possible node found, cannot recover - but retry later, as we always try to make it available
 874         $haenv->log('err', "recovering service '$sid' from fenced node '$fenced_node' failed, no recovery node found");
 875
 876         if ($cd->{state} eq 'disabled') {
 877             # allow getting a service out of recovery manually if an admin disables it.
 878             delete $sd->{failed_nodes}; # clean up on recovery to stopped
 879             $change_service_state->($self, $sid, 'stopped'); # must NOT go through request_stop
 880             return;
 881         }
 882     }
 883 }
 884
 885 1;