src/PVE/HA/Manager.pm

   1 package PVE::HA::Manager;
   2
   3 use strict;
   4 use warnings;
   5 use Digest::MD5 qw(md5_base64);
   6
   7 use Data::Dumper;
   8 use PVE::Tools;
   9 use PVE::HA::Tools ':exit_codes';
  10 use PVE::HA::NodeStatus;
  11
  12 sub new {
  13     my ($this, $haenv) = @_;
  14
  15     my $class = ref($this) || $this;
  16
  17     my $self = bless { haenv => $haenv }, $class;
  18
  19     my $old_ms = $haenv->read_manager_status();
  20
  21     # we only copy the state part of the manager which cannot be auto generated
  22
  23     $self->{ns} = PVE::HA::NodeStatus->new($haenv, $old_ms->{node_status} || {});
  24
  25     # fixme: use separate class  PVE::HA::ServiceStatus
  26     $self->{ss} = $old_ms->{service_status} || {};
  27
  28     $self->{ms} = { master_node => $haenv->nodename() };
  29
  30     return $self;
  31 }
  32
  33 sub cleanup {
  34     my ($self) = @_;
  35
  36     # todo: ?
  37 }
  38
  39 sub flush_master_status {
  40     my ($self) = @_;
  41
  42     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
  43
  44     $ms->{node_status} = $ns->{status};
  45     $ms->{service_status} = $ss;
  46     $ms->{timestamp} = $haenv->get_time();
  47
  48     $haenv->write_manager_status($ms);
  49 }
  50
  51 sub get_service_group {
  52     my ($groups, $online_node_usage, $service_conf) = @_;
  53
  54     my $group = {};
  55     # add all online nodes to default group to allow try_next when no group set
  56     foreach my $node (keys %$online_node_usage) {
  57         $group->{nodes}->{$node} = 1;
  58     }
  59
  60     # overwrite default if service is bound to a specific group
  61     $group =  $groups->{ids}->{$service_conf->{group}} if $service_conf->{group} &&
  62         $groups->{ids}->{$service_conf->{group}};
  63
  64     return $group;
  65 }
  66
  67 # groups available nodes with their priority as group index
  68 sub get_node_priority_groups {
  69     my ($group, $online_node_usage) = @_;
  70
  71     my $pri_groups = {};
  72     my $group_members = {};
  73     foreach my $entry (keys %{$group->{nodes}}) {
  74         my ($node, $pri) = ($entry, 0);
  75         if ($entry =~ m/^(\S+):(\d+)$/) {
  76             ($node, $pri) = ($1, $2);
  77         }
  78         next if !defined($online_node_usage->{$node}); # offline
  79         $pri_groups->{$pri}->{$node} = 1;
  80         $group_members->{$node} = $pri;
  81     }
  82
  83     # add non-group members to unrestricted groups (priority -1)
  84     if (!$group->{restricted}) {
  85         my $pri = -1;
  86         foreach my $node (keys %$online_node_usage) {
  87             next if defined($group_members->{$node});
  88             $pri_groups->{$pri}->{$node} = 1;
  89             $group_members->{$node} = -1;
  90         }
  91     }
  92
  93     return ($pri_groups, $group_members);
  94 }
  95
  96 sub select_service_node {
  97     my ($groups, $online_node_usage, $service_conf, $current_node, $try_next, $tried_nodes) = @_;
  98
  99     my $group = get_service_group($groups, $online_node_usage, $service_conf);
 100
 101     my ($pri_groups, $group_members) = get_node_priority_groups($group, $online_node_usage);
 102
 103     my @pri_list = sort {$b <=> $a} keys %$pri_groups;
 104     return undef if !scalar(@pri_list);
 105
 106     # stay on current node if possible (avoids random migrations)
 107     if (!$try_next && $group->{nofailback} && defined($group_members->{$current_node})) {
 108         return $current_node;
 109     }
 110
 111     # select node from top priority node list
 112
 113     my $top_pri = $pri_list[0];
 114
 115     # try to avoid nodes where the service failed already if we want to relocate
 116     if ($try_next) {
 117         foreach my $node (@$tried_nodes) {
 118             delete $pri_groups->{$top_pri}->{$node};
 119         }
 120     }
 121
 122     my @nodes = sort {
 123         $online_node_usage->{$a} <=> $online_node_usage->{$b} || $a cmp $b
 124     } keys %{$pri_groups->{$top_pri}};
 125
 126     my $found;
 127     for (my $i = scalar(@nodes) - 1; $i >= 0; $i--) {
 128         my $node = $nodes[$i];
 129         if ($node eq $current_node) {
 130             $found = $i;
 131             last;
 132         }
 133     }
 134
 135     if ($try_next) {
 136
 137         if (defined($found) && ($found < (scalar(@nodes) - 1))) {
 138             return $nodes[$found + 1];
 139         } else {
 140             return $nodes[0];
 141         }
 142
 143     } else {
 144
 145         return $nodes[$found] if defined($found);
 146
 147         return $nodes[0];
 148
 149     }
 150 }
 151
 152 my $uid_counter = 0;
 153
 154 sub compute_new_uuid {
 155     my ($state) = @_;
 156
 157     $uid_counter++;
 158     return md5_base64($state . $$ . time() . $uid_counter);
 159 }
 160
 161 my $valid_service_states = {
 162     stopped => 1,
 163     request_stop => 1,
 164     started => 1,
 165     fence => 1,
 166     migrate => 1,
 167     relocate => 1,
 168     freeze => 1,
 169     error => 1,
 170 };
 171
 172 sub recompute_online_node_usage {
 173     my ($self) = @_;
 174
 175     my $online_node_usage = {};
 176
 177     my $online_nodes = $self->{ns}->list_online_nodes();
 178
 179     foreach my $node (@$online_nodes) {
 180         $online_node_usage->{$node} = 0;
 181     }
 182
 183     foreach my $sid (keys %{$self->{ss}}) {
 184         my $sd = $self->{ss}->{$sid};
 185         my $state = $sd->{state};
 186         if (defined($online_node_usage->{$sd->{node}})) {
 187             if (($state eq 'started') || ($state eq 'request_stop') ||
 188                 ($state eq 'fence') || ($state eq 'freeze') || ($state eq 'error')) {
 189                 $online_node_usage->{$sd->{node}}++;
 190             } elsif (($state eq 'migrate') || ($state eq 'relocate')) {
 191                 $online_node_usage->{$sd->{target}}++;
 192             } elsif ($state eq 'stopped') {
 193                 # do nothing
 194             } else {
 195                 die "should not be reached";
 196             }
 197         }
 198     }
 199
 200     $self->{online_node_usage} = $online_node_usage;
 201 }
 202
 203 my $change_service_state = sub {
 204     my ($self, $sid, $new_state, %params) = @_;
 205
 206     my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
 207
 208     my $sd = $ss->{$sid} || die "no such service '$sid";
 209
 210     my $old_state = $sd->{state};
 211     my $old_node = $sd->{node};
 212     my $old_failed_nodes = $sd->{failed_nodes};
 213
 214     die "no state change" if $old_state eq $new_state; # just to be sure
 215
 216     die "invalid CRM service state '$new_state'\n" if !$valid_service_states->{$new_state};
 217
 218     foreach my $k (keys %$sd) { delete $sd->{$k}; };
 219
 220     $sd->{state} = $new_state;
 221     $sd->{node} = $old_node;
 222     $sd->{failed_nodes} = $old_failed_nodes;
 223
 224     my $text_state = '';
 225     foreach my $k (sort keys %params) {
 226         my $v = $params{$k};
 227         $text_state .= ", " if $text_state;
 228         $text_state .= "$k = $v";
 229         $sd->{$k} = $v;
 230     }
 231
 232     $self->recompute_online_node_usage();
 233
 234     $sd->{uid} = compute_new_uuid($new_state);
 235
 236     $text_state = "  ($text_state)" if $text_state;
 237     $haenv->log('info', "service '$sid': state changed from '${old_state}'" .
 238                 " to '${new_state}'$text_state");
 239 };
 240
 241 # clean up a possible bad state from a recovered service to allow its start
 242 my $fence_recovery_cleanup = sub {
 243     my ($self, $sid, $fenced_node) = @_;
 244
 245     my $haenv = $self->{haenv};
 246
 247     my (undef, $type, $id) = PVE::HA::Tools::parse_sid($sid);
 248     my $plugin = PVE::HA::Resources->lookup($type);
 249
 250     # should not happen
 251     die "unknown resource type '$type'" if !$plugin;
 252
 253     # locks may block recovery, cleanup those which are safe to remove after fencing
 254     my $removable_locks = ['backup', 'mounted'];
 255     if (my $removed_lock = $plugin->remove_locks($haenv, $id, $removable_locks, $fenced_node)) {
 256         $haenv->log('warning', "removed leftover lock '$removed_lock' from recovered " .
 257                     "service '$sid' to allow its start.");
 258     }
 259 };
 260
 261 # after a node was fenced this recovers the service to a new node
 262 my $recover_fenced_service = sub {
 263     my ($self, $sid, $cd) = @_;
 264
 265     my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
 266
 267     my $sd = $ss->{$sid};
 268
 269     if ($sd->{state} ne 'fence') { # should not happen
 270         $haenv->log('err', "cannot recover service '$sid' from fencing," .
 271                     " wrong state '$sd->{state}'");
 272         return;
 273     }
 274
 275     my $fenced_node = $sd->{node}; # for logging purpose
 276
 277     $self->recompute_online_node_usage(); # we want the most current node state
 278
 279     my $recovery_node = select_service_node($self->{groups},
 280                                             $self->{online_node_usage},
 281                                             $cd, $sd->{node});
 282
 283     if ($recovery_node) {
 284         $haenv->log('info', "recover service '$sid' from fenced node " .
 285                     "'$fenced_node' to node '$recovery_node'");
 286
 287         &$fence_recovery_cleanup($self, $sid, $fenced_node);
 288
 289         $haenv->steal_service($sid, $sd->{node}, $recovery_node);
 290
 291         # $sd *is normally read-only*, fencing is the exception
 292         $cd->{node} = $sd->{node} = $recovery_node;
 293         my $new_state = ($cd->{state} eq 'started') ? 'started' : 'request_stop';
 294         &$change_service_state($self, $sid, $new_state, node => $recovery_node);
 295     } else {
 296         # no possible node found, cannot recover
 297         $haenv->log('err', "recovering service '$sid' from fenced node " .
 298                     "'$fenced_node' failed, no recovery node found");
 299         &$change_service_state($self, $sid, 'error');
 300     }
 301 };
 302
 303 # read LRM status for all nodes
 304 sub read_lrm_status {
 305     my ($self) = @_;
 306
 307     my $nodes = $self->{ns}->list_nodes();
 308     my $haenv = $self->{haenv};
 309
 310     my $results = {};
 311     my $modes = {};
 312     foreach my $node (@$nodes) {
 313         my $lrm_status = $haenv->read_lrm_status($node);
 314         $modes->{$node} = $lrm_status->{mode} || 'active';
 315         foreach my $uid (keys %{$lrm_status->{results}}) {
 316             next if $results->{$uid}; # should not happen
 317             $results->{$uid} = $lrm_status->{results}->{$uid};
 318         }
 319     }
 320
 321
 322     return ($results, $modes);
 323 }
 324
 325 # read new crm commands and save them into crm master status
 326 sub update_crm_commands {
 327     my ($self) = @_;
 328
 329     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 330
 331     my $cmdlist = $haenv->read_crm_commands();
 332
 333     foreach my $cmd (split(/\n/, $cmdlist)) {
 334         chomp $cmd;
 335
 336         if ($cmd =~ m/^(migrate|relocate)\s+(\S+)\s+(\S+)$/) {
 337             my ($task, $sid, $node) = ($1, $2, $3);
 338             if (my $sd = $ss->{$sid}) {
 339                 if (!$ns->node_is_online($node)) {
 340                     $haenv->log('err', "crm command error - node not online: $cmd");
 341                 } else {
 342                     if ($node eq $sd->{node}) {
 343                         $haenv->log('info', "ignore crm command - service already on target node: $cmd");
 344                     } else {
 345                         $haenv->log('info', "got crm command: $cmd");
 346                         $ss->{$sid}->{cmd} = [ $task, $node];
 347                     }
 348                 }
 349             } else {
 350                 $haenv->log('err', "crm command error - no such service: $cmd");
 351             }
 352
 353         } else {
 354             $haenv->log('err', "unable to parse crm command: $cmd");
 355         }
 356     }
 357
 358 }
 359
 360 sub manage {
 361     my ($self) = @_;
 362
 363     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 364
 365     $ns->update($haenv->get_node_info());
 366
 367     if (!$ns->node_is_online($haenv->nodename())) {
 368         $haenv->log('info', "master seems offline");
 369         return;
 370     }
 371
 372     my ($lrm_results, $lrm_modes) = $self->read_lrm_status();
 373
 374     my $sc = $haenv->read_service_config();
 375
 376     $self->{groups} = $haenv->read_group_config(); # update
 377
 378     # compute new service status
 379
 380     # add new service
 381     foreach my $sid (sort keys %$sc) {
 382         next if $ss->{$sid}; # already there
 383         my $cd = $sc->{$sid};
 384         $haenv->log('info', "adding new service '$sid' on node '$cd->{node}'");
 385         # assume we are running to avoid relocate running service at add
 386         my $state = ($cd->{state} eq 'started') ? 'started' : 'request_stop';
 387         $ss->{$sid} = { state => $state, node => $cd->{node},
 388                         uid => compute_new_uuid('started') };
 389     }
 390
 391     # remove stale service from manager state
 392     foreach my $sid (keys %$ss) {
 393         next if $sc->{$sid};
 394         $haenv->log('info', "removing stale service '$sid' (no config)");
 395         # remove all service related state information
 396         delete $ss->{$sid};
 397     }
 398
 399     $self->update_crm_commands();
 400
 401     for (;;) {
 402         my $repeat = 0;
 403
 404         $self->recompute_online_node_usage();
 405
 406         foreach my $sid (sort keys %$ss) {
 407             my $sd = $ss->{$sid};
 408             my $cd = $sc->{$sid} || { state => 'disabled' };
 409
 410             my $lrm_res = $sd->{uid} ? $lrm_results->{$sd->{uid}} : undef;
 411
 412             my $last_state = $sd->{state};
 413
 414             if ($last_state eq 'stopped') {
 415
 416                 $self->next_state_stopped($sid, $cd, $sd, $lrm_res);
 417
 418             } elsif ($last_state eq 'started') {
 419
 420                 $self->next_state_started($sid, $cd, $sd, $lrm_res);
 421
 422             } elsif ($last_state eq 'migrate' || $last_state eq 'relocate') {
 423
 424                 $self->next_state_migrate_relocate($sid, $cd, $sd, $lrm_res);
 425
 426             } elsif ($last_state eq 'fence') {
 427
 428                 # do nothing here - wait until fenced
 429
 430             } elsif ($last_state eq 'request_stop') {
 431
 432                 $self->next_state_request_stop($sid, $cd, $sd, $lrm_res);
 433
 434             } elsif ($last_state eq 'freeze') {
 435
 436                 my $lrm_mode = $sd->{node} ? $lrm_modes->{$sd->{node}} : undef;
 437                 # unfreeze
 438                 my $state = ($cd->{state} eq 'started') ? 'started' : 'request_stop';
 439                 &$change_service_state($self, $sid, $state)
 440                     if $lrm_mode && $lrm_mode eq 'active';
 441
 442             } elsif ($last_state eq 'error') {
 443
 444                 $self->next_state_error($sid, $cd, $sd, $lrm_res);
 445
 446             } else {
 447
 448                 die "unknown service state '$last_state'";
 449             }
 450
 451             my $lrm_mode = $sd->{node} ? $lrm_modes->{$sd->{node}} : undef;
 452             if ($lrm_mode && $lrm_mode eq 'restart') {
 453                 if (($sd->{state} eq 'started' || $sd->{state} eq 'stopped' ||
 454                      $sd->{state} eq 'request_stop')) {
 455                     &$change_service_state($self, $sid, 'freeze');
 456                 }
 457             }
 458
 459             $repeat = 1 if $sd->{state} ne $last_state;
 460         }
 461
 462         # handle fencing
 463         my $fenced_nodes = {};
 464         foreach my $sid (sort keys %$ss) {
 465             my $sd = $ss->{$sid};
 466             next if $sd->{state} ne 'fence';
 467
 468             if (!defined($fenced_nodes->{$sd->{node}})) {
 469                 $fenced_nodes->{$sd->{node}} = $ns->fence_node($sd->{node}) || 0;
 470             }
 471
 472             next if !$fenced_nodes->{$sd->{node}};
 473
 474             # node fence was successful - recover service
 475             &$recover_fenced_service($self, $sid, $sc->{$sid});
 476         }
 477
 478         last if !$repeat;
 479     }
 480
 481     $self->flush_master_status();
 482 }
 483
 484 # functions to compute next service states
 485 # $cd: service configuration data (read only)
 486 # $sd: service status data (read only)
 487 #
 488 # Note: use change_service_state() to alter state
 489 #
 490
 491 sub next_state_request_stop {
 492     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 493
 494     my $haenv = $self->{haenv};
 495     my $ns = $self->{ns};
 496
 497     # check result from LRM daemon
 498     if ($lrm_res) {
 499         my $exit_code = $lrm_res->{exit_code};
 500         if ($exit_code == SUCCESS) {
 501             &$change_service_state($self, $sid, 'stopped');
 502             return;
 503         } else {
 504             $haenv->log('err', "service '$sid' stop failed (exit code $exit_code)");
 505             &$change_service_state($self, $sid, 'error'); # fixme: what state?
 506             return;
 507         }
 508     }
 509
 510     if ($ns->node_is_offline_delayed($sd->{node})) {
 511         &$change_service_state($self, $sid, 'fence');
 512         return;
 513     }
 514 }
 515
 516 sub next_state_migrate_relocate {
 517     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 518
 519     my $haenv = $self->{haenv};
 520     my $ns = $self->{ns};
 521
 522     # check result from LRM daemon
 523     if ($lrm_res) {
 524         my $exit_code = $lrm_res->{exit_code};
 525         my $req_state = $cd->{state} eq 'started' ? 'started' : 'request_stop';
 526         if ($exit_code == SUCCESS) {
 527             &$change_service_state($self, $sid, $req_state, node => $sd->{target});
 528             return;
 529         } elsif ($exit_code == EWRONG_NODE) {
 530             $haenv->log('err', "service '$sid' - migration failed: service" .
 531                         " registered on wrong node!");
 532             &$change_service_state($self, $sid, 'error');
 533         } else {
 534             $haenv->log('err', "service '$sid' - migration failed (exit code $exit_code)");
 535             &$change_service_state($self, $sid, $req_state, node => $sd->{node});
 536             return;
 537         }
 538     }
 539
 540     if ($ns->node_is_offline_delayed($sd->{node})) {
 541         &$change_service_state($self, $sid, 'fence');
 542         return;
 543     }
 544 }
 545
 546
 547 sub next_state_stopped {
 548     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 549
 550     my $haenv = $self->{haenv};
 551     my $ns = $self->{ns};
 552
 553     if ($sd->{node} ne $cd->{node}) {
 554         # this can happen if we fence a node with active migrations
 555         # hack: modify $sd (normally this should be considered read-only)
 556         $haenv->log('info', "fixup service '$sid' location ($sd->{node} => $cd->{node})");
 557         $sd->{node} = $cd->{node};
 558     }
 559
 560     if ($sd->{cmd}) {
 561         my ($cmd, $target) = @{$sd->{cmd}};
 562         delete $sd->{cmd};
 563
 564         if ($cmd eq 'migrate' || $cmd eq 'relocate') {
 565             if (!$ns->node_is_online($target)) {
 566                 $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 567             } elsif ($sd->{node} eq $target) {
 568                 $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
 569             } else {
 570                 &$change_service_state($self, $sid, $cmd, node => $sd->{node},
 571                                        target => $target);
 572                 return;
 573             }
 574         } else {
 575             $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 576         }
 577     }
 578
 579     if ($cd->{state} eq 'disabled') {
 580         # NOTE: do nothing here, the stop state is an exception as we do not
 581         # process the LRM result here, thus the LRM always tries to stop the
 582         # service (protection for the case no CRM is active)
 583         return;
 584     }
 585
 586     if ($ns->node_is_offline_delayed($sd->{node})) {
 587         &$change_service_state($self, $sid, 'fence');
 588         return;
 589     }
 590
 591     if ($cd->{state} eq 'stopped') {
 592         # almost the same as 'disabled' state but the service will also get recovered
 593         return;
 594     }
 595
 596     if ($cd->{state} eq 'started') {
 597         # simply mark it started, if it's on the wrong node
 598         # next_state_started will fix that for us
 599         &$change_service_state($self, $sid, 'started', node => $sd->{node});
 600         return;
 601     }
 602
 603     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 604 }
 605
 606 sub record_service_failed_on_node {
 607     my ($self, $sid, $node) = @_;
 608
 609     if (!defined($self->{ss}->{$sid}->{failed_nodes})) {
 610         $self->{ss}->{$sid}->{failed_nodes} = [];
 611     }
 612
 613     push @{$self->{ss}->{$sid}->{failed_nodes}}, $node;
 614 }
 615
 616 sub next_state_started {
 617     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 618
 619     my $haenv = $self->{haenv};
 620     my $master_status = $self->{ms};
 621     my $ns = $self->{ns};
 622
 623     if (!$ns->node_is_online($sd->{node})) {
 624         if ($ns->node_is_offline_delayed($sd->{node})) {
 625             &$change_service_state($self, $sid, 'fence');
 626         }
 627         return;
 628     }
 629
 630     if ($cd->{state} eq 'disabled' || $cd->{state} eq 'stopped') {
 631         &$change_service_state($self, $sid, 'request_stop');
 632         return;
 633     }
 634
 635     if ($cd->{state} eq 'started') {
 636
 637         if ($sd->{cmd}) {
 638             my ($cmd, $target) = @{$sd->{cmd}};
 639             delete $sd->{cmd};
 640
 641             if ($cmd eq 'migrate' || $cmd eq 'relocate') {
 642                 if (!$ns->node_is_online($target)) {
 643                     $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 644                 } elsif ($sd->{node} eq $target) {
 645                     $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
 646                 } else {
 647                     $haenv->log('info', "$cmd service '$sid' to node '$target'");
 648                     &$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
 649                 }
 650             } else {
 651                 $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 652             }
 653         } else {
 654
 655             my $try_next = 0;
 656
 657             if ($lrm_res) {
 658
 659                 my $ec = $lrm_res->{exit_code};
 660                 if ($ec == SUCCESS) {
 661
 662                     if (defined($sd->{failed_nodes})) {
 663                         $haenv->log('info', "relocation policy successful for '$sid' on node '$sd->{node}'," .
 664                                     " failed nodes: " . join(', ', @{$sd->{failed_nodes}}) );
 665                     }
 666
 667                     delete $sd->{failed_nodes};
 668
 669                 } elsif ($ec == ERROR) {
 670                     # apply our relocate policy if we got ERROR from the LRM
 671                     $self->record_service_failed_on_node($sid, $sd->{node});
 672
 673                     if (scalar(@{$sd->{failed_nodes}}) <= $cd->{max_relocate}) {
 674
 675                         # tell select_service_node to relocate if possible
 676                         $try_next = 1;
 677
 678                         $haenv->log('warning', "starting service $sid on node".
 679                                    " '$sd->{node}' failed, relocating service.");
 680
 681                     } else {
 682
 683                         $haenv->log('err', "recovery policy for service $sid " .
 684                                     "failed, entering error state. Failed nodes: ".
 685                                     join(', ', @{$sd->{failed_nodes}}));
 686                         &$change_service_state($self, $sid, 'error');
 687                         return;
 688
 689                     }
 690                 } else {
 691                     $self->record_service_failed_on_node($sid, $sd->{node});
 692
 693                     $haenv->log('err', "service '$sid' got unrecoverable error" .
 694                                 " (exit code $ec))");
 695                     # we have no save way out (yet) for other errors
 696                     &$change_service_state($self, $sid, 'error');
 697                     return;
 698                 }
 699             }
 700
 701             my $node = select_service_node($self->{groups}, $self->{online_node_usage},
 702                                            $cd, $sd->{node}, $try_next, $sd->{failed_nodes});
 703
 704             if ($node && ($sd->{node} ne $node)) {
 705                 if ($cd->{type} eq 'vm') {
 706                     $haenv->log('info', "migrate service '$sid' to node '$node' (running)");
 707                     &$change_service_state($self, $sid, 'migrate', node => $sd->{node}, target => $node);
 708                 } else {
 709                     $haenv->log('info', "relocate service '$sid' to node '$node'");
 710                     &$change_service_state($self, $sid, 'relocate', node => $sd->{node}, target => $node);
 711                 }
 712             } else {
 713                 if ($try_next && !defined($node)) {
 714                     $haenv->log('warning', "Start Error Recovery: Tried all available " .
 715                                 " nodes for service '$sid', retry start on current node. " .
 716                                 "Tried nodes: " . join(', ', @{$sd->{failed_nodes}}));
 717                 }
 718                 # ensure service get started again if it went unexpected down
 719                 # but ensure also no LRM result gets lost
 720                 $sd->{uid} = compute_new_uuid($sd->{state}) if defined($lrm_res);
 721             }
 722         }
 723
 724         return;
 725     }
 726
 727     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 728 }
 729
 730 sub next_state_error {
 731     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 732
 733     my $ns = $self->{ns};
 734     my $ms = $self->{ms};
 735
 736     if ($cd->{state} eq 'disabled') {
 737         # clean up on error recovery
 738         delete $sd->{failed_nodes};
 739
 740         &$change_service_state($self, $sid, 'stopped');
 741         return;
 742     }
 743
 744 }
 745
 746 1;