src/PVE/HA/Manager.pm

   1 package PVE::HA::Manager;
   2
   3 use strict;
   4 use warnings;
   5 use Digest::MD5 qw(md5_base64);
   6
   7 use Data::Dumper;
   8 use PVE::Tools;
   9 use PVE::HA::Tools ':exit_codes';
  10 use PVE::HA::NodeStatus;
  11
  12 sub new {
  13     my ($this, $haenv) = @_;
  14
  15     my $class = ref($this) || $this;
  16
  17     my $self = bless { haenv => $haenv }, $class;
  18
  19     my $old_ms = $haenv->read_manager_status();
  20
  21     # we only copy the state part of the manager which cannot be auto generated
  22
  23     $self->{ns} = PVE::HA::NodeStatus->new($haenv, $old_ms->{node_status} || {});
  24
  25     # fixme: use separate class  PVE::HA::ServiceStatus
  26     $self->{ss} = $old_ms->{service_status} || {};
  27
  28     $self->{ms} = { master_node => $haenv->nodename() };
  29
  30     return $self;
  31 }
  32
  33 sub cleanup {
  34     my ($self) = @_;
  35
  36     # todo: ?
  37 }
  38
  39 sub flush_master_status {
  40     my ($self) = @_;
  41
  42     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
  43
  44     $ms->{node_status} = $ns->{status};
  45     $ms->{service_status} = $ss;
  46     $ms->{timestamp} = $haenv->get_time();
  47
  48     $haenv->write_manager_status($ms);
  49 }
  50
  51 sub get_service_group {
  52     my ($groups, $online_node_usage, $service_conf) = @_;
  53
  54     my $group = {};
  55     # add all online nodes to default group to allow try_next when no group set
  56     foreach my $node (keys %$online_node_usage) {
  57         $group->{nodes}->{$node} = 1;
  58     }
  59
  60     # overwrite default if service is bound to a specific group
  61     $group =  $groups->{ids}->{$service_conf->{group}} if $service_conf->{group} &&
  62         $groups->{ids}->{$service_conf->{group}};
  63
  64     return $group;
  65 }
  66
  67 # groups available nodes with their priority as group index
  68 sub get_node_priority_groups {
  69     my ($group, $online_node_usage) = @_;
  70
  71     my $pri_groups = {};
  72     my $group_members = {};
  73     foreach my $entry (keys %{$group->{nodes}}) {
  74         my ($node, $pri) = ($entry, 0);
  75         if ($entry =~ m/^(\S+):(\d+)$/) {
  76             ($node, $pri) = ($1, $2);
  77         }
  78         next if !defined($online_node_usage->{$node}); # offline
  79         $pri_groups->{$pri}->{$node} = 1;
  80         $group_members->{$node} = $pri;
  81     }
  82
  83     # add non-group members to unrestricted groups (priority -1)
  84     if (!$group->{restricted}) {
  85         my $pri = -1;
  86         foreach my $node (keys %$online_node_usage) {
  87             next if defined($group_members->{$node});
  88             $pri_groups->{$pri}->{$node} = 1;
  89             $group_members->{$node} = -1;
  90         }
  91     }
  92
  93     return ($pri_groups, $group_members);
  94 }
  95
  96 sub select_service_node {
  97     my ($groups, $online_node_usage, $service_conf, $current_node, $try_next, $tried_nodes) = @_;
  98
  99     my $group = get_service_group($groups, $online_node_usage, $service_conf);
 100
 101     my ($pri_groups, $group_members) = get_node_priority_groups($group, $online_node_usage);
 102
 103     my @pri_list = sort {$b <=> $a} keys %$pri_groups;
 104     return undef if !scalar(@pri_list);
 105
 106     # stay on current node if possible (avoids random migrations)
 107     if (!$try_next && $group->{nofailback} && defined($group_members->{$current_node})) {
 108         return $current_node;
 109     }
 110
 111     # select node from top priority node list
 112
 113     my $top_pri = $pri_list[0];
 114
 115     # try to avoid nodes where the service failed already if we want to relocate
 116     if ($try_next) {
 117         foreach my $node (@$tried_nodes) {
 118             delete $pri_groups->{$top_pri}->{$node};
 119         }
 120     }
 121
 122     my @nodes = sort {
 123         $online_node_usage->{$a} <=> $online_node_usage->{$b} || $a cmp $b
 124     } keys %{$pri_groups->{$top_pri}};
 125
 126     my $found;
 127     for (my $i = scalar(@nodes) - 1; $i >= 0; $i--) {
 128         my $node = $nodes[$i];
 129         if ($node eq $current_node) {
 130             $found = $i;
 131             last;
 132         }
 133     }
 134
 135     if ($try_next) {
 136
 137         if (defined($found) && ($found < (scalar(@nodes) - 1))) {
 138             return $nodes[$found + 1];
 139         } else {
 140             return $nodes[0];
 141         }
 142
 143     } else {
 144
 145         return $nodes[$found] if defined($found);
 146
 147         return $nodes[0];
 148
 149     }
 150 }
 151
 152 my $uid_counter = 0;
 153
 154 sub compute_new_uuid {
 155     my ($state) = @_;
 156
 157     $uid_counter++;
 158     return md5_base64($state . $$ . time() . $uid_counter);
 159 }
 160
 161 my $valid_service_states = {
 162     stopped => 1,
 163     request_stop => 1,
 164     started => 1,
 165     fence => 1,
 166     migrate => 1,
 167     relocate => 1,
 168     freeze => 1,
 169     error => 1,
 170 };
 171
 172 sub recompute_online_node_usage {
 173     my ($self) = @_;
 174
 175     my $online_node_usage = {};
 176
 177     my $online_nodes = $self->{ns}->list_online_nodes();
 178
 179     foreach my $node (@$online_nodes) {
 180         $online_node_usage->{$node} = 0;
 181     }
 182
 183     foreach my $sid (keys %{$self->{ss}}) {
 184         my $sd = $self->{ss}->{$sid};
 185         my $state = $sd->{state};
 186         if (defined($online_node_usage->{$sd->{node}})) {
 187             if (($state eq 'started') || ($state eq 'request_stop') ||
 188                 ($state eq 'fence') || ($state eq 'freeze') || ($state eq 'error')) {
 189                 $online_node_usage->{$sd->{node}}++;
 190             } elsif (($state eq 'migrate') || ($state eq 'relocate')) {
 191                 $online_node_usage->{$sd->{target}}++;
 192             } elsif ($state eq 'stopped') {
 193                 # do nothing
 194             } else {
 195                 die "should not be reached";
 196             }
 197         }
 198     }
 199
 200     $self->{online_node_usage} = $online_node_usage;
 201 }
 202
 203 my $change_service_state = sub {
 204     my ($self, $sid, $new_state, %params) = @_;
 205
 206     my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
 207
 208     my $sd = $ss->{$sid} || die "no such service '$sid";
 209
 210     my $old_state = $sd->{state};
 211     my $old_node = $sd->{node};
 212     my $old_failed_nodes = $sd->{failed_nodes};
 213
 214     die "no state change" if $old_state eq $new_state; # just to be sure
 215
 216     die "invalid CRM service state '$new_state'\n" if !$valid_service_states->{$new_state};
 217
 218     foreach my $k (keys %$sd) { delete $sd->{$k}; };
 219
 220     $sd->{state} = $new_state;
 221     $sd->{node} = $old_node;
 222     $sd->{failed_nodes} = $old_failed_nodes;
 223
 224     my $text_state = '';
 225     foreach my $k (sort keys %params) {
 226         my $v = $params{$k};
 227         $text_state .= ", " if $text_state;
 228         $text_state .= "$k = $v";
 229         $sd->{$k} = $v;
 230     }
 231
 232     $self->recompute_online_node_usage();
 233
 234     $sd->{uid} = compute_new_uuid($new_state);
 235
 236     $text_state = "  ($text_state)" if $text_state;
 237     $haenv->log('info', "service '$sid': state changed from '${old_state}'" .
 238                 " to '${new_state}'$text_state");
 239 };
 240
 241 # clean up a possible bad state from a recovered service to allow its start
 242 my $fence_recovery_cleanup = sub {
 243     my ($self, $sid, $fenced_node) = @_;
 244
 245     my $haenv = $self->{haenv};
 246
 247     my (undef, $type, $id) = PVE::HA::Tools::parse_sid($sid);
 248     my $plugin = PVE::HA::Resources->lookup($type);
 249
 250     # should not happen
 251     die "unknown resource type '$type'" if !$plugin;
 252
 253     # locks may block recovery, cleanup those which are safe to remove after fencing
 254     my $removable_locks = ['backup', 'mounted'];
 255     if (my $removed_lock = $plugin->remove_locks($haenv, $id, $removable_locks, $fenced_node)) {
 256         $haenv->log('warning', "removed leftover lock '$removed_lock' from recovered " .
 257                     "service '$sid' to allow its start.");
 258     }
 259 };
 260
 261 # after a node was fenced this recovers the service to a new node
 262 my $recover_fenced_service = sub {
 263     my ($self, $sid, $cd) = @_;
 264
 265     my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
 266
 267     my $sd = $ss->{$sid};
 268
 269     if ($sd->{state} ne 'fence') { # should not happen
 270         $haenv->log('err', "cannot recover service '$sid' from fencing," .
 271                     " wrong state '$sd->{state}'");
 272         return;
 273     }
 274
 275     my $fenced_node = $sd->{node}; # for logging purpose
 276
 277     $self->recompute_online_node_usage(); # we want the most current node state
 278
 279     my $recovery_node = select_service_node($self->{groups},
 280                                             $self->{online_node_usage},
 281                                             $cd, $sd->{node});
 282
 283     if ($recovery_node) {
 284         $haenv->log('info', "recover service '$sid' from fenced node " .
 285                     "'$fenced_node' to node '$recovery_node'");
 286
 287         &$fence_recovery_cleanup($self, $sid, $fenced_node);
 288
 289         $haenv->steal_service($sid, $sd->{node}, $recovery_node);
 290
 291         # $sd *is normally read-only*, fencing is the exception
 292         $cd->{node} = $sd->{node} = $recovery_node;
 293         &$change_service_state($self, $sid, 'started', node => $recovery_node);
 294     } else {
 295         # no node found, let the service in 'fence' state and try again
 296         $haenv->log('err', "recovering service '$sid' from fenced node " .
 297                     "'$fenced_node' failed, no recovery node found");
 298     }
 299 };
 300
 301 # read LRM status for all nodes
 302 sub read_lrm_status {
 303     my ($self) = @_;
 304
 305     my $nodes = $self->{ns}->list_nodes();
 306     my $haenv = $self->{haenv};
 307
 308     my $results = {};
 309     my $modes = {};
 310     foreach my $node (@$nodes) {
 311         my $lrm_status = $haenv->read_lrm_status($node);
 312         $modes->{$node} = $lrm_status->{mode} || 'active';
 313         foreach my $uid (keys %{$lrm_status->{results}}) {
 314             next if $results->{$uid}; # should not happen
 315             $results->{$uid} = $lrm_status->{results}->{$uid};
 316         }
 317     }
 318
 319
 320     return ($results, $modes);
 321 }
 322
 323 # read new crm commands and save them into crm master status
 324 sub update_crm_commands {
 325     my ($self) = @_;
 326
 327     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 328
 329     my $cmdlist = $haenv->read_crm_commands();
 330
 331     foreach my $cmd (split(/\n/, $cmdlist)) {
 332         chomp $cmd;
 333
 334         if ($cmd =~ m/^(migrate|relocate)\s+(\S+)\s+(\S+)$/) {
 335             my ($task, $sid, $node) = ($1, $2, $3);
 336             if (my $sd = $ss->{$sid}) {
 337                 if (!$ns->node_is_online($node)) {
 338                     $haenv->log('err', "crm command error - node not online: $cmd");
 339                 } else {
 340                     if ($node eq $sd->{node}) {
 341                         $haenv->log('info', "ignore crm command - service already on target node: $cmd");
 342                     } else {
 343                         $haenv->log('info', "got crm command: $cmd");
 344                         $ss->{$sid}->{cmd} = [ $task, $node];
 345                     }
 346                 }
 347             } else {
 348                 $haenv->log('err', "crm command error - no such service: $cmd");
 349             }
 350
 351         } else {
 352             $haenv->log('err', "unable to parse crm command: $cmd");
 353         }
 354     }
 355
 356 }
 357
 358 sub manage {
 359     my ($self) = @_;
 360
 361     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 362
 363     $ns->update($haenv->get_node_info());
 364
 365     if (!$ns->node_is_online($haenv->nodename())) {
 366         $haenv->log('info', "master seems offline");
 367         return;
 368     }
 369
 370     my ($lrm_results, $lrm_modes) = $self->read_lrm_status();
 371
 372     my $sc = $haenv->read_service_config();
 373
 374     $self->{groups} = $haenv->read_group_config(); # update
 375
 376     # compute new service status
 377
 378     # add new service
 379     foreach my $sid (sort keys %$sc) {
 380         next if $ss->{$sid}; # already there
 381         $haenv->log('info', "adding new service '$sid' on node '$sc->{$sid}->{node}'");
 382         # assume we are running to avoid relocate running service at add
 383         $ss->{$sid} = { state => 'started', node => $sc->{$sid}->{node},
 384                         uid => compute_new_uuid('started') };
 385     }
 386
 387     # remove stale service from manager state
 388     foreach my $sid (keys %$ss) {
 389         next if $sc->{$sid};
 390         $haenv->log('info', "removing stale service '$sid' (no config)");
 391         # remove all service related state information
 392         delete $ss->{$sid};
 393     }
 394
 395     $self->update_crm_commands();
 396
 397     for (;;) {
 398         my $repeat = 0;
 399
 400         $self->recompute_online_node_usage();
 401
 402         foreach my $sid (sort keys %$ss) {
 403             my $sd = $ss->{$sid};
 404             my $cd = $sc->{$sid} || { state => 'disabled' };
 405
 406             my $lrm_res = $sd->{uid} ? $lrm_results->{$sd->{uid}} : undef;
 407
 408             my $last_state = $sd->{state};
 409
 410             if ($last_state eq 'stopped') {
 411
 412                 $self->next_state_stopped($sid, $cd, $sd, $lrm_res);
 413
 414             } elsif ($last_state eq 'started') {
 415
 416                 $self->next_state_started($sid, $cd, $sd, $lrm_res);
 417
 418             } elsif ($last_state eq 'migrate' || $last_state eq 'relocate') {
 419
 420                 $self->next_state_migrate_relocate($sid, $cd, $sd, $lrm_res);
 421
 422             } elsif ($last_state eq 'fence') {
 423
 424                 # do nothing here - wait until fenced
 425
 426             } elsif ($last_state eq 'request_stop') {
 427
 428                 $self->next_state_request_stop($sid, $cd, $sd, $lrm_res);
 429
 430             } elsif ($last_state eq 'freeze') {
 431
 432                 my $lrm_mode = $sd->{node} ? $lrm_modes->{$sd->{node}} : undef;
 433                 # unfreeze
 434                 &$change_service_state($self, $sid, 'started')
 435                     if $lrm_mode && $lrm_mode eq 'active';
 436
 437             } elsif ($last_state eq 'error') {
 438
 439                 $self->next_state_error($sid, $cd, $sd, $lrm_res);
 440
 441             } else {
 442
 443                 die "unknown service state '$last_state'";
 444             }
 445
 446             my $lrm_mode = $sd->{node} ? $lrm_modes->{$sd->{node}} : undef;
 447             if ($lrm_mode && $lrm_mode eq 'restart') {
 448                 if (($sd->{state} eq 'started' || $sd->{state} eq 'stopped' ||
 449                      $sd->{state} eq 'request_stop')) {
 450                     &$change_service_state($self, $sid, 'freeze');
 451                 }
 452             }
 453
 454             $repeat = 1 if $sd->{state} ne $last_state;
 455         }
 456
 457         # handle fencing
 458         my $fenced_nodes = {};
 459         foreach my $sid (sort keys %$ss) {
 460             my $sd = $ss->{$sid};
 461             next if $sd->{state} ne 'fence';
 462
 463             if (!defined($fenced_nodes->{$sd->{node}})) {
 464                 $fenced_nodes->{$sd->{node}} = $ns->fence_node($sd->{node}) || 0;
 465             }
 466
 467             next if !$fenced_nodes->{$sd->{node}};
 468
 469             # node fence was successful - recover service
 470             &$recover_fenced_service($self, $sid, $sc->{$sid});
 471         }
 472
 473         last if !$repeat;
 474     }
 475
 476     $self->flush_master_status();
 477 }
 478
 479 # functions to compute next service states
 480 # $cd: service configuration data (read only)
 481 # $sd: service status data (read only)
 482 #
 483 # Note: use change_service_state() to alter state
 484 #
 485
 486 sub next_state_request_stop {
 487     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 488
 489     my $haenv = $self->{haenv};
 490     my $ns = $self->{ns};
 491
 492     # check result from LRM daemon
 493     if ($lrm_res) {
 494         my $exit_code = $lrm_res->{exit_code};
 495         if ($exit_code == SUCCESS) {
 496             &$change_service_state($self, $sid, 'stopped');
 497             return;
 498         } else {
 499             $haenv->log('err', "service '$sid' stop failed (exit code $exit_code)");
 500             &$change_service_state($self, $sid, 'error'); # fixme: what state?
 501             return;
 502         }
 503     }
 504
 505     if ($ns->node_is_offline_delayed($sd->{node})) {
 506         &$change_service_state($self, $sid, 'fence');
 507         return;
 508     }
 509 }
 510
 511 sub next_state_migrate_relocate {
 512     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 513
 514     my $haenv = $self->{haenv};
 515     my $ns = $self->{ns};
 516
 517     # check result from LRM daemon
 518     if ($lrm_res) {
 519         my $exit_code = $lrm_res->{exit_code};
 520         my $req_state = $cd->{state} eq 'enabled' ? 'started' : 'request_stop';
 521         if ($exit_code == SUCCESS) {
 522             &$change_service_state($self, $sid, $req_state, node => $sd->{target});
 523             return;
 524         } elsif ($exit_code == EWRONG_NODE) {
 525             $haenv->log('err', "service '$sid' - migration failed: service" .
 526                         " registered on wrong node!");
 527             &$change_service_state($self, $sid, 'error');
 528         } else {
 529             $haenv->log('err', "service '$sid' - migration failed (exit code $exit_code)");
 530             &$change_service_state($self, $sid, $req_state, node => $sd->{node});
 531             return;
 532         }
 533     }
 534
 535     if ($ns->node_is_offline_delayed($sd->{node})) {
 536         &$change_service_state($self, $sid, 'fence');
 537         return;
 538     }
 539 }
 540
 541
 542 sub next_state_stopped {
 543     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 544
 545     my $haenv = $self->{haenv};
 546     my $ns = $self->{ns};
 547
 548     if ($sd->{node} ne $cd->{node}) {
 549         # this can happen if we fence a node with active migrations
 550         # hack: modify $sd (normally this should be considered read-only)
 551         $haenv->log('info', "fixup service '$sid' location ($sd->{node} => $cd->{node})");
 552         $sd->{node} = $cd->{node};
 553     }
 554
 555     if ($sd->{cmd}) {
 556         my ($cmd, $target) = @{$sd->{cmd}};
 557         delete $sd->{cmd};
 558
 559         if ($cmd eq 'migrate' || $cmd eq 'relocate') {
 560             if (!$ns->node_is_online($target)) {
 561                 $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 562             } elsif ($sd->{node} eq $target) {
 563                 $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
 564             } else {
 565                 &$change_service_state($self, $sid, $cmd, node => $sd->{node},
 566                                        target => $target);
 567                 return;
 568             }
 569         } else {
 570             $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 571         }
 572     }
 573
 574     if ($cd->{state} eq 'disabled') {
 575         # NOTE: do nothing here, the stop state is an exception as we do not
 576         # process the LRM result here, thus the LRM always tries to stop the
 577         # service (protection for the case no CRM is active)
 578         return;
 579     }
 580
 581     if ($cd->{state} eq 'enabled') {
 582         # simply mark it started, if it's on the wrong node
 583         # next_state_started will fix that for us
 584         &$change_service_state($self, $sid, 'started', node => $sd->{node});
 585         return;
 586     }
 587
 588     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 589 }
 590
 591 sub record_service_failed_on_node {
 592     my ($self, $sid, $node) = @_;
 593
 594     if (!defined($self->{ss}->{$sid}->{failed_nodes})) {
 595         $self->{ss}->{$sid}->{failed_nodes} = [];
 596     }
 597
 598     push @{$self->{ss}->{$sid}->{failed_nodes}}, $node;
 599 }
 600
 601 sub next_state_started {
 602     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 603
 604     my $haenv = $self->{haenv};
 605     my $master_status = $self->{ms};
 606     my $ns = $self->{ns};
 607
 608     if (!$ns->node_is_online($sd->{node})) {
 609         if ($ns->node_is_offline_delayed($sd->{node})) {
 610             &$change_service_state($self, $sid, 'fence');
 611         }
 612         return;
 613     }
 614
 615     if ($cd->{state} eq 'disabled') {
 616         &$change_service_state($self, $sid, 'request_stop');
 617         return;
 618     }
 619
 620     if ($cd->{state} eq 'enabled') {
 621
 622         if ($sd->{cmd}) {
 623             my ($cmd, $target) = @{$sd->{cmd}};
 624             delete $sd->{cmd};
 625
 626             if ($cmd eq 'migrate' || $cmd eq 'relocate') {
 627                 if (!$ns->node_is_online($target)) {
 628                     $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 629                 } elsif ($sd->{node} eq $target) {
 630                     $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
 631                 } else {
 632                     $haenv->log('info', "$cmd service '$sid' to node '$target'");
 633                     &$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
 634                 }
 635             } else {
 636                 $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 637             }
 638         } else {
 639
 640             my $try_next = 0;
 641
 642             if ($lrm_res) {
 643
 644                 my $ec = $lrm_res->{exit_code};
 645                 if ($ec == SUCCESS) {
 646
 647                     if (defined($sd->{failed_nodes})) {
 648                         $haenv->log('info', "relocation policy successful for '$sid' on node '$sd->{node}'," .
 649                                     " failed nodes: " . join(', ', @{$sd->{failed_nodes}}) );
 650                     }
 651
 652                     delete $sd->{failed_nodes};
 653
 654                 } elsif ($ec == ERROR) {
 655                     # apply our relocate policy if we got ERROR from the LRM
 656                     $self->record_service_failed_on_node($sid, $sd->{node});
 657
 658                     if (scalar(@{$sd->{failed_nodes}}) <= $cd->{max_relocate}) {
 659
 660                         # tell select_service_node to relocate if possible
 661                         $try_next = 1;
 662
 663                         $haenv->log('warning', "starting service $sid on node".
 664                                    " '$sd->{node}' failed, relocating service.");
 665
 666                     } else {
 667
 668                         $haenv->log('err', "recovery policy for service $sid " .
 669                                     "failed, entering error state. Failed nodes: ".
 670                                     join(', ', @{$sd->{failed_nodes}}));
 671                         &$change_service_state($self, $sid, 'error');
 672                         return;
 673
 674                     }
 675                 } else {
 676                     $self->record_service_failed_on_node($sid, $sd->{node});
 677
 678                     $haenv->log('err', "service '$sid' got unrecoverable error" .
 679                                 " (exit code $ec))");
 680                     # we have no save way out (yet) for other errors
 681                     &$change_service_state($self, $sid, 'error');
 682                     return;
 683                 }
 684             }
 685
 686             my $node = select_service_node($self->{groups}, $self->{online_node_usage},
 687                                            $cd, $sd->{node}, $try_next, $sd->{failed_nodes});
 688
 689             if ($node && ($sd->{node} ne $node)) {
 690                 if ($cd->{type} eq 'vm') {
 691                     $haenv->log('info', "migrate service '$sid' to node '$node' (running)");
 692                     &$change_service_state($self, $sid, 'migrate', node => $sd->{node}, target => $node);
 693                 } else {
 694                     $haenv->log('info', "relocate service '$sid' to node '$node'");
 695                     &$change_service_state($self, $sid, 'relocate', node => $sd->{node}, target => $node);
 696                 }
 697             } else {
 698                 if ($try_next && !defined($node)) {
 699                     $haenv->log('warning', "Start Error Recovery: Tried all available " .
 700                                 " nodes for service '$sid', retry start on current node. " .
 701                                 "Tried nodes: " . join(', ', @{$sd->{failed_nodes}}));
 702                 }
 703                 # ensure service get started again if it went unexpected down
 704                 # but ensure also no LRM result gets lost
 705                 $sd->{uid} = compute_new_uuid($sd->{state}) if defined($lrm_res);
 706             }
 707         }
 708
 709         return;
 710     }
 711
 712     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 713 }
 714
 715 sub next_state_error {
 716     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 717
 718     my $ns = $self->{ns};
 719     my $ms = $self->{ms};
 720
 721     if ($cd->{state} eq 'disabled') {
 722         # clean up on error recovery
 723         delete $sd->{failed_nodes};
 724
 725         &$change_service_state($self, $sid, 'stopped');
 726         return;
 727     }
 728
 729 }
 730
 731 1;