src/PVE/HA/Manager.pm

   1 package PVE::HA::Manager;
   2
   3 use strict;
   4 use warnings;
   5 use Digest::MD5 qw(md5_base64);
   6
   7 use PVE::Tools;
   8 use PVE::HA::Tools ':exit_codes';
   9 use PVE::HA::NodeStatus;
  10
  11 sub new {
  12     my ($this, $haenv) = @_;
  13
  14     my $class = ref($this) || $this;
  15
  16     my $self = bless { haenv => $haenv }, $class;
  17
  18     my $old_ms = $haenv->read_manager_status();
  19
  20     # we only copy the state part of the manager which cannot be auto generated
  21
  22     $self->{ns} = PVE::HA::NodeStatus->new($haenv, $old_ms->{node_status} || {});
  23
  24     # fixme: use separate class  PVE::HA::ServiceStatus
  25     $self->{ss} = $old_ms->{service_status} || {};
  26
  27     $self->{ms} = { master_node => $haenv->nodename() };
  28
  29     return $self;
  30 }
  31
  32 sub cleanup {
  33     my ($self) = @_;
  34
  35     # todo: ?
  36 }
  37
  38 sub flush_master_status {
  39     my ($self) = @_;
  40
  41     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
  42
  43     $ms->{node_status} = $ns->{status};
  44     $ms->{service_status} = $ss;
  45     $ms->{timestamp} = $haenv->get_time();
  46
  47     $haenv->write_manager_status($ms);
  48 }
  49
  50 sub get_service_group {
  51     my ($groups, $online_node_usage, $service_conf) = @_;
  52
  53     my $group = {};
  54     # add all online nodes to default group to allow try_next when no group set
  55     foreach my $node (keys %$online_node_usage) {
  56         $group->{nodes}->{$node} = 1;
  57     }
  58
  59     # overwrite default if service is bound to a specific group
  60     $group =  $groups->{ids}->{$service_conf->{group}} if $service_conf->{group} &&
  61         $groups->{ids}->{$service_conf->{group}};
  62
  63     return $group;
  64 }
  65
  66 # groups available nodes with their priority as group index
  67 sub get_node_priority_groups {
  68     my ($group, $online_node_usage) = @_;
  69
  70     my $pri_groups = {};
  71     my $group_members = {};
  72     foreach my $entry (keys %{$group->{nodes}}) {
  73         my ($node, $pri) = ($entry, 0);
  74         if ($entry =~ m/^(\S+):(\d+)$/) {
  75             ($node, $pri) = ($1, $2);
  76         }
  77         next if !defined($online_node_usage->{$node}); # offline
  78         $pri_groups->{$pri}->{$node} = 1;
  79         $group_members->{$node} = $pri;
  80     }
  81
  82     # add non-group members to unrestricted groups (priority -1)
  83     if (!$group->{restricted}) {
  84         my $pri = -1;
  85         foreach my $node (keys %$online_node_usage) {
  86             next if defined($group_members->{$node});
  87             $pri_groups->{$pri}->{$node} = 1;
  88             $group_members->{$node} = -1;
  89         }
  90     }
  91
  92     return ($pri_groups, $group_members);
  93 }
  94
  95 sub select_service_node {
  96     my ($groups, $online_node_usage, $service_conf, $current_node, $try_next, $tried_nodes) = @_;
  97
  98     my $group = get_service_group($groups, $online_node_usage, $service_conf);
  99
 100     my ($pri_groups, $group_members) = get_node_priority_groups($group, $online_node_usage);
 101
 102     my @pri_list = sort {$b <=> $a} keys %$pri_groups;
 103     return undef if !scalar(@pri_list);
 104
 105     # stay on current node if possible (avoids random migrations)
 106     if (!$try_next && $group->{nofailback} && defined($group_members->{$current_node})) {
 107         return $current_node;
 108     }
 109
 110     # select node from top priority node list
 111
 112     my $top_pri = $pri_list[0];
 113
 114     # try to avoid nodes where the service failed already if we want to relocate
 115     if ($try_next) {
 116         foreach my $node (@$tried_nodes) {
 117             delete $pri_groups->{$top_pri}->{$node};
 118         }
 119     }
 120
 121     my @nodes = sort {
 122         $online_node_usage->{$a} <=> $online_node_usage->{$b} || $a cmp $b
 123     } keys %{$pri_groups->{$top_pri}};
 124
 125     my $found;
 126     for (my $i = scalar(@nodes) - 1; $i >= 0; $i--) {
 127         my $node = $nodes[$i];
 128         if ($node eq $current_node) {
 129             $found = $i;
 130             last;
 131         }
 132     }
 133
 134     if ($try_next) {
 135         if (defined($found) && ($found < (scalar(@nodes) - 1))) {
 136             return $nodes[$found + 1];
 137         } else {
 138             return $nodes[0];
 139         }
 140     } elsif (defined($found)) {
 141         return $nodes[$found];
 142     } else {
 143         return $nodes[0];
 144     }
 145 }
 146
 147 my $uid_counter = 0;
 148
 149 sub compute_new_uuid {
 150     my ($state) = @_;
 151
 152     $uid_counter++;
 153     return md5_base64($state . $$ . time() . $uid_counter);
 154 }
 155
 156 my $valid_service_states = {
 157     stopped => 1,
 158     request_stop => 1,
 159     started => 1,
 160     fence => 1,
 161     migrate => 1,
 162     relocate => 1,
 163     freeze => 1,
 164     error => 1,
 165 };
 166
 167 sub recompute_online_node_usage {
 168     my ($self) = @_;
 169
 170     my $online_node_usage = {};
 171
 172     my $online_nodes = $self->{ns}->list_online_nodes();
 173
 174     foreach my $node (@$online_nodes) {
 175         $online_node_usage->{$node} = 0;
 176     }
 177
 178     foreach my $sid (keys %{$self->{ss}}) {
 179         my $sd = $self->{ss}->{$sid};
 180         my $state = $sd->{state};
 181         if (defined($online_node_usage->{$sd->{node}})) {
 182             if (($state eq 'started') || ($state eq 'request_stop') ||
 183                 ($state eq 'fence') || ($state eq 'freeze') || ($state eq 'error')) {
 184                 $online_node_usage->{$sd->{node}}++;
 185             } elsif (($state eq 'migrate') || ($state eq 'relocate')) {
 186                 # count it for both, source and target as load is put on both
 187                 $online_node_usage->{$sd->{node}}++;
 188                 $online_node_usage->{$sd->{target}}++;
 189             } elsif ($state eq 'stopped') {
 190                 # do nothing
 191             } else {
 192                 die "should not be reached";
 193             }
 194         }
 195     }
 196
 197     $self->{online_node_usage} = $online_node_usage;
 198 }
 199
 200 my $change_service_state = sub {
 201     my ($self, $sid, $new_state, %params) = @_;
 202
 203     my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
 204
 205     my $sd = $ss->{$sid} || die "no such service '$sid";
 206
 207     my $old_state = $sd->{state};
 208     my $old_node = $sd->{node};
 209     my $old_failed_nodes = $sd->{failed_nodes};
 210
 211     die "no state change" if $old_state eq $new_state; # just to be sure
 212
 213     die "invalid CRM service state '$new_state'\n" if !$valid_service_states->{$new_state};
 214
 215     foreach my $k (keys %$sd) { delete $sd->{$k}; };
 216
 217     $sd->{state} = $new_state;
 218     $sd->{node} = $old_node;
 219     $sd->{failed_nodes} = $old_failed_nodes if defined($old_failed_nodes);
 220
 221     my $text_state = '';
 222     foreach my $k (sort keys %params) {
 223         my $v = $params{$k};
 224         $text_state .= ", " if $text_state;
 225         $text_state .= "$k = $v";
 226         $sd->{$k} = $v;
 227     }
 228
 229     $self->recompute_online_node_usage();
 230
 231     $sd->{uid} = compute_new_uuid($new_state);
 232
 233     $text_state = "  ($text_state)" if $text_state;
 234     $haenv->log('info', "service '$sid': state changed from '${old_state}'" .
 235                 " to '${new_state}'$text_state");
 236 };
 237
 238 # clean up a possible bad state from a recovered service to allow its start
 239 my $fence_recovery_cleanup = sub {
 240     my ($self, $sid, $fenced_node) = @_;
 241
 242     my $haenv = $self->{haenv};
 243
 244     my (undef, $type, $id) = $haenv->parse_sid($sid);
 245     my $plugin = PVE::HA::Resources->lookup($type);
 246
 247     # should not happen
 248     die "unknown resource type '$type'" if !$plugin;
 249
 250     # locks may block recovery, cleanup those which are safe to remove after fencing,
 251     # i.e., after the original node was reset and thus all it's state
 252     my $removable_locks = ['backup', 'mounted', 'migrate', 'clone', 'rollback', 'snapshot', 'snapshot-delete', 'suspending', 'suspended'];
 253     if (my $removed_lock = $plugin->remove_locks($haenv, $id, $removable_locks, $fenced_node)) {
 254         $haenv->log('warning', "removed leftover lock '$removed_lock' from recovered " .
 255                     "service '$sid' to allow its start.");
 256     }
 257 };
 258
 259 # after a node was fenced this recovers the service to a new node
 260 my $recover_fenced_service = sub {
 261     my ($self, $sid, $cd) = @_;
 262
 263     my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
 264
 265     my $sd = $ss->{$sid};
 266
 267     if ($sd->{state} ne 'fence') { # should not happen
 268         $haenv->log('err', "cannot recover service '$sid' from fencing," .
 269                     " wrong state '$sd->{state}'");
 270         return;
 271     }
 272
 273     my $fenced_node = $sd->{node}; # for logging purpose
 274
 275     $self->recompute_online_node_usage(); # we want the most current node state
 276
 277     my $recovery_node = select_service_node($self->{groups},
 278                                             $self->{online_node_usage},
 279                                             $cd, $sd->{node});
 280
 281     if ($recovery_node) {
 282         $haenv->log('info', "recover service '$sid' from fenced node " .
 283                     "'$fenced_node' to node '$recovery_node'");
 284
 285         &$fence_recovery_cleanup($self, $sid, $fenced_node);
 286
 287         $haenv->steal_service($sid, $sd->{node}, $recovery_node);
 288         $self->{online_node_usage}->{$recovery_node}++;
 289
 290         # $sd *is normally read-only*, fencing is the exception
 291         $cd->{node} = $sd->{node} = $recovery_node;
 292         my $new_state = ($cd->{state} eq 'started') ? 'started' : 'request_stop';
 293         &$change_service_state($self, $sid, $new_state, node => $recovery_node);
 294     } else {
 295         # no possible node found, cannot recover
 296         $haenv->log('err', "recovering service '$sid' from fenced node " .
 297                     "'$fenced_node' failed, no recovery node found");
 298         &$change_service_state($self, $sid, 'error');
 299     }
 300 };
 301
 302 # read LRM status for all nodes
 303 sub read_lrm_status {
 304     my ($self) = @_;
 305
 306     my $nodes = $self->{ns}->list_nodes();
 307     my $haenv = $self->{haenv};
 308
 309     my $results = {};
 310     my $modes = {};
 311     foreach my $node (@$nodes) {
 312         my $lrm_status = $haenv->read_lrm_status($node);
 313         $modes->{$node} = $lrm_status->{mode} || 'active';
 314         foreach my $uid (keys %{$lrm_status->{results}}) {
 315             next if $results->{$uid}; # should not happen
 316             $results->{$uid} = $lrm_status->{results}->{$uid};
 317         }
 318     }
 319
 320
 321     return ($results, $modes);
 322 }
 323
 324 # read new crm commands and save them into crm master status
 325 sub update_crm_commands {
 326     my ($self) = @_;
 327
 328     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 329
 330     my $cmdlist = $haenv->read_crm_commands();
 331
 332     foreach my $cmd (split(/\n/, $cmdlist)) {
 333         chomp $cmd;
 334
 335         if ($cmd =~ m/^(migrate|relocate)\s+(\S+)\s+(\S+)$/) {
 336             my ($task, $sid, $node) = ($1, $2, $3);
 337             if (my $sd = $ss->{$sid}) {
 338                 if (!$ns->node_is_online($node)) {
 339                     $haenv->log('err', "crm command error - node not online: $cmd");
 340                 } else {
 341                     if ($node eq $sd->{node}) {
 342                         $haenv->log('info', "ignore crm command - service already on target node: $cmd");
 343                     } else {
 344                         $haenv->log('info', "got crm command: $cmd");
 345                         $ss->{$sid}->{cmd} = [ $task, $node ];
 346                     }
 347                 }
 348             } else {
 349                 $haenv->log('err', "crm command error - no such service: $cmd");
 350             }
 351
 352         } elsif ($cmd =~ m/^stop\s+(\S+)\s+(\S+)$/) {
 353             my ($sid, $timeout) = ($1, $2);
 354             if (my $sd = $ss->{$sid}) {
 355                 $haenv->log('info', "got crm command: $cmd");
 356                 $ss->{$sid}->{cmd} = [ 'stop', $timeout ];
 357             } else {
 358                 $haenv->log('err', "crm command error - no such service: $cmd");
 359             }
 360         } else {
 361             $haenv->log('err', "unable to parse crm command: $cmd");
 362         }
 363     }
 364
 365 }
 366
 367 sub manage {
 368     my ($self) = @_;
 369
 370     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 371
 372     my ($node_info) = $haenv->get_node_info();
 373     my ($lrm_results, $lrm_modes) = $self->read_lrm_status();
 374
 375     $ns->update($node_info, $lrm_modes);
 376
 377     if (!$ns->node_is_operational($haenv->nodename())) {
 378         $haenv->log('info', "master seems offline");
 379         return;
 380     }
 381
 382     my $sc = $haenv->read_service_config();
 383
 384     $self->{groups} = $haenv->read_group_config(); # update
 385
 386     # compute new service status
 387
 388     # add new service
 389     foreach my $sid (sort keys %$sc) {
 390         next if $ss->{$sid}; # already there
 391         my $cd = $sc->{$sid};
 392         next if $cd->{state} eq 'ignored';
 393
 394         $haenv->log('info', "adding new service '$sid' on node '$cd->{node}'");
 395         # assume we are running to avoid relocate running service at add
 396         my $state = ($cd->{state} eq 'started') ? 'started' : 'request_stop';
 397         $ss->{$sid} = { state => $state, node => $cd->{node},
 398                         uid => compute_new_uuid('started') };
 399     }
 400
 401     # remove stale or ignored services from manager state
 402     foreach my $sid (keys %$ss) {
 403         next if $sc->{$sid} && $sc->{$sid}->{state} ne 'ignored';
 404
 405         my $reason =  defined($sc->{$sid}) ? 'ignored state requested' : 'no config';
 406         $haenv->log('info', "removing stale service '$sid' ($reason)");
 407
 408         # remove all service related state information
 409         delete $ss->{$sid};
 410     }
 411
 412     $self->update_crm_commands();
 413
 414     for (;;) {
 415         my $repeat = 0;
 416
 417         $self->recompute_online_node_usage();
 418
 419         foreach my $sid (sort keys %$ss) {
 420             my $sd = $ss->{$sid};
 421             my $cd = $sc->{$sid} || { state => 'disabled' };
 422
 423             my $lrm_res = $sd->{uid} ? $lrm_results->{$sd->{uid}} : undef;
 424
 425             my $last_state = $sd->{state};
 426
 427             if ($last_state eq 'stopped') {
 428
 429                 $self->next_state_stopped($sid, $cd, $sd, $lrm_res);
 430
 431             } elsif ($last_state eq 'started') {
 432
 433                 $self->next_state_started($sid, $cd, $sd, $lrm_res);
 434
 435             } elsif ($last_state eq 'migrate' || $last_state eq 'relocate') {
 436
 437                 $self->next_state_migrate_relocate($sid, $cd, $sd, $lrm_res);
 438
 439             } elsif ($last_state eq 'fence') {
 440
 441                 # do nothing here - wait until fenced
 442
 443             } elsif ($last_state eq 'request_stop') {
 444
 445                 $self->next_state_request_stop($sid, $cd, $sd, $lrm_res);
 446
 447             } elsif ($last_state eq 'freeze') {
 448
 449                 my $lrm_mode = $sd->{node} ? $lrm_modes->{$sd->{node}} : undef;
 450                 # unfreeze
 451                 my $state = ($cd->{state} eq 'started') ? 'started' : 'request_stop';
 452                 &$change_service_state($self, $sid, $state)
 453                     if $lrm_mode && $lrm_mode eq 'active';
 454
 455             } elsif ($last_state eq 'error') {
 456
 457                 $self->next_state_error($sid, $cd, $sd, $lrm_res);
 458
 459             } else {
 460
 461                 die "unknown service state '$last_state'";
 462             }
 463
 464             my $lrm_mode = $sd->{node} ? $lrm_modes->{$sd->{node}} : undef;
 465             if ($lrm_mode && $lrm_mode eq 'restart') {
 466                 if (($sd->{state} eq 'started' || $sd->{state} eq 'stopped' ||
 467                      $sd->{state} eq 'request_stop')) {
 468                     &$change_service_state($self, $sid, 'freeze');
 469                 }
 470             }
 471
 472             $repeat = 1 if $sd->{state} ne $last_state;
 473         }
 474
 475         # handle fencing
 476         my $fenced_nodes = {};
 477         foreach my $sid (sort keys %$ss) {
 478             my $sd = $ss->{$sid};
 479             next if $sd->{state} ne 'fence';
 480
 481             if (!defined($fenced_nodes->{$sd->{node}})) {
 482                 $fenced_nodes->{$sd->{node}} = $ns->fence_node($sd->{node}) || 0;
 483             }
 484
 485             next if !$fenced_nodes->{$sd->{node}};
 486
 487             # node fence was successful - recover service
 488             &$recover_fenced_service($self, $sid, $sc->{$sid});
 489         }
 490
 491         last if !$repeat;
 492     }
 493
 494     $self->flush_master_status();
 495 }
 496
 497 # functions to compute next service states
 498 # $cd: service configuration data (read only)
 499 # $sd: service status data (read only)
 500 #
 501 # Note: use change_service_state() to alter state
 502 #
 503
 504 sub next_state_request_stop {
 505     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 506
 507     my $haenv = $self->{haenv};
 508     my $ns = $self->{ns};
 509
 510     # check result from LRM daemon
 511     if ($lrm_res) {
 512         my $exit_code = $lrm_res->{exit_code};
 513         if ($exit_code == SUCCESS) {
 514             &$change_service_state($self, $sid, 'stopped');
 515             return;
 516         } else {
 517             $haenv->log('err', "service '$sid' stop failed (exit code $exit_code)");
 518             &$change_service_state($self, $sid, 'error'); # fixme: what state?
 519             return;
 520         }
 521     }
 522
 523     if ($ns->node_is_offline_delayed($sd->{node})) {
 524         &$change_service_state($self, $sid, 'fence');
 525         return;
 526     }
 527 }
 528
 529 sub next_state_migrate_relocate {
 530     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 531
 532     my $haenv = $self->{haenv};
 533     my $ns = $self->{ns};
 534
 535     # check result from LRM daemon
 536     if ($lrm_res) {
 537         my $exit_code = $lrm_res->{exit_code};
 538         my $req_state = $cd->{state} eq 'started' ? 'started' : 'request_stop';
 539         if ($exit_code == SUCCESS) {
 540             &$change_service_state($self, $sid, $req_state, node => $sd->{target});
 541             return;
 542         } elsif ($exit_code == EWRONG_NODE) {
 543             $haenv->log('err', "service '$sid' - migration failed: service" .
 544                         " registered on wrong node!");
 545             &$change_service_state($self, $sid, 'error');
 546         } else {
 547             $haenv->log('err', "service '$sid' - migration failed (exit code $exit_code)");
 548             &$change_service_state($self, $sid, $req_state, node => $sd->{node});
 549             return;
 550         }
 551     }
 552
 553     if ($ns->node_is_offline_delayed($sd->{node})) {
 554         &$change_service_state($self, $sid, 'fence');
 555         return;
 556     }
 557 }
 558
 559 sub next_state_stopped {
 560     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 561
 562     my $haenv = $self->{haenv};
 563     my $ns = $self->{ns};
 564
 565     if ($sd->{node} ne $cd->{node}) {
 566         # this can happen if we fence a node with active migrations
 567         # hack: modify $sd (normally this should be considered read-only)
 568         $haenv->log('info', "fixup service '$sid' location ($sd->{node} => $cd->{node})");
 569         $sd->{node} = $cd->{node};
 570     }
 571
 572     if ($sd->{cmd}) {
 573         my $cmd = shift @{$sd->{cmd}};
 574
 575         if ($cmd eq 'migrate' || $cmd eq 'relocate') {
 576             my $target = shift @{$sd->{cmd}};
 577             if (!$ns->node_is_online($target)) {
 578                 $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 579             } elsif ($sd->{node} eq $target) {
 580                 $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
 581             } else {
 582                 &$change_service_state($self, $sid, $cmd, node => $sd->{node},
 583                                        target => $target);
 584                 return;
 585             }
 586         } elsif ($cmd eq 'stop') {
 587                 $haenv->log('info', "ignore service '$sid' $cmd request - service already stopped");
 588         } else {
 589             $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 590         }
 591         delete $sd->{cmd};
 592     }
 593
 594     if ($cd->{state} eq 'disabled') {
 595         # NOTE: do nothing here, the stop state is an exception as we do not
 596         # process the LRM result here, thus the LRM always tries to stop the
 597         # service (protection for the case no CRM is active)
 598         return;
 599     }
 600
 601     if ($ns->node_is_offline_delayed($sd->{node})) {
 602         &$change_service_state($self, $sid, 'fence');
 603         return;
 604     }
 605
 606     if ($cd->{state} eq 'stopped') {
 607         # almost the same as 'disabled' state but the service will also get recovered
 608         return;
 609     }
 610
 611     if ($cd->{state} eq 'started') {
 612         # simply mark it started, if it's on the wrong node
 613         # next_state_started will fix that for us
 614         &$change_service_state($self, $sid, 'started', node => $sd->{node});
 615         return;
 616     }
 617
 618     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 619 }
 620
 621 sub record_service_failed_on_node {
 622     my ($self, $sid, $node) = @_;
 623
 624     if (!defined($self->{ss}->{$sid}->{failed_nodes})) {
 625         $self->{ss}->{$sid}->{failed_nodes} = [];
 626     }
 627
 628     push @{$self->{ss}->{$sid}->{failed_nodes}}, $node;
 629 }
 630
 631 sub next_state_started {
 632     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 633
 634     my $haenv = $self->{haenv};
 635     my $master_status = $self->{ms};
 636     my $ns = $self->{ns};
 637
 638     if (!$ns->node_is_online($sd->{node})) {
 639         if ($ns->node_is_offline_delayed($sd->{node})) {
 640             &$change_service_state($self, $sid, 'fence');
 641         }
 642         if ($ns->get_node_state($sd->{node}) ne 'maintenance') {
 643             return;
 644         }
 645     }
 646
 647     if ($cd->{state} eq 'disabled' || $cd->{state} eq 'stopped') {
 648         &$change_service_state($self, $sid, 'request_stop');
 649         return;
 650     }
 651
 652     if ($cd->{state} eq 'started') {
 653
 654         if ($sd->{cmd}) {
 655             my $cmd = shift @{$sd->{cmd}};
 656
 657             if ($cmd eq 'migrate' || $cmd eq 'relocate') {
 658                 my $target = shift @{$sd->{cmd}};
 659                 if (!$ns->node_is_online($target)) {
 660                     $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 661                 } elsif ($sd->{node} eq $target) {
 662                     $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
 663                 } else {
 664                     $haenv->log('info', "$cmd service '$sid' to node '$target'");
 665                     &$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
 666                 }
 667             } elsif ($cmd eq 'stop') {
 668                 my $timeout = shift @{$sd->{cmd}};
 669                 if ($timeout == 0) {
 670                     $haenv->log('info', "request immediate service hard-stop for service '$sid'");
 671                 } else {
 672                     $haenv->log('info', "request graceful stop with timeout '$timeout' for service '$sid'");
 673                 }
 674                 &$change_service_state($self, $sid, 'request_stop', timeout => $timeout);
 675                 $haenv->update_service_config($sid, {'state' => 'stopped'});
 676             } else {
 677                 $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 678             }
 679
 680             delete $sd->{cmd};
 681
 682         } else {
 683
 684             my $try_next = 0;
 685
 686             if ($lrm_res) {
 687
 688                 my $ec = $lrm_res->{exit_code};
 689                 if ($ec == SUCCESS) {
 690
 691                     if (defined($sd->{failed_nodes})) {
 692                         $haenv->log('info', "relocation policy successful for '$sid' on node '$sd->{node}'," .
 693                                     " failed nodes: " . join(', ', @{$sd->{failed_nodes}}) );
 694                     }
 695
 696                     delete $sd->{failed_nodes};
 697
 698                     # store flag to indicate successful start - only valid while state == 'started'
 699                     $sd->{running} = 1;
 700
 701                 } elsif ($ec == ERROR) {
 702
 703                     delete $sd->{running};
 704
 705                     # apply our relocate policy if we got ERROR from the LRM
 706                     $self->record_service_failed_on_node($sid, $sd->{node});
 707
 708                     if (scalar(@{$sd->{failed_nodes}}) <= $cd->{max_relocate}) {
 709
 710                         # tell select_service_node to relocate if possible
 711                         $try_next = 1;
 712
 713                         $haenv->log('warning', "starting service $sid on node".
 714                                    " '$sd->{node}' failed, relocating service.");
 715
 716                     } else {
 717
 718                         $haenv->log('err', "recovery policy for service $sid " .
 719                                     "failed, entering error state. Failed nodes: ".
 720                                     join(', ', @{$sd->{failed_nodes}}));
 721                         &$change_service_state($self, $sid, 'error');
 722                         return;
 723
 724                     }
 725                 } else {
 726                     $self->record_service_failed_on_node($sid, $sd->{node});
 727
 728                     $haenv->log('err', "service '$sid' got unrecoverable error" .
 729                                 " (exit code $ec))");
 730                     # we have no save way out (yet) for other errors
 731                     &$change_service_state($self, $sid, 'error');
 732                     return;
 733                 }
 734             }
 735
 736             my $node = select_service_node($self->{groups}, $self->{online_node_usage},
 737                                            $cd, $sd->{node}, $try_next, $sd->{failed_nodes});
 738
 739             if ($node && ($sd->{node} ne $node)) {
 740                 $self->{online_node_usage}->{$node}++;
 741                 if ($cd->{type} eq 'vm') {
 742                     $haenv->log('info', "migrate service '$sid' to node '$node' (running)");
 743                     &$change_service_state($self, $sid, 'migrate', node => $sd->{node}, target => $node);
 744                 } else {
 745                     $haenv->log('info', "relocate service '$sid' to node '$node'");
 746                     &$change_service_state($self, $sid, 'relocate', node => $sd->{node}, target => $node);
 747                 }
 748             } else {
 749                 if ($try_next && !defined($node)) {
 750                     $haenv->log('warning', "Start Error Recovery: Tried all available " .
 751                                 " nodes for service '$sid', retry start on current node. " .
 752                                 "Tried nodes: " . join(', ', @{$sd->{failed_nodes}}));
 753                 }
 754                 # ensure service get started again if it went unexpected down
 755                 # but ensure also no LRM result gets lost
 756                 $sd->{uid} = compute_new_uuid($sd->{state}) if defined($lrm_res);
 757             }
 758         }
 759
 760         return;
 761     }
 762
 763     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 764 }
 765
 766 sub next_state_error {
 767     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 768
 769     my $ns = $self->{ns};
 770     my $ms = $self->{ms};
 771
 772     if ($cd->{state} eq 'disabled') {
 773         # clean up on error recovery
 774         delete $sd->{failed_nodes};
 775
 776         &$change_service_state($self, $sid, 'stopped');
 777         return;
 778     }
 779
 780 }
 781
 782 1;