+# after a node was fenced this recovers the service to a new node
+my $recover_fenced_service = sub {
+ my ($self, $sid, $cd) = @_;
+
+ my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
+
+ my $sd = $ss->{$sid};
+
+ if ($sd->{state} ne 'fence') { # should not happen
+ $haenv->log('err', "cannot recover service '$sid' from fencing," .
+ " wrong state '$sd->{state}'");
+ return;
+ }
+
+ my $fenced_node = $sd->{node}; # for logging purpose
+
+ $self->recompute_online_node_usage(); # we want the most current node state
+
+ my $recovery_node = select_service_node($self->{groups},
+ $self->{online_node_usage},
+ $cd, $sd->{node});
+
+ if ($recovery_node) {
+ $haenv->log('info', "recover service '$sid' from fenced node " .
+ "'$fenced_node' to node '$recovery_node'");
+
+ &$fence_recovery_cleanup($self, $sid, $fenced_node);
+
+ $haenv->steal_service($sid, $sd->{node}, $recovery_node);
+
+ # $sd *is normally read-only*, fencing is the exception
+ $cd->{node} = $sd->{node} = $recovery_node;
+ my $new_state = ($cd->{state} eq 'started') ? 'started' : 'request_stop';
+ &$change_service_state($self, $sid, $new_state, node => $recovery_node);
+ } else {
+ # no possible node found, cannot recover
+ $haenv->log('err', "recovering service '$sid' from fenced node " .
+ "'$fenced_node' failed, no recovery node found");
+ &$change_service_state($self, $sid, 'error');
+ }
+};
+
+# read LRM status for all nodes