[pve-ha-manager.git] / src / PVE / HA / Manager.pm

package PVE::HA::Manager;

use strict;
use warnings;
use Digest::MD5 qw(md5_base64);

use Data::Dumper;
use PVE::Tools;
use PVE::HA::Tools ':exit_codes';
use PVE::HA::NodeStatus;

my $fence_delay = 60;

sub new {
    my ($this, $haenv) = @_;

    my $class = ref($this) || $this;

    my $ms = $haenv->read_manager_status();

    $ms->{master_node} = $haenv->nodename();

    my $ns = PVE::HA::NodeStatus->new($haenv, $ms->{node_status} || {});

    # fixme: use separate class  PVE::HA::ServiceStatus
    my $ss = $ms->{service_status} || {};

    my $self = bless {
	haenv => $haenv,
	ms => $ms, # master status
	ns => $ns, # PVE::HA::NodeStatus
	ss => $ss, # service status
    }, $class;

    return $self;
}

sub cleanup {
    my ($self) = @_;

    # todo: ?
}

sub flush_master_status {
    my ($self) = @_;

    my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});

    $ms->{node_status} = $ns->{status};
    $ms->{service_status} = $ss;
    $ms->{timestamp} = $haenv->get_time();
    
    $haenv->write_manager_status($ms);
} 

sub select_service_node {
    my ($groups, $online_node_usage, $service_conf, $current_node, $try_next) = @_;

    my $group = { 'nodes' => { $service_conf->{node} => 1 } }; # default group

    $group =  $groups->{ids}->{$service_conf->{group}} if $service_conf->{group} && 
	$groups->{ids}->{$service_conf->{group}};

    my $pri_groups = {};
    my $group_members = {};
    foreach my $entry (keys %{$group->{nodes}}) {
	my ($node, $pri) = ($entry, 0);
	if ($entry =~ m/^(\S+):(\d+)$/) {
	    ($node, $pri) = ($1, $2);
	}
	next if !defined($online_node_usage->{$node}); # offline
	$pri_groups->{$pri}->{$node} = 1;
	$group_members->{$node} = $pri;
    }

    
    # add non-group members to unrestricted groups (priority -1)
    if (!$group->{restricted}) {
	my $pri = -1;
	foreach my $node (keys %$online_node_usage) {
	    next if defined($group_members->{$node});
	    $pri_groups->{$pri}->{$node} = 1;
	    $group_members->{$node} = -1;
	}
    }


    my @pri_list = sort {$b <=> $a} keys %$pri_groups;
    return undef if !scalar(@pri_list);
    
    if (!$try_next && $group->{nofailback} && defined($group_members->{$current_node})) {
	return $current_node;
    }

    # select node from top priority node list

    my $top_pri = $pri_list[0];

    my @nodes = sort { 
	$online_node_usage->{$a} <=> $online_node_usage->{$b} || $a cmp $b
    } keys %{$pri_groups->{$top_pri}};

    my $found;
    for (my $i = scalar(@nodes) - 1; $i >= 0; $i--) {
	my $node = $nodes[$i];
	if ($node eq $current_node) {
	    $found = $i;
	    last;
	}
    }

    if ($try_next) {

	if (defined($found) && ($found < (scalar(@nodes) - 1))) {
	    return $nodes[$found + 1];
	} else {
	    return $nodes[0];
	}

    } else {

	return $nodes[$found] if defined($found);

	return $nodes[0];

    }
}

my $uid_counter = 0;

sub compute_new_uuid {
    my ($state) = @_;
    
    $uid_counter++;
    return md5_base64($state . $$ . time() . $uid_counter);
}

my $valid_service_states = {
    stopped => 1,
    request_stop => 1,
    started => 1,
    fence => 1,
    migrate => 1,
    relocate => 1,
    freeze => 1,
    error => 1,
};

sub recompute_online_node_usage {
    my ($self) = @_;

    my $online_node_usage = {};

    my $online_nodes = $self->{ns}->list_online_nodes();

    foreach my $node (@$online_nodes) {
	$online_node_usage->{$node} = 0;
    }

    foreach my $sid (keys %{$self->{ss}}) {
	my $sd = $self->{ss}->{$sid};
	my $state = $sd->{state};
	if (defined($online_node_usage->{$sd->{node}})) {
	    if (($state eq 'started') || ($state eq 'request_stop') || 
		($state eq 'fence') || ($state eq 'freeze') || ($state eq 'error')) {
		$online_node_usage->{$sd->{node}}++;
	    } elsif (($state eq 'migrate') || ($state eq 'relocate')) {
		$online_node_usage->{$sd->{target}}++;
	    } elsif ($state eq 'stopped') {
		# do nothing
	    } else {
		die "should not be reached";
	    }
	}
    }

    $self->{online_node_usage} = $online_node_usage;
}

my $change_service_state = sub {
    my ($self, $sid, $new_state, %params) = @_;

    my ($haenv, $ss) = ($self->{haenv}, $self->{ss});

    my $sd = $ss->{$sid} || die "no such service '$sid";

    my $old_state = $sd->{state};
    my $old_node = $sd->{node};

    die "no state change" if $old_state eq $new_state; # just to be sure

    die "invalid CRM service state '$new_state'\n" if !$valid_service_states->{$new_state};

    foreach my $k (keys %$sd) { delete $sd->{$k}; };

    $sd->{state} = $new_state;
    $sd->{node} = $old_node;

    my $text_state = '';
    foreach my $k (sort keys %params) {
	my $v = $params{$k};
	$text_state .= ", " if $text_state;
	$text_state .= "$k = $v";
	$sd->{$k} = $v;
    }

    $self->recompute_online_node_usage();

    $sd->{uid} = compute_new_uuid($new_state);
    

    $text_state = " ($text_state)" if $text_state;
    $haenv->log('info', "service '$sid': state changed from '${old_state}' to '${new_state}' $text_state");
};

# read LRM status for all nodes 
sub read_lrm_status {
    my ($self) = @_;

    my $nodes = $self->{ns}->list_nodes();
    my $haenv = $self->{haenv};

    my $results = {};
    my $modes = {};
    foreach my $node (@$nodes) {
	my $lrm_status = $haenv->read_lrm_status($node);
	$modes->{$node} = $lrm_status->{mode} || 'active';
	foreach my $uid (keys %{$lrm_status->{results}}) {
	    next if $results->{$uid}; # should not happen
	    $results->{$uid} = $lrm_status->{results}->{$uid};
	}
    }

    
    return ($results, $modes);
}

# read new crm commands and save them into crm master status
sub update_crm_commands {
    my ($self) = @_;

    my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});

    my $cmdlist = $haenv->read_crm_commands();
    
    foreach my $cmd (split(/\n/, $cmdlist)) {
	chomp $cmd;

	if ($cmd =~ m/^(migrate|relocate)\s+(\S+)\s+(\S+)$/) {
	    my ($task, $sid, $node) = ($1, $2, $3); 
	    if (my $sd = $ss->{$sid}) {
		if (!$ns->node_is_online($node)) {
		    $haenv->log('err', "crm command error - node not online: $cmd");
		} else {
		    if ($node eq $sd->{node}) {
			$haenv->log('info', "ignore crm command - service already on target node: $cmd");
		    } else { 
			$haenv->log('info', "got crm command: $cmd");
			$ss->{$sid}->{cmd} = [ $task, $node];
		    }
		}
	    } else {
		$haenv->log('err', "crm command error - no such service: $cmd");
	    }

	} else {
	    $haenv->log('err', "unable to parse crm command: $cmd");
	}
    }

}

sub manage {
    my ($self) = @_;

    my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});

    $ns->update($haenv->get_node_info());

    if (!$ns->node_is_online($haenv->nodename())) {
	$haenv->log('info', "master seems offline");
	return;
    }

    my ($lrm_results, $lrm_modes) = $self->read_lrm_status();

    my $sc = $haenv->read_service_config();

    $self->{groups} = $haenv->read_group_config(); # update

    # compute new service status

    # add new service
    foreach my $sid (sort keys %$sc) {
	next if $ss->{$sid}; # already there
	$haenv->log('info', "adding new service '$sid' on node '$sc->{$sid}->{node}'");
	# assume we are running to avoid relocate running service at add
	$ss->{$sid} = { state => 'started', node => $sc->{$sid}->{node},
			uid => compute_new_uuid('started') };
    }

    # remove stale service from manager state
    foreach my $sid (keys %$ss) {
	next if $sc->{$sid};
	$haenv->log('info', "removing stale service '$sid' (no config)");
	delete $ss->{$sid};
    }
    
    $self->update_crm_commands();

    for (;;) {
	my $repeat = 0;
	
	$self->recompute_online_node_usage();

	foreach my $sid (keys %$ss) {
	    my $sd = $ss->{$sid};
	    my $cd = $sc->{$sid} || { state => 'disabled' };

	    my $lrm_res = $sd->{uid} ? $lrm_results->{$sd->{uid}} : undef;

	    my $last_state = $sd->{state};

	    if ($last_state eq 'stopped') {

		$self->next_state_stopped($sid, $cd, $sd, $lrm_res);

	    } elsif ($last_state eq 'started') {

		$self->next_state_started($sid, $cd, $sd, $lrm_res);

	    } elsif ($last_state eq 'migrate' || $last_state eq 'relocate') {

		$self->next_state_migrate_relocate($sid, $cd, $sd, $lrm_res);

	    } elsif ($last_state eq 'fence') {

		# do nothing here - wait until fenced

	    } elsif ($last_state eq 'request_stop') {

		$self->next_state_request_stop($sid, $cd, $sd, $lrm_res);

	    } elsif ($last_state eq 'freeze') {

		my $lrm_mode = $sd->{node} ? $lrm_modes->{$sd->{node}} : undef;
		# unfreeze
		&$change_service_state($self, $sid, 'started') 
		    if $lrm_mode && $lrm_mode eq 'active';

	    } elsif ($last_state eq 'error') {

		$self->next_state_error($sid, $cd, $sd, $lrm_res);

	    } else {

		die "unknown service state '$last_state'";
	    }

	    my $lrm_mode = $sd->{node} ? $lrm_modes->{$sd->{node}} : undef;
	    if ($lrm_mode && $lrm_mode eq 'restart') {
		if (($sd->{state} eq 'started' || $sd->{state} eq 'stopped' ||
		     $sd->{state} eq 'request_stop')) {
		    &$change_service_state($self, $sid, 'freeze');
		}
	    }

	    $repeat = 1 if $sd->{state} ne $last_state;
	}

	# handle fencing
	my $fenced_nodes = {};
	foreach my $sid (keys %$ss) {
	    my $sd = $ss->{$sid};
	    next if $sd->{state} ne 'fence';

	    if (!defined($fenced_nodes->{$sd->{node}})) {
		$fenced_nodes->{$sd->{node}} = $ns->fence_node($sd->{node}) || 0;
	    }

	    next if !$fenced_nodes->{$sd->{node}};

	    # node fence was successful - mark service as stopped
	    &$change_service_state($self, $sid, 'stopped');	    
	}

	last if !$repeat;
    }

    $self->flush_master_status();
}

# functions to compute next service states
# $cd: service configuration data (read only)
# $sd: service status data (read only)
#
# Note: use change_service_state() to alter state
#

sub next_state_request_stop {
    my ($self, $sid, $cd, $sd, $lrm_res) = @_;

    my $haenv = $self->{haenv};
    my $ns = $self->{ns};

    # check result from LRM daemon
    if ($lrm_res) {
	my $exit_code = $lrm_res->{exit_code};
	if ($exit_code == SUCCESS) {
	    &$change_service_state($self, $sid, 'stopped');
	    return;
	} else {
	    $haenv->log('err', "service '$sid' stop failed (exit code $exit_code)");
	    &$change_service_state($self, $sid, 'error'); # fixme: what state?
	    return;
	}
    }

    if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
	&$change_service_state($self, $sid, 'fence');
	return;
    }
}

sub next_state_migrate_relocate {
    my ($self, $sid, $cd, $sd, $lrm_res) = @_;

    my $haenv = $self->{haenv};
    my $ns = $self->{ns};

    # check result from LRM daemon
    if ($lrm_res) {
	my $exit_code = $lrm_res->{exit_code};
	if ($exit_code == SUCCESS) {
	    &$change_service_state($self, $sid, 'started', node => $sd->{target});
	    return;
	} else {
	    $haenv->log('err', "service '$sid' - migration failed (exit code $exit_code)");
	    &$change_service_state($self, $sid, 'started', node => $sd->{node});
	    return;
	}
    }

    if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
	&$change_service_state($self, $sid, 'fence');
	return;
    }
}


sub next_state_stopped {
    my ($self, $sid, $cd, $sd, $lrm_res) = @_;

    my $haenv = $self->{haenv};
    my $ns = $self->{ns};

    if ($sd->{node} ne $cd->{node}) {
	# this can happen if we fence a node with active migrations
	# hack: modify $sd (normally this should be considered read-only)
	$haenv->log('info', "fixup service '$sid' location ($sd->{node} => $cd->{node}");
	$sd->{node} = $cd->{node}; 
    }

    if ($sd->{cmd}) {
	my ($cmd, $target) = @{$sd->{cmd}};
	delete $sd->{cmd};

	if ($cmd eq 'migrate' || $cmd eq 'relocate') {
	    if (!$ns->node_is_online($target)) {
		$haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
	    } elsif ($sd->{node} eq $target) {
		$haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
	    } else {
		eval {
		    $haenv->change_service_location($sid, $sd->{node}, $target);
		    $cd->{node} = $sd->{node} = $target; # fixme: $sd is read-only??!!	    
		    $haenv->log('info', "$cmd service '$sid' to node '$target' (stopped)");
		};
		if (my $err = $@) {
		    $haenv->log('err', "$cmd service '$sid' to node '$target' failed - $err");
		}
	    }
	} else {
	    $haenv->log('err', "unknown command '$cmd' for service '$sid'"); 
	}
    } 

    if ($cd->{state} eq 'disabled') {
	# do nothing
	return;
    } 

    if ($cd->{state} eq 'enabled') {
	if (my $node = select_service_node($self->{groups}, $self->{online_node_usage}, $cd, $sd->{node})) {
	    if ($node && ($sd->{node} ne $node)) {
		eval {
		    $haenv->change_service_location($sid, $sd->{node}, $node);
		    $cd->{node} = $sd->{node} = $node; # fixme: $sd is read-only??!!
		};
		if (my $err = $@) {
		    $haenv->log('err', "move service '$sid' to node '$node' failed - $err");
		} else {
		    &$change_service_state($self, $sid, 'started', node => $node);
		}
	    } else {
		&$change_service_state($self, $sid, 'started', node => $node);
	    }
	} else {
	    # fixme: warn 
	}

	return;
    }

    $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
}

sub next_state_started {
    my ($self, $sid, $cd, $sd, $lrm_res) = @_;

    my $haenv = $self->{haenv};
    my $master_status = $self->{ms};
    my $ns = $self->{ns};

    if (!$ns->node_is_online($sd->{node})) {
	if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
	    &$change_service_state($self, $sid, 'fence');
	}
	return;
    }
	
    if ($cd->{state} eq 'disabled') {
	&$change_service_state($self, $sid, 'request_stop');
	return;
    }

    if ($cd->{state} eq 'enabled') {

	if ($sd->{cmd}) {
	    my ($cmd, $target) = @{$sd->{cmd}};
	    delete $sd->{cmd};

	    if ($cmd eq 'migrate' || $cmd eq 'relocate') {
		if (!$ns->node_is_online($target)) {
		    $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
		} elsif ($sd->{node} eq $target) {
		    $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
		} else {
		    $haenv->log('info', "$cmd service '$sid' to node '$target'");
		    &$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
		}
	    } else {
		$haenv->log('err', "unknown command '$cmd' for service '$sid'"); 
	    }
	} else {

	    my $try_next = 0;
	    if ($lrm_res) {
		my $ec = $lrm_res->{exit_code};
		if ($ec == SUCCESS) {

		    $master_status->{relocate_trial}->{$sid} = 0;

		} elsif ($ec == ETRY_AGAIN) {

		    # do nothing, the LRM wants to try again

		} elsif ($ec == ERROR) {
		    # apply our relocate policy if we got ERROR from the LRM

		    my $try = $master_status->{relocate_trial}->{$sid} || 0;

		    if ($try < $cd->{max_relocate}) {

			$try++;
			# tell select_service_node to relocate if possible
			$try_next = 1;

			$haenv->log('warning', "starting service $sid on node".
				   " '$sd->{node}' failed, relocating service.");
			$master_status->{relocate_trial}->{$sid} = $try;

		    } else {

			$haenv->log('err', "recovery policy for service".
				   " $sid failed, entering error state!");
			&$change_service_state($self, $sid, 'error');
			return;

		    }
		} else {
		    $haenv->log('err', "service '$sid' got unrecoverable error" .
				" (exit code $ec))");
		    # we have no save way out (yet) for other errors
		    &$change_service_state($self, $sid, 'error');
		}
	    }

	    my $node = select_service_node($self->{groups}, $self->{online_node_usage}, 
					   $cd, $sd->{node}, $try_next);

	    if ($node && ($sd->{node} ne $node)) {
		if ($cd->{type} eq 'vm') {
		    $haenv->log('info', "migrate service '$sid' to node '$node' (running)");
		    &$change_service_state($self, $sid, 'migrate', node => $sd->{node}, target => $node);
		} else {
		    $haenv->log('info', "relocate service '$sid' to node '$node'");
		    &$change_service_state($self, $sid, 'relocate', node => $sd->{node}, target => $node);
		}
	    } else {
		# do nothing
	    }
	}

	return;
    } 

    $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
}

sub next_state_error {
    my ($self, $sid, $cd, $sd, $lrm_res) = @_;

    my $ns = $self->{ns};

    if ($cd->{state} eq 'disabled') {
	&$change_service_state($self, $sid, 'stopped');
	return;
    }

    if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
	&$change_service_state($self, $sid, 'fence');
	return;
    }

}

1;
Commit	Line	Data
c0bbd038 DM	1	package PVE::HA::Manager;
	2
	3	use strict;
	4	use warnings;
c4a221bc	5	use Digest::MD5 qw(md5_base64);
c0bbd038 DM	6
c0bbd038 DM	7	use Data::Dumper;
c142ebc9	8	use PVE::Tools;
a89ff919	9	use PVE::HA::Tools ':exit_codes';
c0bbd038 DM	10	use PVE::HA::NodeStatus;
c0bbd038 DM	11
ceac1930	12	my $fence_delay = 60;
5385a606	13
c0bbd038	14	sub new {
8f0bb968	15	my ($this, $haenv) = @_;
c0bbd038 DM	16
	17	my $class = ref($this) \|\| $this;
	18
8f0bb968 DM	19	my $ms = $haenv->read_manager_status();
	20
	21	$ms->{master_node} = $haenv->nodename();
	22
	23	my $ns = PVE::HA::NodeStatus->new($haenv, $ms->{node_status} \|\| {});
	24
59fd7207 DM	25	# fixme: use separate class PVE::HA::ServiceStatus
	26	my $ss = $ms->{service_status} \|\| {};
	27
c0bbd038	28	my $self = bless {
8f0bb968 DM	29	haenv => $haenv,
	30	ms => $ms, # master status
	31	ns => $ns, # PVE::HA::NodeStatus
59fd7207	32	ss => $ss, # service status
c0bbd038 DM	33	}, $class;
	34
	35	return $self;
	36	}
	37
d84da043 DM	38	sub cleanup {
	39	my ($self) = @_;
	40
	41	# todo: ?
	42	}
	43
8f0bb968	44	sub flush_master_status {
c0bbd038 DM	45	my ($self) = @_;
c0bbd038 DM	46
59fd7207	47	my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
c0bbd038	48
8f0bb968	49	$ms->{node_status} = $ns->{status};
59fd7207	50	$ms->{service_status} = $ss;
d2f612cf DM	51	$ms->{timestamp} = $haenv->get_time();
d2f612cf DM	52
8f0bb968 DM	53	$haenv->write_manager_status($ms);
8f0bb968 DM	54	}
c0bbd038	55
f7ccd1b3	56	sub select_service_node {
270d4406	57	my ($groups, $online_node_usage, $service_conf, $current_node, $try_next) = @_;
f7ccd1b3	58
e0a56314	59	my $group = { 'nodes' => { $service_conf->{node} => 1 } }; # default group
abc920b4	60
270d4406 DM	61	$group = $groups->{ids}->{$service_conf->{group}} if $service_conf->{group} &&
270d4406 DM	62	$groups->{ids}->{$service_conf->{group}};
abc920b4 DM	63
	64	my $pri_groups = {};
	65	my $group_members = {};
e0a56314	66	foreach my $entry (keys %{$group->{nodes}}) {
abc920b4 DM	67	my ($node, $pri) = ($entry, 0);
	68	if ($entry =~ m/^(\S+):(\d+)$/) {
	69	($node, $pri) = ($1, $2);
	70	}
270d4406	71	next if !defined($online_node_usage->{$node}); # offline
abc920b4 DM	72	$pri_groups->{$pri}->{$node} = 1;
	73	$group_members->{$node} = $pri;
	74	}
f7ccd1b3	75
270d4406	76
abc920b4 DM	77	# add non-group members to unrestricted groups (priority -1)
	78	if (!$group->{restricted}) {
	79	my $pri = -1;
270d4406	80	foreach my $node (keys %$online_node_usage) {
abc920b4 DM	81	next if defined($group_members->{$node});
	82	$pri_groups->{$pri}->{$node} = 1;
	83	$group_members->{$node} = -1;
	84	}
	85	}
	86
270d4406	87
abc920b4 DM	88	my @pri_list = sort {$b <=> $a} keys %$pri_groups;
abc920b4 DM	89	return undef if !scalar(@pri_list);
270d4406	90
abc920b4 DM	91	if (!$try_next && $group->{nofailback} && defined($group_members->{$current_node})) {
	92	return $current_node;
	93	}
	94
	95	# select node from top priority node list
	96
	97	my $top_pri = $pri_list[0];
	98
c142ebc9 DM	99	my @nodes = sort {
	100	$online_node_usage->{$a} <=> $online_node_usage->{$b} \|\| $a cmp $b
	101	} keys %{$pri_groups->{$top_pri}};
abc920b4 DM	102
	103	my $found;
	104	for (my $i = scalar(@nodes) - 1; $i >= 0; $i--) {
	105	my $node = $nodes[$i];
	106	if ($node eq $current_node) {
	107	$found = $i;
	108	last;
	109	}
	110	}
	111
abc920b4 DM	112	if ($try_next) {
	113
	114	if (defined($found) && ($found < (scalar(@nodes) - 1))) {
	115	return $nodes[$found + 1];
	116	} else {
	117	return $nodes[0];
	118	}
	119
	120	} else {
	121
	122	return $nodes[$found] if defined($found);
	123
	124	return $nodes[0];
	125
	126	}
f7ccd1b3 DM	127	}
f7ccd1b3 DM	128
c4a221bc DM	129	my $uid_counter = 0;
c4a221bc DM	130
d55aa611 DM	131	sub compute_new_uuid {
	132	my ($state) = @_;
	133
	134	$uid_counter++;
	135	return md5_base64($state . $$ . time() . $uid_counter);
	136	}
	137
618fbeda DM	138	my $valid_service_states = {
	139	stopped => 1,
	140	request_stop => 1,
	141	started => 1,
	142	fence => 1,
	143	migrate => 1,
b0fdf86a	144	relocate => 1,
9c7d068b	145	freeze => 1,
618fbeda DM	146	error => 1,
	147	};
	148
270d4406 DM	149	sub recompute_online_node_usage {
	150	my ($self) = @_;
	151
	152	my $online_node_usage = {};
	153
	154	my $online_nodes = $self->{ns}->list_online_nodes();
	155
	156	foreach my $node (@$online_nodes) {
	157	$online_node_usage->{$node} = 0;
	158	}
	159
	160	foreach my $sid (keys %{$self->{ss}}) {
	161	my $sd = $self->{ss}->{$sid};
	162	my $state = $sd->{state};
	163	if (defined($online_node_usage->{$sd->{node}})) {
	164	if (($state eq 'started') \|\| ($state eq 'request_stop') \|\|
9c7d068b	165	($state eq 'fence') \|\| ($state eq 'freeze') \|\| ($state eq 'error')) {
270d4406 DM	166	$online_node_usage->{$sd->{node}}++;
	167	} elsif (($state eq 'migrate') \|\| ($state eq 'relocate')) {
	168	$online_node_usage->{$sd->{target}}++;
	169	} elsif ($state eq 'stopped') {
	170	# do nothing
	171	} else {
	172	die "should not be reached";
	173	}
	174	}
	175	}
	176
	177	$self->{online_node_usage} = $online_node_usage;
	178	}
	179
4e01bc86 DM	180	my $change_service_state = sub {
	181	my ($self, $sid, $new_state, %params) = @_;
	182
	183	my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
	184
	185	my $sd = $ss->{$sid} \|\| die "no such service '$sid";
	186
	187	my $old_state = $sd->{state};
e4ffb299	188	my $old_node = $sd->{node};
4e01bc86 DM	189
	190	die "no state change" if $old_state eq $new_state; # just to be sure
	191
618fbeda DM	192	die "invalid CRM service state '$new_state'\n" if !$valid_service_states->{$new_state};
618fbeda DM	193
e4ffb299 DM	194	foreach my $k (keys %$sd) { delete $sd->{$k}; };
	195
	196	$sd->{state} = $new_state;
	197	$sd->{node} = $old_node;
	198
	199	my $text_state = '';
ba623362	200	foreach my $k (sort keys %params) {
4e01bc86	201	my $v = $params{$k};
e4ffb299 DM	202	$text_state .= ", " if $text_state;
e4ffb299 DM	203	$text_state .= "$k = $v";
4e01bc86 DM	204	$sd->{$k} = $v;
4e01bc86 DM	205	}
270d4406 DM	206
	207	$self->recompute_online_node_usage();
	208
d55aa611 DM	209	$sd->{uid} = compute_new_uuid($new_state);
d55aa611 DM	210
4e01bc86	211
e4ffb299	212	$text_state = " ($text_state)" if $text_state;
e5986717	213	$haenv->log('info', "service '$sid': state changed from '${old_state}' to '${new_state}' $text_state");
4e01bc86 DM	214	};
4e01bc86 DM	215
9c7d068b	216	# read LRM status for all nodes
c4a221bc	217	sub read_lrm_status {
332170bd	218	my ($self) = @_;
c4a221bc	219
9c7d068b	220	my $nodes = $self->{ns}->list_nodes();
c4a221bc DM	221	my $haenv = $self->{haenv};
c4a221bc DM	222
9c7d068b DM	223	my $results = {};
9c7d068b DM	224	my $modes = {};
332170bd	225	foreach my $node (@$nodes) {
9c7d068b	226	my $lrm_status = $haenv->read_lrm_status($node);
02ffd753	227	$modes->{$node} = $lrm_status->{mode} \|\| 'active';
9c7d068b DM	228	foreach my $uid (keys %{$lrm_status->{results}}) {
	229	next if $results->{$uid}; # should not happen
	230	$results->{$uid} = $lrm_status->{results}->{$uid};
c4a221bc DM	231	}
	232	}
	233
9c7d068b DM	234
9c7d068b DM	235	return ($results, $modes);
c4a221bc DM	236	}
c4a221bc DM	237
aa98a844 DM	238	# read new crm commands and save them into crm master status
	239	sub update_crm_commands {
	240	my ($self) = @_;
	241
	242	my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
	243
	244	my $cmdlist = $haenv->read_crm_commands();
	245
	246	foreach my $cmd (split(/\n/, $cmdlist)) {
	247	chomp $cmd;
	248
b0fdf86a DM	249	if ($cmd =~ m/^(migrate\|relocate)\s+(\S+)\s+(\S+)$/) {
b0fdf86a DM	250	my ($task, $sid, $node) = ($1, $2, $3);
aa98a844 DM	251	if (my $sd = $ss->{$sid}) {
	252	if (!$ns->node_is_online($node)) {
	253	$haenv->log('err', "crm command error - node not online: $cmd");
	254	} else {
	255	if ($node eq $sd->{node}) {
	256	$haenv->log('info', "ignore crm command - service already on target node: $cmd");
	257	} else {
	258	$haenv->log('info', "got crm command: $cmd");
b0fdf86a	259	$ss->{$sid}->{cmd} = [ $task, $node];
aa98a844 DM	260	}
	261	}
	262	} else {
	263	$haenv->log('err', "crm command error - no such service: $cmd");
	264	}
	265
	266	} else {
	267	$haenv->log('err', "unable to parse crm command: $cmd");
	268	}
	269	}
	270
	271	}
	272
8f0bb968 DM	273	sub manage {
8f0bb968 DM	274	my ($self) = @_;
c0bbd038	275
59fd7207	276	my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
c0bbd038	277
332170bd	278	$ns->update($haenv->get_node_info());
c79442f2 DM	279
c79442f2 DM	280	if (!$ns->node_is_online($haenv->nodename())) {
e5986717	281	$haenv->log('info', "master seems offline");
c79442f2 DM	282	return;
	283	}
	284
9c7d068b	285	my ($lrm_results, $lrm_modes) = $self->read_lrm_status();
c4a221bc	286
f7ccd1b3 DM	287	my $sc = $haenv->read_service_config();
f7ccd1b3 DM	288
abc920b4 DM	289	$self->{groups} = $haenv->read_group_config(); # update
abc920b4 DM	290
f7ccd1b3 DM	291	# compute new service status
	292
	293	# add new service
cc32a8f3	294	foreach my $sid (sort keys %$sc) {
f7ccd1b3	295	next if $ss->{$sid}; # already there
cc32a8f3	296	$haenv->log('info', "adding new service '$sid' on node '$sc->{$sid}->{node}'");
f7ccd1b3	297	# assume we are running to avoid relocate running service at add
d55aa611 DM	298	$ss->{$sid} = { state => 'started', node => $sc->{$sid}->{node},
d55aa611 DM	299	uid => compute_new_uuid('started') };
f7ccd1b3 DM	300	}
f7ccd1b3 DM	301
4e5764af DM	302	# remove stale service from manager state
	303	foreach my $sid (keys %$ss) {
	304	next if $sc->{$sid};
	305	$haenv->log('info', "removing stale service '$sid' (no config)");
	306	delete $ss->{$sid};
	307	}
	308
aa98a844 DM	309	$self->update_crm_commands();
aa98a844 DM	310
c79442f2 DM	311	for (;;) {
c79442f2 DM	312	my $repeat = 0;
270d4406 DM	313
270d4406 DM	314	$self->recompute_online_node_usage();
f7ccd1b3	315
c79442f2 DM	316	foreach my $sid (keys %$ss) {
	317	my $sd = $ss->{$sid};
	318	my $cd = $sc->{$sid} \|\| { state => 'disabled' };
f7ccd1b3	319
9c7d068b	320	my $lrm_res = $sd->{uid} ? $lrm_results->{$sd->{uid}} : undef;
a875fbe8	321
c79442f2 DM	322	my $last_state = $sd->{state};
	323
	324	if ($last_state eq 'stopped') {
	325
abc920b4	326	$self->next_state_stopped($sid, $cd, $sd, $lrm_res);
f7ccd1b3	327
c79442f2	328	} elsif ($last_state eq 'started') {
f7ccd1b3	329
abc920b4	330	$self->next_state_started($sid, $cd, $sd, $lrm_res);
f7ccd1b3	331
b0fdf86a	332	} elsif ($last_state eq 'migrate' \|\| $last_state eq 'relocate') {
f7ccd1b3	333
8aaa0e36	334	$self->next_state_migrate_relocate($sid, $cd, $sd, $lrm_res);
f7ccd1b3	335
c79442f2	336	} elsif ($last_state eq 'fence') {
f7ccd1b3	337
21e37ed4	338	# do nothing here - wait until fenced
f7ccd1b3	339
c79442f2	340	} elsif ($last_state eq 'request_stop') {
f7ccd1b3	341
0df5b3dd	342	$self->next_state_request_stop($sid, $cd, $sd, $lrm_res);
618fbeda	343
9c7d068b DM	344	} elsif ($last_state eq 'freeze') {
	345
	346	my $lrm_mode = $sd->{node} ? $lrm_modes->{$sd->{node}} : undef;
9c7d068b DM	347	# unfreeze
9c7d068b DM	348	&$change_service_state($self, $sid, 'started')
02ffd753	349	if $lrm_mode && $lrm_mode eq 'active';
9c7d068b	350
e88469ba DM	351	} elsif ($last_state eq 'error') {
e88469ba DM	352
a2881965	353	$self->next_state_error($sid, $cd, $sd, $lrm_res);
e88469ba	354
a875fbe8 DM	355	} else {
	356
	357	die "unknown service state '$last_state'";
618fbeda	358	}
21e37ed4	359
9c7d068b	360	my $lrm_mode = $sd->{node} ? $lrm_modes->{$sd->{node}} : undef;
07adc6a6 DM	361	if ($lrm_mode && $lrm_mode eq 'restart') {
	362	if (($sd->{state} eq 'started' \|\| $sd->{state} eq 'stopped' \|\|
	363	$sd->{state} eq 'request_stop')) {
	364	&$change_service_state($self, $sid, 'freeze');
	365	}
9c7d068b	366	}
07adc6a6	367
c79442f2	368	$repeat = 1 if $sd->{state} ne $last_state;
f7ccd1b3 DM	369	}
f7ccd1b3 DM	370
21e37ed4 DM	371	# handle fencing
	372	my $fenced_nodes = {};
	373	foreach my $sid (keys %$ss) {
	374	my $sd = $ss->{$sid};
	375	next if $sd->{state} ne 'fence';
	376
	377	if (!defined($fenced_nodes->{$sd->{node}})) {
	378	$fenced_nodes->{$sd->{node}} = $ns->fence_node($sd->{node}) \|\| 0;
	379	}
	380
	381	next if !$fenced_nodes->{$sd->{node}};
	382
63f6a08c	383	# node fence was successful - mark service as stopped
21e37ed4 DM	384	&$change_service_state($self, $sid, 'stopped');
	385	}
	386
c79442f2	387	last if !$repeat;
f7ccd1b3	388	}
f7ccd1b3	389
8f0bb968	390	$self->flush_master_status();
c0bbd038 DM	391	}
c0bbd038 DM	392
a875fbe8 DM	393	# functions to compute next service states
	394	# $cd: service configuration data (read only)
	395	# $sd: service status data (read only)
	396	#
	397	# Note: use change_service_state() to alter state
	398	#
	399
0df5b3dd DM	400	sub next_state_request_stop {
	401	my ($self, $sid, $cd, $sd, $lrm_res) = @_;
	402
	403	my $haenv = $self->{haenv};
	404	my $ns = $self->{ns};
	405
	406	# check result from LRM daemon
	407	if ($lrm_res) {
	408	my $exit_code = $lrm_res->{exit_code};
a89ff919	409	if ($exit_code == SUCCESS) {
0df5b3dd DM	410	&$change_service_state($self, $sid, 'stopped');
	411	return;
	412	} else {
33f01524	413	$haenv->log('err', "service '$sid' stop failed (exit code $exit_code)");
0df5b3dd DM	414	&$change_service_state($self, $sid, 'error'); # fixme: what state?
	415	return;
	416	}
	417	}
	418
5385a606	419	if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
0df5b3dd DM	420	&$change_service_state($self, $sid, 'fence');
	421	return;
	422	}
	423	}
	424
8aaa0e36 DM	425	sub next_state_migrate_relocate {
	426	my ($self, $sid, $cd, $sd, $lrm_res) = @_;
	427
	428	my $haenv = $self->{haenv};
	429	my $ns = $self->{ns};
	430
	431	# check result from LRM daemon
	432	if ($lrm_res) {
	433	my $exit_code = $lrm_res->{exit_code};
a89ff919	434	if ($exit_code == SUCCESS) {
8aaa0e36 DM	435	&$change_service_state($self, $sid, 'started', node => $sd->{target});
	436	return;
	437	} else {
	438	$haenv->log('err', "service '$sid' - migration failed (exit code $exit_code)");
	439	&$change_service_state($self, $sid, 'started', node => $sd->{node});
	440	return;
	441	}
	442	}
	443
5385a606	444	if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
8aaa0e36 DM	445	&$change_service_state($self, $sid, 'fence');
	446	return;
	447	}
	448	}
	449
	450
a875fbe8	451	sub next_state_stopped {
abc920b4	452	my ($self, $sid, $cd, $sd, $lrm_res) = @_;
a875fbe8 DM	453
a875fbe8 DM	454	my $haenv = $self->{haenv};
e88469ba	455	my $ns = $self->{ns};
a875fbe8	456
ff6f1c5c DM	457	if ($sd->{node} ne $cd->{node}) {
	458	# this can happen if we fence a node with active migrations
	459	# hack: modify $sd (normally this should be considered read-only)
	460	$haenv->log('info', "fixup service '$sid' location ($sd->{node} => $cd->{node}");
	461	$sd->{node} = $cd->{node};
	462	}
	463
94b7ebe2 DM	464	if ($sd->{cmd}) {
	465	my ($cmd, $target) = @{$sd->{cmd}};
	466	delete $sd->{cmd};
	467
b0fdf86a	468	if ($cmd eq 'migrate' \|\| $cmd eq 'relocate') {
94b7ebe2	469	if (!$ns->node_is_online($target)) {
b0fdf86a	470	$haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
e88469ba	471	} elsif ($sd->{node} eq $target) {
b0fdf86a	472	$haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
94b7ebe2	473	} else {
6da27e23 DM	474	eval {
	475	$haenv->change_service_location($sid, $sd->{node}, $target);
	476	$cd->{node} = $sd->{node} = $target; # fixme: $sd is read-only??!!
	477	$haenv->log('info', "$cmd service '$sid' to node '$target' (stopped)");
	478	};
	479	if (my $err = $@) {
	480	$haenv->log('err', "$cmd service '$sid' to node '$target' failed - $err");
	481	}
94b7ebe2 DM	482	}
	483	} else {
	484	$haenv->log('err', "unknown command '$cmd' for service '$sid'");
	485	}
	486	}
	487
a875fbe8 DM	488	if ($cd->{state} eq 'disabled') {
a875fbe8 DM	489	# do nothing
e88469ba DM	490	return;
	491	}
	492
	493	if ($cd->{state} eq 'enabled') {
270d4406	494	if (my $node = select_service_node($self->{groups}, $self->{online_node_usage}, $cd, $sd->{node})) {
a875fbe8	495	if ($node && ($sd->{node} ne $node)) {
6da27e23 DM	496	eval {
	497	$haenv->change_service_location($sid, $sd->{node}, $node);
	498	$cd->{node} = $sd->{node} = $node; # fixme: $sd is read-only??!!
	499	};
	500	if (my $err = $@) {
	501	$haenv->log('err', "move service '$sid' to node '$node' failed - $err");
	502	} else {
	503	&$change_service_state($self, $sid, 'started', node => $node);
	504	}
	505	} else {
	506	&$change_service_state($self, $sid, 'started', node => $node);
a875fbe8	507	}
a875fbe8 DM	508	} else {
	509	# fixme: warn
	510	}
e88469ba DM	511
e88469ba DM	512	return;
a875fbe8	513	}
e88469ba DM	514
e88469ba DM	515	$haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
a875fbe8 DM	516	}
	517
	518	sub next_state_started {
abc920b4	519	my ($self, $sid, $cd, $sd, $lrm_res) = @_;
a875fbe8 DM	520
a875fbe8 DM	521	my $haenv = $self->{haenv};
ea4443cc	522	my $master_status = $self->{ms};
a875fbe8 DM	523	my $ns = $self->{ns};
	524
	525	if (!$ns->node_is_online($sd->{node})) {
5385a606 DM	526	if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
	527	&$change_service_state($self, $sid, 'fence');
	528	}
e88469ba DM	529	return;
e88469ba DM	530	}
a875fbe8	531
e88469ba DM	532	if ($cd->{state} eq 'disabled') {
	533	&$change_service_state($self, $sid, 'request_stop');
	534	return;
	535	}
	536
	537	if ($cd->{state} eq 'enabled') {
e88469ba DM	538
	539	if ($sd->{cmd}) {
	540	my ($cmd, $target) = @{$sd->{cmd}};
	541	delete $sd->{cmd};
	542
b0fdf86a	543	if ($cmd eq 'migrate' \|\| $cmd eq 'relocate') {
e88469ba	544	if (!$ns->node_is_online($target)) {
b0fdf86a	545	$haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
e88469ba	546	} elsif ($sd->{node} eq $target) {
b0fdf86a	547	$haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
e88469ba	548	} else {
a3cb8dcb	549	$haenv->log('info', "$cmd service '$sid' to node '$target'");
b0fdf86a	550	&$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
e88469ba	551	}
a875fbe8	552	} else {
e88469ba	553	$haenv->log('err', "unknown command '$cmd' for service '$sid'");
a875fbe8 DM	554	}
a875fbe8 DM	555	} else {
b0fdf86a	556
abc920b4	557	my $try_next = 0;
ea4443cc	558	if ($lrm_res) {
e9e1cd68 TL	559	my $ec = $lrm_res->{exit_code};
	560	if ($ec == SUCCESS) {
	561
	562	$master_status->{relocate_trial}->{$sid} = 0;
	563
	564	} elsif ($ec == ETRY_AGAIN) {
	565
	566	# do nothing, the LRM wants to try again
	567
	568	} elsif ($ec == ERROR) {
	569	# apply our relocate policy if we got ERROR from the LRM
ea4443cc TL	570
	571	my $try = $master_status->{relocate_trial}->{$sid} \|\| 0;
	572
	573	if ($try < $cd->{max_relocate}) {
	574
	575	$try++;
e9e1cd68 TL	576	# tell select_service_node to relocate if possible
e9e1cd68 TL	577	$try_next = 1;
ea4443cc TL	578
	579	$haenv->log('warning', "starting service $sid on node".
	580	" '$sd->{node}' failed, relocating service.");
	581	$master_status->{relocate_trial}->{$sid} = $try;
	582
	583	} else {
	584
	585	$haenv->log('err', "recovery policy for service".
	586	" $sid failed, entering error state!");
	587	&$change_service_state($self, $sid, 'error');
	588	return;
	589
	590	}
e9e1cd68 TL	591	} else {
	592	$haenv->log('err', "service '$sid' got unrecoverable error" .
	593	" (exit code $ec))");
	594	# we have no save way out (yet) for other errors
	595	&$change_service_state($self, $sid, 'error');
ea4443cc	596	}
abc920b4 DM	597	}
abc920b4 DM	598
270d4406 DM	599	my $node = select_service_node($self->{groups}, $self->{online_node_usage},
270d4406 DM	600	$cd, $sd->{node}, $try_next);
abc920b4	601
b0fdf86a	602	if ($node && ($sd->{node} ne $node)) {
c0255b2c TL	603	if ($cd->{type} eq 'vm') {
	604	$haenv->log('info', "migrate service '$sid' to node '$node' (running)");
	605	&$change_service_state($self, $sid, 'migrate', node => $sd->{node}, target => $node);
	606	} else {
	607	$haenv->log('info', "relocate service '$sid' to node '$node'");
	608	&$change_service_state($self, $sid, 'relocate', node => $sd->{node}, target => $node);
	609	}
b0fdf86a DM	610	} else {
	611	# do nothing
	612	}
a875fbe8	613	}
e88469ba DM	614
	615	return;
	616	}
	617
	618	$haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
a875fbe8	619	}
c0bbd038	620
a2881965 TL	621	sub next_state_error {
	622	my ($self, $sid, $cd, $sd, $lrm_res) = @_;
	623
	624	my $ns = $self->{ns};
	625
	626	if ($cd->{state} eq 'disabled') {
	627	&$change_service_state($self, $sid, 'stopped');
	628	return;
	629	}
	630
	631	if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
	632	&$change_service_state($self, $sid, 'fence');
	633	return;
	634	}
	635
	636	}
	637
c0bbd038	638	1;