[pve-ha-manager.git] / src / PVE / HA / Manager.pm

package PVE::HA::Manager;

use strict;
use warnings;
use Digest::MD5 qw(md5_base64);

use Data::Dumper;
use PVE::Tools;
use PVE::HA::NodeStatus;

my $fence_delay = 30;

sub new {
    my ($this, $haenv) = @_;

    my $class = ref($this) || $this;

    my $ms = $haenv->read_manager_status();

    $ms->{master_node} = $haenv->nodename();

    my $ns = PVE::HA::NodeStatus->new($haenv, $ms->{node_status} || {});

    # fixme: use separate class  PVE::HA::ServiceStatus
    my $ss = $ms->{service_status} || {};

    my $self = bless {
	haenv => $haenv,
	ms => $ms, # master status
	ns => $ns, # PVE::HA::NodeStatus
	ss => $ss, # service status
    }, $class;

    return $self;
}

sub cleanup {
    my ($self) = @_;

    # todo: ?
}

sub flush_master_status {
    my ($self) = @_;

    my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});

    $ms->{node_status} = $ns->{status};
    $ms->{service_status} = $ss;

    $haenv->write_manager_status($ms);
} 

sub select_service_node {
    my ($groups, $online_node_usage, $service_conf, $current_node, $try_next) = @_;

    my $group = { 'nodes' => $service_conf->{node} }; # default group

    $group =  $groups->{ids}->{$service_conf->{group}} if $service_conf->{group} && 
	$groups->{ids}->{$service_conf->{group}};

    my $pri_groups = {};
    my $group_members = {};
    foreach my $entry (PVE::Tools::split_list($group->{nodes})) {
	my ($node, $pri) = ($entry, 0);
	if ($entry =~ m/^(\S+):(\d+)$/) {
	    ($node, $pri) = ($1, $2);
	}
	next if !defined($online_node_usage->{$node}); # offline
	$pri_groups->{$pri}->{$node} = 1;
	$group_members->{$node} = $pri;
    }

    
    # add non-group members to unrestricted groups (priority -1)
    if (!$group->{restricted}) {
	my $pri = -1;
	foreach my $node (keys %$online_node_usage) {
	    next if defined($group_members->{$node});
	    $pri_groups->{$pri}->{$node} = 1;
	    $group_members->{$node} = -1;
	}
    }


    my @pri_list = sort {$b <=> $a} keys %$pri_groups;
    return undef if !scalar(@pri_list);
    
    if (!$try_next && $group->{nofailback} && defined($group_members->{$current_node})) {
	return $current_node;
    }

    # select node from top priority node list

    my $top_pri = $pri_list[0];

    my @nodes = sort { 
	$online_node_usage->{$a} <=> $online_node_usage->{$b} || $a cmp $b
    } keys %{$pri_groups->{$top_pri}};

    my $found;
    for (my $i = scalar(@nodes) - 1; $i >= 0; $i--) {
	my $node = $nodes[$i];
	if ($node eq $current_node) {
	    $found = $i;
	    last;
	}
    }

    if ($try_next) {

	if (defined($found) && ($found < (scalar(@nodes) - 1))) {
	    return $nodes[$found + 1];
	} else {
	    return $nodes[0];
	}

    } else {

	return $nodes[$found] if defined($found);

	return $nodes[0];

    }
}

my $uid_counter = 0;

sub compute_new_uuid {
    my ($state) = @_;
    
    $uid_counter++;
    return md5_base64($state . $$ . time() . $uid_counter);
}

my $valid_service_states = {
    stopped => 1,
    request_stop => 1,
    started => 1,
    fence => 1,
    migrate => 1,
    relocate => 1,
    error => 1,
};

sub recompute_online_node_usage {
    my ($self) = @_;

    my $online_node_usage = {};

    my $online_nodes = $self->{ns}->list_online_nodes();

    foreach my $node (@$online_nodes) {
	$online_node_usage->{$node} = 0;
    }

    foreach my $sid (keys %{$self->{ss}}) {
	my $sd = $self->{ss}->{$sid};
	my $state = $sd->{state};
	if (defined($online_node_usage->{$sd->{node}})) {
	    if (($state eq 'started') || ($state eq 'request_stop') || 
		($state eq 'fence') || ($state eq 'error')) {
		$online_node_usage->{$sd->{node}}++;
	    } elsif (($state eq 'migrate') || ($state eq 'relocate')) {
		$online_node_usage->{$sd->{target}}++;
	    } elsif ($state eq 'stopped') {
		# do nothing
	    } else {
		die "should not be reached";
	    }
	}
    }

    $self->{online_node_usage} = $online_node_usage;
}

my $change_service_state = sub {
    my ($self, $sid, $new_state, %params) = @_;

    my ($haenv, $ss) = ($self->{haenv}, $self->{ss});

    my $sd = $ss->{$sid} || die "no such service '$sid";

    my $old_state = $sd->{state};
    my $old_node = $sd->{node};

    die "no state change" if $old_state eq $new_state; # just to be sure

    die "invalid CRM service state '$new_state'\n" if !$valid_service_states->{$new_state};

    foreach my $k (keys %$sd) { delete $sd->{$k}; };

    $sd->{state} = $new_state;
    $sd->{node} = $old_node;

    my $text_state = '';
    foreach my $k (keys %params) {
	my $v = $params{$k};
	$text_state .= ", " if $text_state;
	$text_state .= "$k = $v";
	$sd->{$k} = $v;
    }

    $self->recompute_online_node_usage();

    $sd->{uid} = compute_new_uuid($new_state);
    

    $text_state = " ($text_state)" if $text_state;
    $haenv->log('info', "service '$sid': state changed from '${old_state}' to '${new_state}' $text_state");
};

# read LRM status for all active nodes 
sub read_lrm_status {
    my ($self) = @_;

    my $nodes = $self->{ns}->list_online_nodes();
    my $haenv = $self->{haenv};

    my $res = {};

    foreach my $node (@$nodes) {
	my $ls = $haenv->read_lrm_status($node);
	foreach my $uid (keys %$ls) {
	    next if $res->{$uid}; # should not happen
	    $res->{$uid} = $ls->{$uid};
	}
    }

    return $res;
}

# read new crm commands and save them into crm master status
sub update_crm_commands {
    my ($self) = @_;

    my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});

    my $cmdlist = $haenv->read_crm_commands();
    
    foreach my $cmd (split(/\n/, $cmdlist)) {
	chomp $cmd;

	if ($cmd =~ m/^(migrate|relocate)\s+(\S+)\s+(\S+)$/) {
	    my ($task, $sid, $node) = ($1, $2, $3); 
	    if (my $sd = $ss->{$sid}) {
		if (!$ns->node_is_online($node)) {
		    $haenv->log('err', "crm command error - node not online: $cmd");
		} else {
		    if ($node eq $sd->{node}) {
			$haenv->log('info', "ignore crm command - service already on target node: $cmd");
		    } else { 
			$haenv->log('info', "got crm command: $cmd");
			$ss->{$sid}->{cmd} = [ $task, $node];
		    }
		}
	    } else {
		$haenv->log('err', "crm command error - no such service: $cmd");
	    }

	} else {
	    $haenv->log('err', "unable to parse crm command: $cmd");
	}
    }

}

sub manage {
    my ($self) = @_;

    my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});

    $ns->update($haenv->get_node_info());

    if (!$ns->node_is_online($haenv->nodename())) {
	$haenv->log('info', "master seems offline");
	return;
    }

    my $lrm_status = $self->read_lrm_status();

    my $sc = $haenv->read_service_config();

    $self->{groups} = $haenv->read_group_config(); # update

    # compute new service status

    # add new service
    foreach my $sid (keys %$sc) {
	next if $ss->{$sid}; # already there
	$haenv->log('info', "adding new service '$sid'");
	# assume we are running to avoid relocate running service at add
	$ss->{$sid} = { state => 'started', node => $sc->{$sid}->{node},
			uid => compute_new_uuid('started') };
    }

    # remove stale service from manager state
    foreach my $sid (keys %$ss) {
	next if $sc->{$sid};
	$haenv->log('info', "removing stale service '$sid' (no config)");
	delete $ss->{$sid};
    }
    
    $self->update_crm_commands();

    for (;;) {
	my $repeat = 0;
	
	$self->recompute_online_node_usage();

	foreach my $sid (keys %$ss) {
	    my $sd = $ss->{$sid};
	    my $cd = $sc->{$sid} || { state => 'disabled' };

	    my $lrm_res = $sd->{uid} ? $lrm_status->{$sd->{uid}} : undef;

	    my $last_state = $sd->{state};

	    if ($last_state eq 'stopped') {

		$self->next_state_stopped($sid, $cd, $sd, $lrm_res);

	    } elsif ($last_state eq 'started') {

		$self->next_state_started($sid, $cd, $sd, $lrm_res);

	    } elsif ($last_state eq 'migrate' || $last_state eq 'relocate') {

		$self->next_state_migrate_relocate($sid, $cd, $sd, $lrm_res);

	    } elsif ($last_state eq 'fence') {

		# do nothing here - wait until fenced

	    } elsif ($last_state eq 'request_stop') {

		$self->next_state_request_stop($sid, $cd, $sd, $lrm_res);

	    } elsif ($last_state eq 'error') {

		# fixme: 

	    } else {

		die "unknown service state '$last_state'";
	    }

	    $repeat = 1 if $sd->{state} ne $last_state;
	}

	# handle fencing
	my $fenced_nodes = {};
	foreach my $sid (keys %$ss) {
	    my $sd = $ss->{$sid};
	    next if $sd->{state} ne 'fence';

	    if (!defined($fenced_nodes->{$sd->{node}})) {
		$fenced_nodes->{$sd->{node}} = $ns->fence_node($sd->{node}) || 0;
	    }

	    next if !$fenced_nodes->{$sd->{node}};

	    # node fence was sucessful - mark service as stopped
	    &$change_service_state($self, $sid, 'stopped');	    
	}

	last if !$repeat;
    }

    $self->flush_master_status();
}

# functions to compute next service states
# $cd: service configuration data (read only)
# $sd: service status data (read only)
#
# Note: use change_service_state() to alter state
#

sub next_state_request_stop {
    my ($self, $sid, $cd, $sd, $lrm_res) = @_;

    my $haenv = $self->{haenv};
    my $ns = $self->{ns};

    # check result from LRM daemon
    if ($lrm_res) {
	my $exit_code = $lrm_res->{exit_code};
	if ($exit_code == 0) {
	    &$change_service_state($self, $sid, 'stopped');
	    return;
	} else {
	    &$change_service_state($self, $sid, 'error'); # fixme: what state?
	    return;
	}
    }

    if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
	&$change_service_state($self, $sid, 'fence');
	return;
    }
}

sub next_state_migrate_relocate {
    my ($self, $sid, $cd, $sd, $lrm_res) = @_;

    my $haenv = $self->{haenv};
    my $ns = $self->{ns};

    # check result from LRM daemon
    if ($lrm_res) {
	my $exit_code = $lrm_res->{exit_code};
	if ($exit_code == 0) {
	    &$change_service_state($self, $sid, 'started', node => $sd->{target});
	    return;
	} else {
	    $haenv->log('err', "service '$sid' - migration failed (exit code $exit_code)");
	    &$change_service_state($self, $sid, 'started', node => $sd->{node});
	    return;
	}
    }

    if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
	&$change_service_state($self, $sid, 'fence');
	return;
    }
}


sub next_state_stopped {
    my ($self, $sid, $cd, $sd, $lrm_res) = @_;

    my $haenv = $self->{haenv};
    my $ns = $self->{ns};

    if ($sd->{node} ne $cd->{node}) {
	# this can happen if we fence a node with active migrations
	# hack: modify $sd (normally this should be considered read-only)
	$haenv->log('info', "fixup service '$sid' location ($sd->{node} => $cd->{node}");
	$sd->{node} = $cd->{node}; 
    }

    if ($sd->{cmd}) {
	my ($cmd, $target) = @{$sd->{cmd}};
	delete $sd->{cmd};

	if ($cmd eq 'migrate' || $cmd eq 'relocate') {
	    if (!$ns->node_is_online($target)) {
		$haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
	    } elsif ($sd->{node} eq $target) {
		$haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
	    } else {
		$haenv->change_service_location($sid, $target);
		$cd->{node} = $sd->{node} = $target; # fixme: $sd is read-only??!!	    
		$haenv->log('info', "$cmd service '$sid' to node '$target' (stopped)");
	    }
	} else {
	    $haenv->log('err', "unknown command '$cmd' for service '$sid'"); 
	}
    } 

    if ($cd->{state} eq 'disabled') {
	# do nothing
	return;
    } 

    if ($cd->{state} eq 'enabled') {
	if (my $node = select_service_node($self->{groups}, $self->{online_node_usage}, $cd, $sd->{node})) {
	    if ($node && ($sd->{node} ne $node)) {
		$haenv->change_service_location($sid, $node);
	    }
	    &$change_service_state($self, $sid, 'started', node => $node);
	} else {
	    # fixme: warn 
	}

	return;
    }

    $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
}

sub next_state_started {
    my ($self, $sid, $cd, $sd, $lrm_res) = @_;

    my $haenv = $self->{haenv};
    my $ns = $self->{ns};

    if (!$ns->node_is_online($sd->{node})) {
	if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
	    &$change_service_state($self, $sid, 'fence');
	}
	return;
    }
	
    if ($cd->{state} eq 'disabled') {
	&$change_service_state($self, $sid, 'request_stop');
	return;
    }

    if ($cd->{state} eq 'enabled') {

	if ($sd->{cmd}) {
	    my ($cmd, $target) = @{$sd->{cmd}};
	    delete $sd->{cmd};

	    if ($cmd eq 'migrate' || $cmd eq 'relocate') {
		if (!$ns->node_is_online($target)) {
		    $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
		} elsif ($sd->{node} eq $target) {
		    $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
		} else {
		    $haenv->log('info', "$cmd service '$sid' to node '$target' (running)");
		    &$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
		}
	    } else {
		$haenv->log('err', "unknown command '$cmd' for service '$sid'"); 
	    }
	} else {

	    my $try_next = 0;
	    if ($lrm_res && ($lrm_res->{exit_code} != 0)) { # fixme: other exit codes?
		$try_next = 1;
	    }

	    my $node = select_service_node($self->{groups}, $self->{online_node_usage}, 
					   $cd, $sd->{node}, $try_next);

	    if ($node && ($sd->{node} ne $node)) {
		$haenv->log('info', "migrate service '$sid' to node '$node' (running)");
		&$change_service_state($self, $sid, 'migrate', node => $sd->{node}, target => $node);
	    } else {
		# do nothing
	    }
	}

	return;
    } 

    $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
}

1;
Commit	Line	Data
c0bbd038 DM	1	package PVE::HA::Manager;
	2
	3	use strict;
	4	use warnings;
c4a221bc	5	use Digest::MD5 qw(md5_base64);
c0bbd038 DM	6
c0bbd038 DM	7	use Data::Dumper;
c142ebc9	8	use PVE::Tools;
c0bbd038 DM	9	use PVE::HA::NodeStatus;
c0bbd038 DM	10
5385a606 DM	11	my $fence_delay = 30;
5385a606 DM	12
c0bbd038	13	sub new {
8f0bb968	14	my ($this, $haenv) = @_;
c0bbd038 DM	15
	16	my $class = ref($this) \|\| $this;
	17
8f0bb968 DM	18	my $ms = $haenv->read_manager_status();
	19
	20	$ms->{master_node} = $haenv->nodename();
	21
	22	my $ns = PVE::HA::NodeStatus->new($haenv, $ms->{node_status} \|\| {});
	23
59fd7207 DM	24	# fixme: use separate class PVE::HA::ServiceStatus
	25	my $ss = $ms->{service_status} \|\| {};
	26
c0bbd038	27	my $self = bless {
8f0bb968 DM	28	haenv => $haenv,
	29	ms => $ms, # master status
	30	ns => $ns, # PVE::HA::NodeStatus
59fd7207	31	ss => $ss, # service status
c0bbd038 DM	32	}, $class;
	33
	34	return $self;
	35	}
	36
d84da043 DM	37	sub cleanup {
	38	my ($self) = @_;
	39
	40	# todo: ?
	41	}
	42
8f0bb968	43	sub flush_master_status {
c0bbd038 DM	44	my ($self) = @_;
c0bbd038 DM	45
59fd7207	46	my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
c0bbd038	47
8f0bb968	48	$ms->{node_status} = $ns->{status};
59fd7207 DM	49	$ms->{service_status} = $ss;
59fd7207 DM	50
8f0bb968 DM	51	$haenv->write_manager_status($ms);
8f0bb968 DM	52	}
c0bbd038	53
f7ccd1b3	54	sub select_service_node {
270d4406	55	my ($groups, $online_node_usage, $service_conf, $current_node, $try_next) = @_;
f7ccd1b3	56
abc920b4 DM	57	my $group = { 'nodes' => $service_conf->{node} }; # default group
abc920b4 DM	58
270d4406 DM	59	$group = $groups->{ids}->{$service_conf->{group}} if $service_conf->{group} &&
270d4406 DM	60	$groups->{ids}->{$service_conf->{group}};
abc920b4 DM	61
	62	my $pri_groups = {};
	63	my $group_members = {};
	64	foreach my $entry (PVE::Tools::split_list($group->{nodes})) {
	65	my ($node, $pri) = ($entry, 0);
	66	if ($entry =~ m/^(\S+):(\d+)$/) {
	67	($node, $pri) = ($1, $2);
	68	}
270d4406	69	next if !defined($online_node_usage->{$node}); # offline
abc920b4 DM	70	$pri_groups->{$pri}->{$node} = 1;
	71	$group_members->{$node} = $pri;
	72	}
f7ccd1b3	73
270d4406	74
abc920b4 DM	75	# add non-group members to unrestricted groups (priority -1)
	76	if (!$group->{restricted}) {
	77	my $pri = -1;
270d4406	78	foreach my $node (keys %$online_node_usage) {
abc920b4 DM	79	next if defined($group_members->{$node});
	80	$pri_groups->{$pri}->{$node} = 1;
	81	$group_members->{$node} = -1;
	82	}
	83	}
	84
270d4406	85
abc920b4 DM	86	my @pri_list = sort {$b <=> $a} keys %$pri_groups;
abc920b4 DM	87	return undef if !scalar(@pri_list);
270d4406	88
abc920b4 DM	89	if (!$try_next && $group->{nofailback} && defined($group_members->{$current_node})) {
	90	return $current_node;
	91	}
	92
	93	# select node from top priority node list
	94
	95	my $top_pri = $pri_list[0];
	96
c142ebc9 DM	97	my @nodes = sort {
	98	$online_node_usage->{$a} <=> $online_node_usage->{$b} \|\| $a cmp $b
	99	} keys %{$pri_groups->{$top_pri}};
abc920b4 DM	100
	101	my $found;
	102	for (my $i = scalar(@nodes) - 1; $i >= 0; $i--) {
	103	my $node = $nodes[$i];
	104	if ($node eq $current_node) {
	105	$found = $i;
	106	last;
	107	}
	108	}
	109
abc920b4 DM	110	if ($try_next) {
	111
	112	if (defined($found) && ($found < (scalar(@nodes) - 1))) {
	113	return $nodes[$found + 1];
	114	} else {
	115	return $nodes[0];
	116	}
	117
	118	} else {
	119
	120	return $nodes[$found] if defined($found);
	121
	122	return $nodes[0];
	123
	124	}
f7ccd1b3 DM	125	}
f7ccd1b3 DM	126
c4a221bc DM	127	my $uid_counter = 0;
c4a221bc DM	128
d55aa611 DM	129	sub compute_new_uuid {
	130	my ($state) = @_;
	131
	132	$uid_counter++;
	133	return md5_base64($state . $$ . time() . $uid_counter);
	134	}
	135
618fbeda DM	136	my $valid_service_states = {
	137	stopped => 1,
	138	request_stop => 1,
	139	started => 1,
	140	fence => 1,
	141	migrate => 1,
b0fdf86a	142	relocate => 1,
618fbeda DM	143	error => 1,
	144	};
	145
270d4406 DM	146	sub recompute_online_node_usage {
	147	my ($self) = @_;
	148
	149	my $online_node_usage = {};
	150
	151	my $online_nodes = $self->{ns}->list_online_nodes();
	152
	153	foreach my $node (@$online_nodes) {
	154	$online_node_usage->{$node} = 0;
	155	}
	156
	157	foreach my $sid (keys %{$self->{ss}}) {
	158	my $sd = $self->{ss}->{$sid};
	159	my $state = $sd->{state};
	160	if (defined($online_node_usage->{$sd->{node}})) {
	161	if (($state eq 'started') \|\| ($state eq 'request_stop') \|\|
	162	($state eq 'fence') \|\| ($state eq 'error')) {
	163	$online_node_usage->{$sd->{node}}++;
	164	} elsif (($state eq 'migrate') \|\| ($state eq 'relocate')) {
	165	$online_node_usage->{$sd->{target}}++;
	166	} elsif ($state eq 'stopped') {
	167	# do nothing
	168	} else {
	169	die "should not be reached";
	170	}
	171	}
	172	}
	173
	174	$self->{online_node_usage} = $online_node_usage;
	175	}
	176
4e01bc86 DM	177	my $change_service_state = sub {
	178	my ($self, $sid, $new_state, %params) = @_;
	179
	180	my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
	181
	182	my $sd = $ss->{$sid} \|\| die "no such service '$sid";
	183
	184	my $old_state = $sd->{state};
e4ffb299	185	my $old_node = $sd->{node};
4e01bc86 DM	186
	187	die "no state change" if $old_state eq $new_state; # just to be sure
	188
618fbeda DM	189	die "invalid CRM service state '$new_state'\n" if !$valid_service_states->{$new_state};
618fbeda DM	190
e4ffb299 DM	191	foreach my $k (keys %$sd) { delete $sd->{$k}; };
	192
	193	$sd->{state} = $new_state;
	194	$sd->{node} = $old_node;
	195
	196	my $text_state = '';
4e01bc86 DM	197	foreach my $k (keys %params) {
4e01bc86 DM	198	my $v = $params{$k};
e4ffb299 DM	199	$text_state .= ", " if $text_state;
e4ffb299 DM	200	$text_state .= "$k = $v";
4e01bc86 DM	201	$sd->{$k} = $v;
4e01bc86 DM	202	}
270d4406 DM	203
	204	$self->recompute_online_node_usage();
	205
d55aa611 DM	206	$sd->{uid} = compute_new_uuid($new_state);
d55aa611 DM	207
4e01bc86	208
e4ffb299	209	$text_state = " ($text_state)" if $text_state;
e5986717	210	$haenv->log('info', "service '$sid': state changed from '${old_state}' to '${new_state}' $text_state");
4e01bc86 DM	211	};
4e01bc86 DM	212
332170bd	213	# read LRM status for all active nodes
c4a221bc	214	sub read_lrm_status {
332170bd	215	my ($self) = @_;
c4a221bc	216
332170bd	217	my $nodes = $self->{ns}->list_online_nodes();
c4a221bc DM	218	my $haenv = $self->{haenv};
	219
	220	my $res = {};
	221
332170bd	222	foreach my $node (@$nodes) {
c4a221bc DM	223	my $ls = $haenv->read_lrm_status($node);
	224	foreach my $uid (keys %$ls) {
	225	next if $res->{$uid}; # should not happen
	226	$res->{$uid} = $ls->{$uid};
	227	}
	228	}
	229
	230	return $res;
	231	}
	232
aa98a844 DM	233	# read new crm commands and save them into crm master status
	234	sub update_crm_commands {
	235	my ($self) = @_;
	236
	237	my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
	238
	239	my $cmdlist = $haenv->read_crm_commands();
	240
	241	foreach my $cmd (split(/\n/, $cmdlist)) {
	242	chomp $cmd;
	243
b0fdf86a DM	244	if ($cmd =~ m/^(migrate\|relocate)\s+(\S+)\s+(\S+)$/) {
b0fdf86a DM	245	my ($task, $sid, $node) = ($1, $2, $3);
aa98a844 DM	246	if (my $sd = $ss->{$sid}) {
	247	if (!$ns->node_is_online($node)) {
	248	$haenv->log('err', "crm command error - node not online: $cmd");
	249	} else {
	250	if ($node eq $sd->{node}) {
	251	$haenv->log('info', "ignore crm command - service already on target node: $cmd");
	252	} else {
	253	$haenv->log('info', "got crm command: $cmd");
b0fdf86a	254	$ss->{$sid}->{cmd} = [ $task, $node];
aa98a844 DM	255	}
	256	}
	257	} else {
	258	$haenv->log('err', "crm command error - no such service: $cmd");
	259	}
	260
	261	} else {
	262	$haenv->log('err', "unable to parse crm command: $cmd");
	263	}
	264	}
	265
	266	}
	267
8f0bb968 DM	268	sub manage {
8f0bb968 DM	269	my ($self) = @_;
c0bbd038	270
59fd7207	271	my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
c0bbd038	272
332170bd	273	$ns->update($haenv->get_node_info());
c79442f2 DM	274
c79442f2 DM	275	if (!$ns->node_is_online($haenv->nodename())) {
e5986717	276	$haenv->log('info', "master seems offline");
c79442f2 DM	277	return;
	278	}
	279
332170bd	280	my $lrm_status = $self->read_lrm_status();
c4a221bc	281
f7ccd1b3 DM	282	my $sc = $haenv->read_service_config();
f7ccd1b3 DM	283
abc920b4 DM	284	$self->{groups} = $haenv->read_group_config(); # update
abc920b4 DM	285
f7ccd1b3 DM	286	# compute new service status
	287
	288	# add new service
	289	foreach my $sid (keys %$sc) {
	290	next if $ss->{$sid}; # already there
e5986717	291	$haenv->log('info', "adding new service '$sid'");
f7ccd1b3	292	# assume we are running to avoid relocate running service at add
d55aa611 DM	293	$ss->{$sid} = { state => 'started', node => $sc->{$sid}->{node},
d55aa611 DM	294	uid => compute_new_uuid('started') };
f7ccd1b3 DM	295	}
f7ccd1b3 DM	296
4e5764af DM	297	# remove stale service from manager state
	298	foreach my $sid (keys %$ss) {
	299	next if $sc->{$sid};
	300	$haenv->log('info', "removing stale service '$sid' (no config)");
	301	delete $ss->{$sid};
	302	}
	303
aa98a844 DM	304	$self->update_crm_commands();
aa98a844 DM	305
c79442f2 DM	306	for (;;) {
c79442f2 DM	307	my $repeat = 0;
270d4406 DM	308
270d4406 DM	309	$self->recompute_online_node_usage();
f7ccd1b3	310
c79442f2 DM	311	foreach my $sid (keys %$ss) {
	312	my $sd = $ss->{$sid};
	313	my $cd = $sc->{$sid} \|\| { state => 'disabled' };
f7ccd1b3	314
a875fbe8 DM	315	my $lrm_res = $sd->{uid} ? $lrm_status->{$sd->{uid}} : undef;
a875fbe8 DM	316
c79442f2 DM	317	my $last_state = $sd->{state};
	318
	319	if ($last_state eq 'stopped') {
	320
abc920b4	321	$self->next_state_stopped($sid, $cd, $sd, $lrm_res);
f7ccd1b3	322
c79442f2	323	} elsif ($last_state eq 'started') {
f7ccd1b3	324
abc920b4	325	$self->next_state_started($sid, $cd, $sd, $lrm_res);
f7ccd1b3	326
b0fdf86a	327	} elsif ($last_state eq 'migrate' \|\| $last_state eq 'relocate') {
f7ccd1b3	328
8aaa0e36	329	$self->next_state_migrate_relocate($sid, $cd, $sd, $lrm_res);
f7ccd1b3	330
c79442f2	331	} elsif ($last_state eq 'fence') {
f7ccd1b3	332
21e37ed4	333	# do nothing here - wait until fenced
f7ccd1b3	334
c79442f2	335	} elsif ($last_state eq 'request_stop') {
f7ccd1b3	336
0df5b3dd	337	$self->next_state_request_stop($sid, $cd, $sd, $lrm_res);
618fbeda	338
e88469ba DM	339	} elsif ($last_state eq 'error') {
	340
	341	# fixme:
	342
a875fbe8 DM	343	} else {
	344
	345	die "unknown service state '$last_state'";
618fbeda	346	}
21e37ed4	347
c79442f2	348	$repeat = 1 if $sd->{state} ne $last_state;
f7ccd1b3 DM	349	}
f7ccd1b3 DM	350
21e37ed4 DM	351	# handle fencing
	352	my $fenced_nodes = {};
	353	foreach my $sid (keys %$ss) {
	354	my $sd = $ss->{$sid};
	355	next if $sd->{state} ne 'fence';
	356
	357	if (!defined($fenced_nodes->{$sd->{node}})) {
	358	$fenced_nodes->{$sd->{node}} = $ns->fence_node($sd->{node}) \|\| 0;
	359	}
	360
	361	next if !$fenced_nodes->{$sd->{node}};
	362
	363	# node fence was sucessful - mark service as stopped
	364	&$change_service_state($self, $sid, 'stopped');
	365	}
	366
c79442f2	367	last if !$repeat;
f7ccd1b3	368	}
f7ccd1b3	369
8f0bb968	370	$self->flush_master_status();
c0bbd038 DM	371	}
c0bbd038 DM	372
a875fbe8 DM	373	# functions to compute next service states
	374	# $cd: service configuration data (read only)
	375	# $sd: service status data (read only)
	376	#
	377	# Note: use change_service_state() to alter state
	378	#
	379
0df5b3dd DM	380	sub next_state_request_stop {
	381	my ($self, $sid, $cd, $sd, $lrm_res) = @_;
	382
	383	my $haenv = $self->{haenv};
	384	my $ns = $self->{ns};
	385
	386	# check result from LRM daemon
	387	if ($lrm_res) {
	388	my $exit_code = $lrm_res->{exit_code};
	389	if ($exit_code == 0) {
	390	&$change_service_state($self, $sid, 'stopped');
	391	return;
	392	} else {
	393	&$change_service_state($self, $sid, 'error'); # fixme: what state?
	394	return;
	395	}
	396	}
	397
5385a606	398	if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
0df5b3dd DM	399	&$change_service_state($self, $sid, 'fence');
	400	return;
	401	}
	402	}
	403
8aaa0e36 DM	404	sub next_state_migrate_relocate {
	405	my ($self, $sid, $cd, $sd, $lrm_res) = @_;
	406
	407	my $haenv = $self->{haenv};
	408	my $ns = $self->{ns};
	409
	410	# check result from LRM daemon
	411	if ($lrm_res) {
	412	my $exit_code = $lrm_res->{exit_code};
	413	if ($exit_code == 0) {
	414	&$change_service_state($self, $sid, 'started', node => $sd->{target});
	415	return;
	416	} else {
	417	$haenv->log('err', "service '$sid' - migration failed (exit code $exit_code)");
	418	&$change_service_state($self, $sid, 'started', node => $sd->{node});
	419	return;
	420	}
	421	}
	422
5385a606	423	if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
8aaa0e36 DM	424	&$change_service_state($self, $sid, 'fence');
	425	return;
	426	}
	427	}
	428
	429
a875fbe8	430	sub next_state_stopped {
abc920b4	431	my ($self, $sid, $cd, $sd, $lrm_res) = @_;
a875fbe8 DM	432
a875fbe8 DM	433	my $haenv = $self->{haenv};
e88469ba	434	my $ns = $self->{ns};
a875fbe8	435
ff6f1c5c DM	436	if ($sd->{node} ne $cd->{node}) {
	437	# this can happen if we fence a node with active migrations
	438	# hack: modify $sd (normally this should be considered read-only)
	439	$haenv->log('info', "fixup service '$sid' location ($sd->{node} => $cd->{node}");
	440	$sd->{node} = $cd->{node};
	441	}
	442
94b7ebe2 DM	443	if ($sd->{cmd}) {
	444	my ($cmd, $target) = @{$sd->{cmd}};
	445	delete $sd->{cmd};
	446
b0fdf86a	447	if ($cmd eq 'migrate' \|\| $cmd eq 'relocate') {
94b7ebe2	448	if (!$ns->node_is_online($target)) {
b0fdf86a	449	$haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
e88469ba	450	} elsif ($sd->{node} eq $target) {
b0fdf86a	451	$haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
94b7ebe2 DM	452	} else {
	453	$haenv->change_service_location($sid, $target);
	454	$cd->{node} = $sd->{node} = $target; # fixme: $sd is read-only??!!
b0fdf86a	455	$haenv->log('info', "$cmd service '$sid' to node '$target' (stopped)");
94b7ebe2 DM	456	}
	457	} else {
	458	$haenv->log('err', "unknown command '$cmd' for service '$sid'");
	459	}
	460	}
	461
a875fbe8 DM	462	if ($cd->{state} eq 'disabled') {
a875fbe8 DM	463	# do nothing
e88469ba DM	464	return;
	465	}
	466
	467	if ($cd->{state} eq 'enabled') {
270d4406	468	if (my $node = select_service_node($self->{groups}, $self->{online_node_usage}, $cd, $sd->{node})) {
a875fbe8 DM	469	if ($node && ($sd->{node} ne $node)) {
	470	$haenv->change_service_location($sid, $node);
	471	}
	472	&$change_service_state($self, $sid, 'started', node => $node);
	473	} else {
	474	# fixme: warn
	475	}
e88469ba DM	476
e88469ba DM	477	return;
a875fbe8	478	}
e88469ba DM	479
e88469ba DM	480	$haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
a875fbe8 DM	481	}
	482
	483	sub next_state_started {
abc920b4	484	my ($self, $sid, $cd, $sd, $lrm_res) = @_;
a875fbe8 DM	485
	486	my $haenv = $self->{haenv};
	487	my $ns = $self->{ns};
	488
	489	if (!$ns->node_is_online($sd->{node})) {
5385a606 DM	490	if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
	491	&$change_service_state($self, $sid, 'fence');
	492	}
e88469ba DM	493	return;
e88469ba DM	494	}
a875fbe8	495
e88469ba DM	496	if ($cd->{state} eq 'disabled') {
	497	&$change_service_state($self, $sid, 'request_stop');
	498	return;
	499	}
	500
	501	if ($cd->{state} eq 'enabled') {
e88469ba DM	502
	503	if ($sd->{cmd}) {
	504	my ($cmd, $target) = @{$sd->{cmd}};
	505	delete $sd->{cmd};
	506
b0fdf86a	507	if ($cmd eq 'migrate' \|\| $cmd eq 'relocate') {
e88469ba	508	if (!$ns->node_is_online($target)) {
b0fdf86a	509	$haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
e88469ba	510	} elsif ($sd->{node} eq $target) {
b0fdf86a	511	$haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
e88469ba	512	} else {
b0fdf86a DM	513	$haenv->log('info', "$cmd service '$sid' to node '$target' (running)");
b0fdf86a DM	514	&$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
e88469ba	515	}
a875fbe8	516	} else {
e88469ba	517	$haenv->log('err', "unknown command '$cmd' for service '$sid'");
a875fbe8 DM	518	}
a875fbe8 DM	519	} else {
b0fdf86a	520
abc920b4 DM	521	my $try_next = 0;
	522	if ($lrm_res && ($lrm_res->{exit_code} != 0)) { # fixme: other exit codes?
	523	$try_next = 1;
	524	}
	525
270d4406 DM	526	my $node = select_service_node($self->{groups}, $self->{online_node_usage},
270d4406 DM	527	$cd, $sd->{node}, $try_next);
abc920b4	528
b0fdf86a DM	529	if ($node && ($sd->{node} ne $node)) {
	530	$haenv->log('info', "migrate service '$sid' to node '$node' (running)");
	531	&$change_service_state($self, $sid, 'migrate', node => $sd->{node}, target => $node);
	532	} else {
	533	# do nothing
	534	}
a875fbe8	535	}
e88469ba DM	536
	537	return;
	538	}
	539
	540	$haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
a875fbe8	541	}
c0bbd038 DM	542
c0bbd038 DM	543	1;