[pve-ha-manager.git] / src / PVE / HA / LRM.pm

package PVE::HA::LRM;

# Local Resource Manager

use strict;
use warnings;
use Data::Dumper;
use POSIX qw(:sys_wait_h);

use PVE::SafeSyslog;
use PVE::Tools;
use PVE::HA::Tools ':exit_codes';
use PVE::HA::Resources;

# Server can have several states:

my $valid_states = {
    wait_for_agent_lock => "waiting for agent lock",
    active => "got agent_lock",
    lost_agent_lock => "lost agent_lock",
};

sub new {
    my ($this, $haenv) = @_;

    my $class = ref($this) || $this;

    my $self = bless {
	haenv => $haenv,
	status => { state => 'startup' },
	workers => {},
	results => {},
	restart_tries => {},
	shutdown_request => 0,
	shutdown_errors => 0,
	# mode can be: active, reboot, shutdown, restart
	mode => 'active',
    }, $class;

    $self->set_local_status({ state => 	'wait_for_agent_lock' });   

    return $self;
}

sub shutdown_request {
    my ($self) = @_;

    return if $self->{shutdown_request}; # already in shutdown mode

    my $haenv = $self->{haenv};

    my $nodename = $haenv->nodename();

    my $shutdown = $haenv->is_node_shutdown();

    if ($shutdown) {
	$haenv->log('info', "shutdown LRM, stop all services");
	$self->{mode} = 'shutdown';

	# queue stop jobs for all services

	my $ss = $self->{service_status};

	foreach my $sid (keys %$ss) {
	    my $sd = $ss->{$sid};
	    next if !$sd->{node};
	    next if $sd->{node} ne $nodename;
	    # Note: use undef uid to mark shutdown/stop jobs
	    $self->queue_resource_command($sid, undef, 'request_stop');
	}

    } else {
	$haenv->log('info', "restart LRM, freeze all services");
	$self->{mode} = 'restart';
    }

    $self->{shutdown_request} = 1;

    eval { $self->update_lrm_status(); };
    if (my $err = $@) {
	$self->log('err', "unable to update lrm status file - $err");
    }
}

sub get_local_status {
    my ($self) = @_;

    return $self->{status};
}

sub set_local_status {
    my ($self, $new) = @_;

    die "invalid state '$new->{state}'" if !$valid_states->{$new->{state}};

    my $haenv = $self->{haenv};

    my $old = $self->{status};

    # important: only update if if really changed 
    return if $old->{state} eq $new->{state};

    $haenv->log('info', "status change $old->{state} => $new->{state}");

    $new->{state_change_time} = $haenv->get_time();

    $self->{status} = $new;
}

sub update_lrm_status {
    my ($self) = @_;

    my $haenv = $self->{haenv};

    return 0 if !$haenv->quorate();
    
    my $lrm_status = {	
	state => $self->{status}->{state},
	mode => $self->{mode},
	results => $self->{results},
	timestamp => $haenv->get_time(),
    };
    
    eval { $haenv->write_lrm_status($lrm_status); };
    if (my $err = $@) {
	$haenv->log('err', "unable to write lrm status file - $err");
	return 0;
    }

    return 1;
}

sub get_protected_ha_agent_lock {
    my ($self) = @_;

    my $haenv = $self->{haenv};

    my $count = 0;
    my $starttime = $haenv->get_time();

    for (;;) {
	
	if ($haenv->get_ha_agent_lock()) {
	    if ($self->{ha_agent_wd}) {
		$haenv->watchdog_update($self->{ha_agent_wd});
	    } else {
		my $wfh = $haenv->watchdog_open();
		$self->{ha_agent_wd} = $wfh;
	    }
	    return 1;
	}
	    
	last if ++$count > 5; # try max 5 time

	my $delay = $haenv->get_time() - $starttime;
	last if $delay > 5; # for max 5 seconds

	$haenv->sleep(1);
    }
    
    return 0;
}

sub active_service_count {
    my ($self) = @_;
    
    my $haenv = $self->{haenv};

    my $nodename = $haenv->nodename();

    my $ss = $self->{service_status};

    my $count = 0;
    
    foreach my $sid (keys %$ss) {
	my $sd = $ss->{$sid};
	next if !$sd->{node};
	next if $sd->{node} ne $nodename;
	my $req_state = $sd->{state};
	next if !defined($req_state);
	next if $req_state eq 'stopped';
	next if $req_state eq 'freeze';

	$count++;
    }
    
    return $count;
}

my $wrote_lrm_status_at_startup = 0;

sub do_one_iteration {
    my ($self) = @_;

    my $haenv = $self->{haenv};

    if (!$wrote_lrm_status_at_startup) {
	if ($self->update_lrm_status()) {
	    $wrote_lrm_status_at_startup = 1;
	} else {
	    # do nothing
	    $haenv->sleep(5);
	    return $self->{shutdown_request} ? 0 : 1;
	}
    }
    
    my $status = $self->get_local_status();
    my $state = $status->{state};

    my $ms = $haenv->read_manager_status();
    $self->{service_status} =  $ms->{service_status} || {};

    my $fence_request = PVE::HA::Tools::count_fenced_services($self->{service_status}, $haenv->nodename());
    
    # do state changes first 

    my $ctime = $haenv->get_time();

    if ($state eq 'wait_for_agent_lock') {

	my $service_count = $self->active_service_count();

	if (!$fence_request && $service_count && $haenv->quorate()) {
	    if ($self->get_protected_ha_agent_lock()) {
		$self->set_local_status({ state => 'active' });
	    }
	}
	
    } elsif ($state eq 'lost_agent_lock') {

	if (!$fence_request && $haenv->quorate()) {
	    if ($self->get_protected_ha_agent_lock()) {
		$self->set_local_status({ state => 'active' });
	    }
	}

    } elsif ($state eq 'active') {

	if ($fence_request) {		
	    $haenv->log('err', "node need to be fenced - releasing agent_lock\n");
	    $self->set_local_status({ state => 'lost_agent_lock'});	
	} elsif (!$self->get_protected_ha_agent_lock()) {
	    $self->set_local_status({ state => 'lost_agent_lock'});
	}
    }

    $status = $self->get_local_status();
    $state = $status->{state};

    # do work

    if ($state eq 'wait_for_agent_lock') {

	return 0 if $self->{shutdown_request};
	
	$self->update_lrm_status();
	
	$haenv->sleep(5);
	   
    } elsif ($state eq 'active') {

	my $startime = $haenv->get_time();

	my $max_time = 10;

	my $shutdown = 0;

	# do work (max_time seconds)
	eval {
	    # fixme: set alert timer

	    if ($self->{shutdown_request}) {

		if ($self->{mode} eq 'restart') {

		    my $service_count = $self->active_service_count();

		    if ($service_count == 0) {

			if ($self->run_workers() == 0) {
			    if ($self->{ha_agent_wd}) {
				$haenv->watchdog_close($self->{ha_agent_wd});
				delete $self->{ha_agent_wd};
			    }

			    $shutdown = 1;

			    # restart with no or freezed services, release the lock
			    $haenv->release_ha_agent_lock();
			}
		    }
		} else {

		    if ($self->run_workers() == 0) {
			if ($self->{shutdown_errors} == 0) {
			    if ($self->{ha_agent_wd}) {
				$haenv->watchdog_close($self->{ha_agent_wd});
				delete $self->{ha_agent_wd};
			    }

			    # shutdown with all services stopped thus release the lock
			    $haenv->release_ha_agent_lock();
			}

			$shutdown = 1;
		    }
		}
	    } else {

		$self->manage_resources();

	    }
	};
	if (my $err = $@) {
	    $haenv->log('err', "got unexpected error - $err");
	}

	$self->update_lrm_status();
	
	return 0 if $shutdown;

	$haenv->sleep_until($startime + $max_time);

    } elsif ($state eq 'lost_agent_lock') {
	
	# Note: watchdog is active an will triger soon!

	# so we hope to get the lock back soon!

	if ($self->{shutdown_request}) {

	    my $service_count = $self->active_service_count();

	    if ($service_count > 0) {
		$haenv->log('err', "get shutdown request in state 'lost_agent_lock' - " . 
			    "detected $service_count running services");

	    } else {

		# all services are stopped, so we can close the watchdog

		if ($self->{ha_agent_wd}) {
		    $haenv->watchdog_close($self->{ha_agent_wd});
		    delete $self->{ha_agent_wd};
		}
		
		return 0;
	    }
	}

	$haenv->sleep(5);

    } else {

	die "got unexpected status '$state'\n";

    }

    return 1;
}

sub run_workers {
    my ($self) = @_;

    my $haenv = $self->{haenv};

    my $starttime = $haenv->get_time();

    # start workers
    my $max_workers = 4;

    my $sc = $haenv->read_service_config();

    while (($haenv->get_time() - $starttime) < 5) {
	my $count =  $self->check_active_workers();

	foreach my $sid (keys %{$self->{workers}}) {
	    last if $count >= $max_workers;
	    my $w = $self->{workers}->{$sid};
	    if (!$w->{pid}) {
		if ($haenv->can_fork()) {
		    my $pid = fork();
		    if (!defined($pid)) {
			$haenv->log('err', "fork worker failed");
			$count = 0; last; # abort, try later
		    } elsif ($pid == 0) {
			$haenv->after_fork(); # cleanup

			# do work
			my $res = -1;
			eval {
			    $res = $self->exec_resource_agent($sid, $sc->{$sid}, $w->{state}, $w->{target});
			};
			if (my $err = $@) {
			    $haenv->log('err', $err);
			    POSIX::_exit(-1);
			}  
			POSIX::_exit($res); 
		    } else {
			$count++;
			$w->{pid} = $pid;
		    }
		} else {
		    my $res = -1;
		    eval {
			$res = $self->exec_resource_agent($sid, $sc->{$sid}, $w->{state}, $w->{target});
			$res = $res << 8 if $res > 0;
		    };
		    if (my $err = $@) {
			$haenv->log('err', $err);
		    }
		    if (defined($w->{uid})) {
			$self->resource_command_finished($sid, $w->{uid}, $res);
		    } else {
			$self->stop_command_finished($sid, $res);
		    }
		}
	    }
	}

	last if !$count;

	$haenv->sleep(1);
    }

    return scalar(keys %{$self->{workers}});
}

sub manage_resources {
    my ($self) = @_;

    my $haenv = $self->{haenv};

    my $nodename = $haenv->nodename();

    my $ss = $self->{service_status};

    foreach my $sid (keys %$ss) {
	my $sd = $ss->{$sid};
	next if !$sd->{node};
	next if !$sd->{uid};
	next if $sd->{node} ne $nodename;
	my $req_state = $sd->{state};
	next if !defined($req_state);
	next if $req_state eq 'freeze';
	$self->queue_resource_command($sid, $sd->{uid}, $req_state, $sd->{target});
    }

    return $self->run_workers();
}

sub queue_resource_command {
    my ($self, $sid, $uid, $state, $target) = @_;

    if (my $w = $self->{workers}->{$sid}) {
	return if $w->{pid}; # already started
	# else, delete and overwrite queue entry with new command
	delete $self->{workers}->{$sid};
    }

    $self->{workers}->{$sid} = {
	sid => $sid,
	uid => $uid,
	state => $state,
    };

    $self->{workers}->{$sid}->{target} = $target if $target;
}

sub check_active_workers {
    my ($self) = @_;

    # finish/count workers
    my $count = 0;
    foreach my $sid (keys %{$self->{workers}}) {
	my $w = $self->{workers}->{$sid};
	if (my $pid = $w->{pid}) {
	    # check status
	    my $waitpid = waitpid($pid, WNOHANG);
	    if (defined($waitpid) && ($waitpid == $pid)) {
		if (defined($w->{uid})) {
		    $self->resource_command_finished($sid, $w->{uid}, $?);
		} else {
		    $self->stop_command_finished($sid, $?);
		}
	    } else {
		$count++;
	    }
	}
    }
    
    return $count;
}

sub stop_command_finished {
    my ($self, $sid, $status) = @_;

    my $haenv = $self->{haenv};

    my $w = delete $self->{workers}->{$sid};
    return if !$w; # should not happen

    my $exit_code = -1;

    if ($status == -1) {
	$haenv->log('err', "resource agent $sid finished - failed to execute");
    }  elsif (my $sig = ($status & 127)) {
	$haenv->log('err', "resource agent $sid finished - got signal $sig");
    } else {
	$exit_code = ($status >> 8);
    }

    if ($exit_code != 0) {
	$self->{shutdown_errors}++;
    }
}

sub resource_command_finished {
    my ($self, $sid, $uid, $status) = @_;

    my $haenv = $self->{haenv};

    my $w = delete $self->{workers}->{$sid};
    return if !$w; # should not happen

    my $exit_code = -1;

    if ($status == -1) {
	$haenv->log('err', "resource agent $sid finished - failed to execute");    
    }  elsif (my $sig = ($status & 127)) {
	$haenv->log('err', "resource agent $sid finished - got signal $sig");
    } else {
	$exit_code = ($status >> 8);
    }

    $exit_code = $self->handle_service_exitcode($sid, $w->{state}, $exit_code);

    $self->{results}->{$uid} = {
	sid => $w->{sid},
	state => $w->{state},
	exit_code => $exit_code,
    };

    my $ss = $self->{service_status};

    # compute hash of valid/existing uids
    my $valid_uids = {};
    foreach my $sid (keys %$ss) {
	my $sd = $ss->{$sid};
	next if !$sd->{uid};
	$valid_uids->{$sd->{uid}} = 1;
    }

    my $results = {};
    foreach my $id (keys %{$self->{results}}) {
	next if !$valid_uids->{$id};
	$results->{$id} = $self->{results}->{$id};
    }
    $self->{results} = $results;
}

# processes the exit code from a finished resource agent, so that the CRM knows
# if the LRM wants to retry an action based on the current recovery policies for
# the failed service, or the CRM itself must try to recover from the failure.
sub handle_service_exitcode {
    my ($self, $sid, $cmd, $exit_code) = @_;

    my $haenv = $self->{haenv};
    my $tries = $self->{restart_tries};

    my $sc = $haenv->read_service_config();

    my $max_restart = 0;

    if (my $cd = $sc->{$sid}) {
	$max_restart = $cd->{max_restart};
    }

    if ($cmd eq 'started') {

	if ($exit_code == SUCCESS) {

	    $tries->{$sid} = 0;

	    return $exit_code;

	} elsif ($exit_code == ERROR) {

	    $tries->{$sid} = 0 if !defined($tries->{$sid});

	    $tries->{$sid}++;
	    if ($tries->{$sid} >= $max_restart) {
		$haenv->log('err', "unable to start service $sid on local node".
			   " after $tries->{$sid} retries");
		$tries->{$sid} = 0;
		return ERROR;
	    }

	    # tell CRM that we retry the start
	    return ETRY_AGAIN;
	}
    }

    return $exit_code;

}

sub exec_resource_agent {
    my ($self, $sid, $service_config, $cmd, @params) = @_;

    # setup execution environment

    $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';

    my $haenv = $self->{haenv};

    my $nodename = $haenv->nodename();

    my (undef, $service_type, $service_name) = PVE::HA::Tools::parse_sid($sid);

    my $plugin = PVE::HA::Resources->lookup($service_type);
    if (!$plugin) {
	$haenv->log('err', "service type '$service_type' not implemented");
	return EUNKNOWN_SERVICE_TYPE;
    }

    if (!$service_config) {
	$haenv->log('err', "missing resource configuration for '$sid'");
	return EUNKNOWN_SERVICE;
    }

    if ($service_config->{node} ne $nodename) {
	$haenv->log('err', "service '$sid' not on this node");
	return EWRONG_NODE;
    }

    my $id = $service_name;

    my $running = $plugin->check_running($haenv, $id);

    if ($cmd eq 'started') {

	return SUCCESS if $running;

	$haenv->log("info", "starting service $sid");

	$plugin->start($haenv, $id);

	$running = $plugin->check_running($haenv, $id);

	if ($running) {
	    $haenv->log("info", "service status $sid started");
	    return SUCCESS;
	} else {
	    $haenv->log("warning", "unable to start service $sid");
	    return ERROR;
	}

    } elsif ($cmd eq 'request_stop' || $cmd eq 'stopped') {

	return SUCCESS if !$running;

	$haenv->log("info", "stopping service $sid");

	$plugin->shutdown($haenv, $id);

	$running = $plugin->check_running($haenv, $id);

	if (!$running) {
	    $haenv->log("info", "service status $sid stopped");
	    return SUCCESS;
	} else {
	    $haenv->log("info", "unable to stop stop service $sid (still running)");
	    return ERROR;
	}

    } elsif ($cmd eq 'migrate' || $cmd eq 'relocate') {

	my $target = $params[0];
	if (!defined($target)) {
	    die "$cmd '$sid' failed - missing target\n" if !defined($target);
	    return EINVALID_PARAMETER;
	}

	if ($service_config->{node} eq $target) {
	    # already there
	    return SUCCESS;
	}

	my $online = ($cmd eq 'migrate') ? 1 : 0;

	my $res = $plugin->migrate($haenv, $id, $target, $online);

	# something went wrong if service is still on this node
	if (!$res) {
	    $haenv->log("err", "service $sid not moved (migration error)");
	    return ERROR;
	}

	return SUCCESS;

    } elsif ($cmd eq 'error') {

	if ($running) {
	    $haenv->log("err", "service $sid is in an error state while running");
	} else {
	    $haenv->log("warning", "service $sid is not running and in an error state");
	}
	return SUCCESS; # error always succeeds

    }

    $haenv->log("err", "implement me (cmd '$cmd')");
    return EUNKNOWN_COMMAND;
}


1;
Commit	Line	Data
5f095798 DM	1	package PVE::HA::LRM;
	2
	3	# Local Resource Manager
	4
	5	use strict;
	6	use warnings;
c4a221bc DM	7	use Data::Dumper;
c4a221bc DM	8	use POSIX qw(:sys_wait_h);
5f095798 DM	9
	10	use PVE::SafeSyslog;
	11	use PVE::Tools;
a89ff919	12	use PVE::HA::Tools ':exit_codes';
2a045f55	13	use PVE::HA::Resources;
5f095798 DM	14
	15	# Server can have several states:
	16
	17	my $valid_states = {
ec911edd	18	wait_for_agent_lock => "waiting for agent lock",
0bba8f60	19	active => "got agent_lock",
5f095798 DM	20	lost_agent_lock => "lost agent_lock",
	21	};
	22
	23	sub new {
	24	my ($this, $haenv) = @_;
	25
	26	my $class = ref($this) \|\| $this;
	27
	28	my $self = bless {
	29	haenv => $haenv,
	30	status => { state => 'startup' },
c4a221bc DM	31	workers => {},
c4a221bc DM	32	results => {},
ea4443cc	33	restart_tries => {},
067cdf33	34	shutdown_request => 0,
116dea30	35	shutdown_errors => 0,
9c7d068b DM	36	# mode can be: active, reboot, shutdown, restart
9c7d068b DM	37	mode => 'active',
5f095798 DM	38	}, $class;
5f095798 DM	39
b0bf08a9	40	$self->set_local_status({ state => 'wait_for_agent_lock' });
9c7d068b	41
5f095798 DM	42	return $self;
	43	}
	44
	45	sub shutdown_request {
	46	my ($self) = @_;
	47
f1be5b3a DM	48	return if $self->{shutdown_request}; # already in shutdown mode
f1be5b3a DM	49
499f06e3 DM	50	my $haenv = $self->{haenv};
499f06e3 DM	51
116dea30 DM	52	my $nodename = $haenv->nodename();
116dea30 DM	53
cde77779	54	my $shutdown = $haenv->is_node_shutdown();
499f06e3 DM	55
	56	if ($shutdown) {
	57	$haenv->log('info', "shutdown LRM, stop all services");
	58	$self->{mode} = 'shutdown';
116dea30 DM	59
	60	# queue stop jobs for all services
	61
	62	my $ss = $self->{service_status};
	63
	64	foreach my $sid (keys %$ss) {
	65	my $sd = $ss->{$sid};
	66	next if !$sd->{node};
	67	next if $sd->{node} ne $nodename;
c0edbd7e	68	# Note: use undef uid to mark shutdown/stop jobs
116dea30 DM	69	$self->queue_resource_command($sid, undef, 'request_stop');
	70	}
	71
499f06e3 DM	72	} else {
	73	$haenv->log('info', "restart LRM, freeze all services");
	74	$self->{mode} = 'restart';
	75	}
9c7d068b	76
499f06e3	77	$self->{shutdown_request} = 1;
9c7d068b DM	78
	79	eval { $self->update_lrm_status(); };
	80	if (my $err = $@) {
5bd7aa54	81	$self->log('err', "unable to update lrm status file - $err");
9c7d068b	82	}
5f095798 DM	83	}
	84
	85	sub get_local_status {
	86	my ($self) = @_;
	87
	88	return $self->{status};
	89	}
	90
	91	sub set_local_status {
	92	my ($self, $new) = @_;
	93
	94	die "invalid state '$new->{state}'" if !$valid_states->{$new->{state}};
	95
	96	my $haenv = $self->{haenv};
	97
	98	my $old = $self->{status};
	99
	100	# important: only update if if really changed
	101	return if $old->{state} eq $new->{state};
	102
0bba8f60	103	$haenv->log('info', "status change $old->{state} => $new->{state}");
5f095798 DM	104
	105	$new->{state_change_time} = $haenv->get_time();
	106
	107	$self->{status} = $new;
	108	}
	109
9c7d068b DM	110	sub update_lrm_status {
	111	my ($self) = @_;
	112
5bd7aa54 DM	113	my $haenv = $self->{haenv};
5bd7aa54 DM	114
79829202 DM	115	return 0 if !$haenv->quorate();
79829202 DM	116
9c7d068b	117	my $lrm_status = {
331a9f00	118	state => $self->{status}->{state},
9c7d068b DM	119	mode => $self->{mode},
9c7d068b DM	120	results => $self->{results},
aa330d1c	121	timestamp => $haenv->get_time(),
9c7d068b DM	122	};
9c7d068b DM	123
5bd7aa54 DM	124	eval { $haenv->write_lrm_status($lrm_status); };
	125	if (my $err = $@) {
	126	$haenv->log('err', "unable to write lrm status file - $err");
	127	return 0;
	128	}
	129
	130	return 1;
9c7d068b DM	131	}
9c7d068b DM	132
5f095798 DM	133	sub get_protected_ha_agent_lock {
	134	my ($self) = @_;
	135
	136	my $haenv = $self->{haenv};
	137
	138	my $count = 0;
	139	my $starttime = $haenv->get_time();
	140
	141	for (;;) {
	142
	143	if ($haenv->get_ha_agent_lock()) {
	144	if ($self->{ha_agent_wd}) {
	145	$haenv->watchdog_update($self->{ha_agent_wd});
	146	} else {
	147	my $wfh = $haenv->watchdog_open();
	148	$self->{ha_agent_wd} = $wfh;
	149	}
	150	return 1;
	151	}
	152
	153	last if ++$count > 5; # try max 5 time
	154
	155	my $delay = $haenv->get_time() - $starttime;
	156	last if $delay > 5; # for max 5 seconds
	157
	158	$haenv->sleep(1);
	159	}
	160
	161	return 0;
	162	}
	163
546e2f1f DM	164	sub active_service_count {
	165	my ($self) = @_;
	166
	167	my $haenv = $self->{haenv};
	168
	169	my $nodename = $haenv->nodename();
	170
	171	my $ss = $self->{service_status};
	172
	173	my $count = 0;
	174
	175	foreach my $sid (keys %$ss) {
	176	my $sd = $ss->{$sid};
	177	next if !$sd->{node};
	178	next if $sd->{node} ne $nodename;
	179	my $req_state = $sd->{state};
	180	next if !defined($req_state);
	181	next if $req_state eq 'stopped';
9c7d068b	182	next if $req_state eq 'freeze';
546e2f1f DM	183
	184	$count++;
	185	}
	186
	187	return $count;
	188	}
5bd7aa54 DM	189
	190	my $wrote_lrm_status_at_startup = 0;
	191
5f095798 DM	192	sub do_one_iteration {
	193	my ($self) = @_;
	194
	195	my $haenv = $self->{haenv};
	196
c5ec095f	197	if (!$wrote_lrm_status_at_startup) {
79829202	198	if ($self->update_lrm_status()) {
c5ec095f DM	199	$wrote_lrm_status_at_startup = 1;
	200	} else {
	201	# do nothing
	202	$haenv->sleep(5);
	203	return $self->{shutdown_request} ? 0 : 1;
	204	}
5bd7aa54 DM	205	}
5bd7aa54 DM	206
5f095798 DM	207	my $status = $self->get_local_status();
	208	my $state = $status->{state};
	209
067cdf33 DM	210	my $ms = $haenv->read_manager_status();
	211	$self->{service_status} = $ms->{service_status} \|\| {};
	212
49777d09	213	my $fence_request = PVE::HA::Tools::count_fenced_services($self->{service_status}, $haenv->nodename());
067cdf33	214
5f095798 DM	215	# do state changes first
	216
	217	my $ctime = $haenv->get_time();
	218
b0bf08a9	219	if ($state eq 'wait_for_agent_lock') {
5f095798	220
546e2f1f	221	my $service_count = $self->active_service_count();
5f095798	222
067cdf33	223	if (!$fence_request && $service_count && $haenv->quorate()) {
0bba8f60 DM	224	if ($self->get_protected_ha_agent_lock()) {
0bba8f60 DM	225	$self->set_local_status({ state => 'active' });
5f095798 DM	226	}
	227	}
	228
	229	} elsif ($state eq 'lost_agent_lock') {
	230
067cdf33	231	if (!$fence_request && $haenv->quorate()) {
0bba8f60 DM	232	if ($self->get_protected_ha_agent_lock()) {
0bba8f60 DM	233	$self->set_local_status({ state => 'active' });
5f095798 DM	234	}
	235	}
	236
0bba8f60	237	} elsif ($state eq 'active') {
5f095798	238
067cdf33 DM	239	if ($fence_request) {
	240	$haenv->log('err', "node need to be fenced - releasing agent_lock\n");
	241	$self->set_local_status({ state => 'lost_agent_lock'});
	242	} elsif (!$self->get_protected_ha_agent_lock()) {
5f095798 DM	243	$self->set_local_status({ state => 'lost_agent_lock'});
	244	}
	245	}
	246
	247	$status = $self->get_local_status();
	248	$state = $status->{state};
	249
	250	# do work
	251
	252	if ($state eq 'wait_for_agent_lock') {
	253
	254	return 0 if $self->{shutdown_request};
79829202 DM	255
	256	$self->update_lrm_status();
	257
5f095798 DM	258	$haenv->sleep(5);
5f095798 DM	259
0bba8f60	260	} elsif ($state eq 'active') {
5f095798 DM	261
	262	my $startime = $haenv->get_time();
	263
	264	my $max_time = 10;
	265
	266	my $shutdown = 0;
	267
	268	# do work (max_time seconds)
	269	eval {
	270	# fixme: set alert timer
	271
	272	if ($self->{shutdown_request}) {
	273
499f06e3	274	if ($self->{mode} eq 'restart') {
5f095798	275
499f06e3	276	my $service_count = $self->active_service_count();
5f095798	277
499f06e3	278	if ($service_count == 0) {
5f095798	279
116dea30 DM	280	if ($self->run_workers() == 0) {
	281	if ($self->{ha_agent_wd}) {
	282	$haenv->watchdog_close($self->{ha_agent_wd});
	283	delete $self->{ha_agent_wd};
	284	}
	285
	286	$shutdown = 1;
e23f674c TL	287
	288	# restart with no or freezed services, release the lock
	289	$haenv->release_ha_agent_lock();
116dea30 DM	290	}
	291	}
	292	} else {
	293
	294	if ($self->run_workers() == 0) {
	295	if ($self->{shutdown_errors} == 0) {
	296	if ($self->{ha_agent_wd}) {
	297	$haenv->watchdog_close($self->{ha_agent_wd});
	298	delete $self->{ha_agent_wd};
	299	}
0e5b1a43 TL	300
	301	# shutdown with all services stopped thus release the lock
	302	$haenv->release_ha_agent_lock();
499f06e3	303	}
5f095798	304
499f06e3 DM	305	$shutdown = 1;
499f06e3 DM	306	}
5f095798	307	}
c4a221bc	308	} else {
c4a221bc DM	309
c4a221bc DM	310	$self->manage_resources();
067cdf33	311
5f095798 DM	312	}
	313	};
	314	if (my $err = $@) {
	315	$haenv->log('err', "got unexpected error - $err");
	316	}
	317
79829202 DM	318	$self->update_lrm_status();
79829202 DM	319
5f095798 DM	320	return 0 if $shutdown;
	321
	322	$haenv->sleep_until($startime + $max_time);
	323
	324	} elsif ($state eq 'lost_agent_lock') {
	325
	326	# Note: watchdog is active an will triger soon!
	327
	328	# so we hope to get the lock back soon!
	329
	330	if ($self->{shutdown_request}) {
	331
546e2f1f	332	my $service_count = $self->active_service_count();
5f095798	333
546e2f1f	334	if ($service_count > 0) {
5f095798	335	$haenv->log('err', "get shutdown request in state 'lost_agent_lock' - " .
546e2f1f	336	"detected $service_count running services");
5f095798	337
546e2f1f	338	} else {
5f095798	339
546e2f1f	340	# all services are stopped, so we can close the watchdog
5f095798	341
546e2f1f DM	342	if ($self->{ha_agent_wd}) {
	343	$haenv->watchdog_close($self->{ha_agent_wd});
	344	delete $self->{ha_agent_wd};
	345	}
	346
	347	return 0;
5f095798	348	}
5f095798 DM	349	}
5f095798 DM	350
b0bf08a9 DM	351	$haenv->sleep(5);
b0bf08a9 DM	352
5f095798 DM	353	} else {
	354
	355	die "got unexpected status '$state'\n";
	356
	357	}
	358
	359	return 1;
	360	}
	361
116dea30	362	sub run_workers {
c4a221bc DM	363	my ($self) = @_;
	364
	365	my $haenv = $self->{haenv};
	366
f31b7e94	367	my $starttime = $haenv->get_time();
c4a221bc DM	368
	369	# start workers
	370	my $max_workers = 4;
	371
6dbf93a0	372	my $sc = $haenv->read_service_config();
f31b7e94 DM	373
f31b7e94 DM	374	while (($haenv->get_time() - $starttime) < 5) {
c4a221bc DM	375	my $count = $self->check_active_workers();
	376
	377	foreach my $sid (keys %{$self->{workers}}) {
	378	last if $count >= $max_workers;
	379	my $w = $self->{workers}->{$sid};
	380	if (!$w->{pid}) {
f31b7e94 DM	381	if ($haenv->can_fork()) {
	382	my $pid = fork();
	383	if (!defined($pid)) {
	384	$haenv->log('err', "fork worker failed");
	385	$count = 0; last; # abort, try later
	386	} elsif ($pid == 0) {
a2aae08a TL	387	$haenv->after_fork(); # cleanup
a2aae08a TL	388
f31b7e94 DM	389	# do work
	390	my $res = -1;
	391	eval {
aaabde6a	392	$res = $self->exec_resource_agent($sid, $sc->{$sid}, $w->{state}, $w->{target});
f31b7e94 DM	393	};
	394	if (my $err = $@) {
	395	$haenv->log('err', $err);
	396	POSIX::_exit(-1);
	397	}
	398	POSIX::_exit($res);
	399	} else {
	400	$count++;
	401	$w->{pid} = $pid;
	402	}
	403	} else {
c4a221bc DM	404	my $res = -1;
c4a221bc DM	405	eval {
aaabde6a	406	$res = $self->exec_resource_agent($sid, $sc->{$sid}, $w->{state}, $w->{target});
b33b5743	407	$res = $res << 8 if $res > 0;
c4a221bc DM	408	};
c4a221bc DM	409	if (my $err = $@) {
f31b7e94	410	$haenv->log('err', $err);
116dea30 DM	411	}
	412	if (defined($w->{uid})) {
	413	$self->resource_command_finished($sid, $w->{uid}, $res);
	414	} else {
	415	$self->stop_command_finished($sid, $res);
	416	}
c4a221bc DM	417	}
	418	}
	419	}
	420
	421	last if !$count;
	422
f31b7e94	423	$haenv->sleep(1);
c4a221bc	424	}
116dea30 DM	425
	426	return scalar(keys %{$self->{workers}});
	427	}
	428
	429	sub manage_resources {
	430	my ($self) = @_;
	431
	432	my $haenv = $self->{haenv};
	433
	434	my $nodename = $haenv->nodename();
	435
	436	my $ss = $self->{service_status};
	437
	438	foreach my $sid (keys %$ss) {
	439	my $sd = $ss->{$sid};
	440	next if !$sd->{node};
	441	next if !$sd->{uid};
	442	next if $sd->{node} ne $nodename;
	443	my $req_state = $sd->{state};
	444	next if !defined($req_state);
	445	next if $req_state eq 'freeze';
	446	$self->queue_resource_command($sid, $sd->{uid}, $req_state, $sd->{target});
	447	}
	448
	449	return $self->run_workers();
c4a221bc DM	450	}
c4a221bc DM	451
c4a221bc	452	sub queue_resource_command {
e88469ba	453	my ($self, $sid, $uid, $state, $target) = @_;
c4a221bc DM	454
	455	if (my $w = $self->{workers}->{$sid}) {
	456	return if $w->{pid}; # already started
	457	# else, delete and overwrite queue entry with new command
	458	delete $self->{workers}->{$sid};
	459	}
	460
	461	$self->{workers}->{$sid} = {
	462	sid => $sid,
	463	uid => $uid,
	464	state => $state,
	465	};
e88469ba DM	466
e88469ba DM	467	$self->{workers}->{$sid}->{target} = $target if $target;
c4a221bc DM	468	}
	469
	470	sub check_active_workers {
	471	my ($self) = @_;
	472
	473	# finish/count workers
	474	my $count = 0;
	475	foreach my $sid (keys %{$self->{workers}}) {
	476	my $w = $self->{workers}->{$sid};
	477	if (my $pid = $w->{pid}) {
	478	# check status
	479	my $waitpid = waitpid($pid, WNOHANG);
	480	if (defined($waitpid) && ($waitpid == $pid)) {
c0edbd7e	481	if (defined($w->{uid})) {
116dea30 DM	482	$self->resource_command_finished($sid, $w->{uid}, $?);
	483	} else {
	484	$self->stop_command_finished($sid, $?);
	485	}
c4a221bc DM	486	} else {
	487	$count++;
	488	}
	489	}
	490	}
	491
	492	return $count;
	493	}
	494
116dea30 DM	495	sub stop_command_finished {
	496	my ($self, $sid, $status) = @_;
	497
	498	my $haenv = $self->{haenv};
	499
	500	my $w = delete $self->{workers}->{$sid};
	501	return if !$w; # should not happen
	502
	503	my $exit_code = -1;
	504
	505	if ($status == -1) {
	506	$haenv->log('err', "resource agent $sid finished - failed to execute");
	507	} elsif (my $sig = ($status & 127)) {
	508	$haenv->log('err', "resource agent $sid finished - got signal $sig");
	509	} else {
	510	$exit_code = ($status >> 8);
	511	}
	512
	513	if ($exit_code != 0) {
	514	$self->{shutdown_errors}++;
	515	}
	516	}
	517
c4a221bc DM	518	sub resource_command_finished {
	519	my ($self, $sid, $uid, $status) = @_;
	520
	521	my $haenv = $self->{haenv};
	522
	523	my $w = delete $self->{workers}->{$sid};
	524	return if !$w; # should not happen
	525
	526	my $exit_code = -1;
	527
	528	if ($status == -1) {
0f70400d	529	$haenv->log('err', "resource agent $sid finished - failed to execute");
c4a221bc	530	} elsif (my $sig = ($status & 127)) {
0f70400d	531	$haenv->log('err', "resource agent $sid finished - got signal $sig");
c4a221bc DM	532	} else {
c4a221bc DM	533	$exit_code = ($status >> 8);
c4a221bc DM	534	}
c4a221bc DM	535
ea4443cc TL	536	$exit_code = $self->handle_service_exitcode($sid, $w->{state}, $exit_code);
ea4443cc TL	537
c4a221bc DM	538	$self->{results}->{$uid} = {
	539	sid => $w->{sid},
	540	state => $w->{state},
	541	exit_code => $exit_code,
	542	};
	543
	544	my $ss = $self->{service_status};
	545
	546	# compute hash of valid/existing uids
	547	my $valid_uids = {};
	548	foreach my $sid (keys %$ss) {
	549	my $sd = $ss->{$sid};
	550	next if !$sd->{uid};
	551	$valid_uids->{$sd->{uid}} = 1;
	552	}
	553
	554	my $results = {};
	555	foreach my $id (keys %{$self->{results}}) {
	556	next if !$valid_uids->{$id};
	557	$results->{$id} = $self->{results}->{$id};
	558	}
	559	$self->{results} = $results;
c4a221bc DM	560	}
c4a221bc DM	561
ea4443cc TL	562	# processes the exit code from a finished resource agent, so that the CRM knows
	563	# if the LRM wants to retry an action based on the current recovery policies for
	564	# the failed service, or the CRM itself must try to recover from the failure.
	565	sub handle_service_exitcode {
	566	my ($self, $sid, $cmd, $exit_code) = @_;
	567
	568	my $haenv = $self->{haenv};
	569	my $tries = $self->{restart_tries};
	570
	571	my $sc = $haenv->read_service_config();
aaabde6a DM	572
	573	my $max_restart = 0;
	574
	575	if (my $cd = $sc->{$sid}) {
	576	$max_restart = $cd->{max_restart};
	577	}
ea4443cc TL	578
	579	if ($cmd eq 'started') {
	580
a89ff919	581	if ($exit_code == SUCCESS) {
ea4443cc TL	582
	583	$tries->{$sid} = 0;
	584
	585	return $exit_code;
	586
a89ff919	587	} elsif ($exit_code == ERROR) {
ea4443cc TL	588
	589	$tries->{$sid} = 0 if !defined($tries->{$sid});
	590
	591	$tries->{$sid}++;
aaabde6a	592	if ($tries->{$sid} >= $max_restart) {
ea4443cc TL	593	$haenv->log('err', "unable to start service $sid on local node".
	594	" after $tries->{$sid} retries");
	595	$tries->{$sid} = 0;
a89ff919	596	return ERROR;
ea4443cc TL	597	}
ea4443cc TL	598
a89ff919 TL	599	# tell CRM that we retry the start
a89ff919 TL	600	return ETRY_AGAIN;
ea4443cc TL	601	}
	602	}
	603
	604	return $exit_code;
	605
	606	}
	607
2a045f55 TL	608	sub exec_resource_agent {
	609	my ($self, $sid, $service_config, $cmd, @params) = @_;
	610
	611	# setup execution environment
	612
	613	$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
	614
2a045f55 TL	615	my $haenv = $self->{haenv};
	616
	617	my $nodename = $haenv->nodename();
	618
	619	my (undef, $service_type, $service_name) = PVE::HA::Tools::parse_sid($sid);
	620
	621	my $plugin = PVE::HA::Resources->lookup($service_type);
	622	if (!$plugin) {
	623	$haenv->log('err', "service type '$service_type' not implemented");
	624	return EUNKNOWN_SERVICE_TYPE;
	625	}
	626
aaabde6a DM	627	if (!$service_config) {
	628	$haenv->log('err', "missing resource configuration for '$sid'");
	629	return EUNKNOWN_SERVICE;
	630	}
	631
2a045f55 TL	632	if ($service_config->{node} ne $nodename) {
	633	$haenv->log('err', "service '$sid' not on this node");
	634	return EWRONG_NODE;
	635	}
	636
	637	my $id = $service_name;
	638
	639	my $running = $plugin->check_running($haenv, $id);
	640
	641	if ($cmd eq 'started') {
	642
	643	return SUCCESS if $running;
	644
	645	$haenv->log("info", "starting service $sid");
	646
	647	$plugin->start($haenv, $id);
	648
	649	$running = $plugin->check_running($haenv, $id);
	650
	651	if ($running) {
	652	$haenv->log("info", "service status $sid started");
	653	return SUCCESS;
	654	} else {
	655	$haenv->log("warning", "unable to start service $sid");
	656	return ERROR;
	657	}
	658
	659	} elsif ($cmd eq 'request_stop' \|\| $cmd eq 'stopped') {
	660
	661	return SUCCESS if !$running;
	662
	663	$haenv->log("info", "stopping service $sid");
	664
	665	$plugin->shutdown($haenv, $id);
	666
	667	$running = $plugin->check_running($haenv, $id);
	668
	669	if (!$running) {
	670	$haenv->log("info", "service status $sid stopped");
	671	return SUCCESS;
	672	} else {
	673	$haenv->log("info", "unable to stop stop service $sid (still running)");
	674	return ERROR;
	675	}
	676
	677	} elsif ($cmd eq 'migrate' \|\| $cmd eq 'relocate') {
	678
	679	my $target = $params[0];
	680	if (!defined($target)) {
	681	die "$cmd '$sid' failed - missing target\n" if !defined($target);
	682	return EINVALID_PARAMETER;
	683	}
	684
	685	if ($service_config->{node} eq $target) {
	686	# already there
	687	return SUCCESS;
	688	}
	689
	690	my $online = ($cmd eq 'migrate') ? 1 : 0;
	691
ea28f873	692	my $res = $plugin->migrate($haenv, $id, $target, $online);
2a045f55 TL	693
2a045f55 TL	694	# something went wrong if service is still on this node
ea28f873	695	if (!$res) {
2a045f55 TL	696	$haenv->log("err", "service $sid not moved (migration error)");
	697	return ERROR;
	698	}
	699
	700	return SUCCESS;
	701
	702	} elsif ($cmd eq 'error') {
	703
	704	if ($running) {
	705	$haenv->log("err", "service $sid is in an error state while running");
	706	} else {
	707	$haenv->log("warning", "service $sid is not running and in an error state");
	708	}
	709	return SUCCESS; # error always succeeds
	710
	711	}
	712
	713	$haenv->log("err", "implement me (cmd '$cmd')");
	714	return EUNKNOWN_COMMAND;
	715	}
	716
	717
5f095798	718	1;