[pve-ha-manager.git] / src / PVE / HA / LRM.pm

package PVE::HA::LRM;

# Local Resource Manager

use strict;
use warnings;
use Data::Dumper;
use POSIX qw(:sys_wait_h);

use PVE::SafeSyslog;
use PVE::Tools;
use PVE::HA::Tools;

# Server can have several states:

my $valid_states = {
    wait_for_agent_lock => "waiting for agent lock",
    active => "got agent_lock",
    lost_agent_lock => "lost agent_lock",
};

sub new {
    my ($this, $haenv) = @_;

    my $class = ref($this) || $this;

    my $self = bless {
	haenv => $haenv,
	status => { state => 'startup' },
	workers => {},
	results => {},
	shutdown_request => 0,
	# mode can be: active, reboot, shutdown, restart
	mode => 'active',
    }, $class;

    $self->set_local_status({ state => 	'wait_for_agent_lock' });   

    $self->update_lrm_status();
    
    return $self;
}

sub shutdown_request {
    my ($self) = @_;

    $self->{shutdown_request} = 1;

    $self->{mode} = 'restart'; # fixme: detect shutdown/reboot

    eval { $self->update_lrm_status(); };
    if (my $err = $@) {
	$self->log('err', "unable to update lrm status file");
    }
}

sub get_local_status {
    my ($self) = @_;

    return $self->{status};
}

sub set_local_status {
    my ($self, $new) = @_;

    die "invalid state '$new->{state}'" if !$valid_states->{$new->{state}};

    my $haenv = $self->{haenv};

    my $old = $self->{status};

    # important: only update if if really changed 
    return if $old->{state} eq $new->{state};

    $haenv->log('info', "status change $old->{state} => $new->{state}");

    $new->{state_change_time} = $haenv->get_time();

    $self->{status} = $new;
}

sub update_lrm_status {
    my ($self) = @_;

    my $lrm_status = {	
	mode => $self->{mode},
	results => $self->{results},
    };
    
    $self->{haenv}->write_lrm_status($lrm_status);
}

sub get_protected_ha_agent_lock {
    my ($self) = @_;

    my $haenv = $self->{haenv};

    my $count = 0;
    my $starttime = $haenv->get_time();

    for (;;) {
	
	if ($haenv->get_ha_agent_lock()) {
	    if ($self->{ha_agent_wd}) {
		$haenv->watchdog_update($self->{ha_agent_wd});
	    } else {
		my $wfh = $haenv->watchdog_open();
		$self->{ha_agent_wd} = $wfh;
	    }
	    return 1;
	}
	    
	last if ++$count > 5; # try max 5 time

	my $delay = $haenv->get_time() - $starttime;
	last if $delay > 5; # for max 5 seconds

	$haenv->sleep(1);
    }
    
    return 0;
}

sub active_service_count {
    my ($self) = @_;
    
    my $haenv = $self->{haenv};

    my $nodename = $haenv->nodename();

    my $ss = $self->{service_status};

    my $count = 0;
    
    foreach my $sid (keys %$ss) {
	my $sd = $ss->{$sid};
	next if !$sd->{node};
	next if $sd->{node} ne $nodename;
	my $req_state = $sd->{state};
	next if !defined($req_state);
	next if $req_state eq 'stopped';
	next if $req_state eq 'freeze';

	$count++;
    }
    
    return $count;
}
    
sub do_one_iteration {
    my ($self) = @_;

    my $haenv = $self->{haenv};

    my $status = $self->get_local_status();
    my $state = $status->{state};

    my $ms = $haenv->read_manager_status();
    $self->{service_status} =  $ms->{service_status} || {};

    my $fence_request = PVE::HA::Tools::count_fenced_services($self->{service_status}, $haenv->nodename());
    
    # do state changes first 

    my $ctime = $haenv->get_time();

    if ($state eq 'wait_for_agent_lock') {

	my $service_count = $self->active_service_count();

	if (!$fence_request && $service_count && $haenv->quorate()) {
	    if ($self->get_protected_ha_agent_lock()) {
		$self->set_local_status({ state => 'active' });
	    }
	}
	
    } elsif ($state eq 'lost_agent_lock') {

	if (!$fence_request && $haenv->quorate()) {
	    if ($self->get_protected_ha_agent_lock()) {
		$self->set_local_status({ state => 'active' });
	    }
	}

    } elsif ($state eq 'active') {

	if ($fence_request) {		
	    $haenv->log('err', "node need to be fenced - releasing agent_lock\n");
	    $self->set_local_status({ state => 'lost_agent_lock'});	
	} elsif (!$self->get_protected_ha_agent_lock()) {
	    $self->set_local_status({ state => 'lost_agent_lock'});
	}
    }

    $status = $self->get_local_status();
    $state = $status->{state};

    # do work

    if ($state eq 'wait_for_agent_lock') {

	return 0 if $self->{shutdown_request};

	$haenv->sleep(5);
	   
    } elsif ($state eq 'active') {

	my $startime = $haenv->get_time();

	my $max_time = 10;

	my $shutdown = 0;

	# do work (max_time seconds)
	eval {
	    # fixme: set alert timer

	    if ($self->{shutdown_request}) {

		# fixme: request service stop or relocate ?

		my $service_count = $self->active_service_count();

		if ($service_count == 0) {

		    if ($self->{ha_agent_wd}) {
			$haenv->watchdog_close($self->{ha_agent_wd});
			delete $self->{ha_agent_wd};
		    }

		    $shutdown = 1;
		}
	    } else {

		$self->manage_resources();

	    }
	};
	if (my $err = $@) {
	    $haenv->log('err', "got unexpected error - $err");
	}

	return 0 if $shutdown;

	$haenv->sleep_until($startime + $max_time);

    } elsif ($state eq 'lost_agent_lock') {
	
	# Note: watchdog is active an will triger soon!

	# so we hope to get the lock back soon!

	if ($self->{shutdown_request}) {

	    my $service_count = $self->active_service_count();

	    if ($service_count > 0) {
		$haenv->log('err', "get shutdown request in state 'lost_agent_lock' - " . 
			    "detected $service_count running services");

	    } else {

		# all services are stopped, so we can close the watchdog

		if ($self->{ha_agent_wd}) {
		    $haenv->watchdog_close($self->{ha_agent_wd});
		    delete $self->{ha_agent_wd};
		}
		
		return 0;
	    }
	}

	$haenv->sleep(5);

    } else {

	die "got unexpected status '$state'\n";

    }

    return 1;
}

sub manage_resources {
    my ($self) = @_;

    my $haenv = $self->{haenv};

    my $nodename = $haenv->nodename();

    my $ss = $self->{service_status};

    foreach my $sid (keys %$ss) {
	my $sd = $ss->{$sid};
	next if !$sd->{node};
	next if !$sd->{uid};
	next if $sd->{node} ne $nodename;
	my $req_state = $sd->{state};
	next if !defined($req_state);
	next if $req_state eq 'freeze';
	eval {
	    $self->queue_resource_command($sid, $sd->{uid}, $req_state, $sd->{target});
	};
	if (my $err = $@) {
	    $haenv->log('err', "unable to run resource agent for '$sid' - $err"); # fixme
	}
    }

    my $starttime = $haenv->get_time();

    # start workers
    my $max_workers = 4;

    my $sc = $haenv->read_service_config();

    while (($haenv->get_time() - $starttime) < 5) {
	my $count =  $self->check_active_workers();

	foreach my $sid (keys %{$self->{workers}}) {
	    last if $count >= $max_workers;
	    my $w = $self->{workers}->{$sid};
	    my $cd = $sc->{$sid};
	    if (!$cd) {
		$haenv->log('err', "missing resource configuration for '$sid'");
		next;
	    }
	    if (!$w->{pid}) {
		if ($haenv->can_fork()) {
		    my $pid = fork();
		    if (!defined($pid)) {
			$haenv->log('err', "fork worker failed");
			$count = 0; last; # abort, try later
		    } elsif ($pid == 0) {
			# do work
			my $res = -1;
			eval {
			    $res = $haenv->exec_resource_agent($sid, $cd, $w->{state}, $w->{target});
			};
			if (my $err = $@) {
			    $haenv->log('err', $err);
			    POSIX::_exit(-1);
			}  
			POSIX::_exit($res); 
		    } else {
			$count++;
			$w->{pid} = $pid;
		    }
		} else {
		    my $res = -1;
		    eval {
			$res = $haenv->exec_resource_agent($sid, $cd, $w->{state}, $w->{target});
		    };
		    if (my $err = $@) {
			$haenv->log('err', $err);
		    }		    
		    $self->resource_command_finished($sid, $w->{uid}, $res);
		}
	    }
	}

	last if !$count;

	$haenv->sleep(1);
    }
}

# fixme: use a queue an limit number of parallel workers?
sub queue_resource_command {
    my ($self, $sid, $uid, $state, $target) = @_;

    if (my $w = $self->{workers}->{$sid}) {
	return if $w->{pid}; # already started
	# else, delete and overwrite queue entry with new command
	delete $self->{workers}->{$sid};
    }

    $self->{workers}->{$sid} = {
	sid => $sid,
	uid => $uid,
	state => $state,
    };

    $self->{workers}->{$sid}->{target} = $target if $target;
}

sub check_active_workers {
    my ($self) = @_;

    # finish/count workers
    my $count = 0;
    foreach my $sid (keys %{$self->{workers}}) {
	my $w = $self->{workers}->{$sid};
	if (my $pid = $w->{pid}) {
	    # check status
	    my $waitpid = waitpid($pid, WNOHANG);
	    if (defined($waitpid) && ($waitpid == $pid)) {
		$self->resource_command_finished($sid, $w->{uid}, $?);
	    } else {
		$count++;
	    }
	}
    }
    
    return $count;
}

sub resource_command_finished {
    my ($self, $sid, $uid, $status) = @_;

    my $haenv = $self->{haenv};

    my $w = delete $self->{workers}->{$sid};
    return if !$w; # should not happen

    my $exit_code = -1;

    if ($status == -1) {
	$haenv->log('err', "resource agent $sid finished - failed to execute");    
    }  elsif (my $sig = ($status & 127)) {
	$haenv->log('err', "resource agent $sid finished - got signal $sig");
    } else {
	$exit_code = ($status >> 8);
    }

    $self->{results}->{$uid} = {
	sid => $w->{sid},
	state => $w->{state},
	exit_code => $exit_code,
    };

    my $ss = $self->{service_status};

    # compute hash of valid/existing uids
    my $valid_uids = {};
    foreach my $sid (keys %$ss) {
	my $sd = $ss->{$sid};
	next if !$sd->{uid};
	$valid_uids->{$sd->{uid}} = 1;
    }

    my $results = {};
    foreach my $id (keys %{$self->{results}}) {
	next if !$valid_uids->{$id};
	$results->{$id} = $self->{results}->{$id};
    }
    $self->{results} = $results;

    $self->update_lrm_status();
}

1;
Commit	Line	Data
5f095798 DM	1	package PVE::HA::LRM;
	2
	3	# Local Resource Manager
	4
	5	use strict;
	6	use warnings;
c4a221bc DM	7	use Data::Dumper;
c4a221bc DM	8	use POSIX qw(:sys_wait_h);
5f095798 DM	9
	10	use PVE::SafeSyslog;
	11	use PVE::Tools;
	12	use PVE::HA::Tools;
	13
	14	# Server can have several states:
	15
	16	my $valid_states = {
ec911edd	17	wait_for_agent_lock => "waiting for agent lock",
0bba8f60	18	active => "got agent_lock",
5f095798 DM	19	lost_agent_lock => "lost agent_lock",
	20	};
	21
	22	sub new {
	23	my ($this, $haenv) = @_;
	24
	25	my $class = ref($this) \|\| $this;
	26
	27	my $self = bless {
	28	haenv => $haenv,
	29	status => { state => 'startup' },
c4a221bc DM	30	workers => {},
c4a221bc DM	31	results => {},
067cdf33	32	shutdown_request => 0,
9c7d068b DM	33	# mode can be: active, reboot, shutdown, restart
9c7d068b DM	34	mode => 'active',
5f095798 DM	35	}, $class;
5f095798 DM	36
b0bf08a9	37	$self->set_local_status({ state => 'wait_for_agent_lock' });
9c7d068b DM	38
9c7d068b DM	39	$self->update_lrm_status();
5f095798 DM	40
	41	return $self;
	42	}
	43
	44	sub shutdown_request {
	45	my ($self) = @_;
	46
	47	$self->{shutdown_request} = 1;
9c7d068b DM	48
	49	$self->{mode} = 'restart'; # fixme: detect shutdown/reboot
	50
	51	eval { $self->update_lrm_status(); };
	52	if (my $err = $@) {
	53	$self->log('err', "unable to update lrm status file");
	54	}
5f095798 DM	55	}
	56
	57	sub get_local_status {
	58	my ($self) = @_;
	59
	60	return $self->{status};
	61	}
	62
	63	sub set_local_status {
	64	my ($self, $new) = @_;
	65
	66	die "invalid state '$new->{state}'" if !$valid_states->{$new->{state}};
	67
	68	my $haenv = $self->{haenv};
	69
	70	my $old = $self->{status};
	71
	72	# important: only update if if really changed
	73	return if $old->{state} eq $new->{state};
	74
0bba8f60	75	$haenv->log('info', "status change $old->{state} => $new->{state}");
5f095798 DM	76
	77	$new->{state_change_time} = $haenv->get_time();
	78
	79	$self->{status} = $new;
	80	}
	81
9c7d068b DM	82	sub update_lrm_status {
	83	my ($self) = @_;
	84
	85	my $lrm_status = {
	86	mode => $self->{mode},
	87	results => $self->{results},
	88	};
	89
	90	$self->{haenv}->write_lrm_status($lrm_status);
	91	}
	92
5f095798 DM	93	sub get_protected_ha_agent_lock {
	94	my ($self) = @_;
	95
	96	my $haenv = $self->{haenv};
	97
	98	my $count = 0;
	99	my $starttime = $haenv->get_time();
	100
	101	for (;;) {
	102
	103	if ($haenv->get_ha_agent_lock()) {
	104	if ($self->{ha_agent_wd}) {
	105	$haenv->watchdog_update($self->{ha_agent_wd});
	106	} else {
	107	my $wfh = $haenv->watchdog_open();
	108	$self->{ha_agent_wd} = $wfh;
	109	}
	110	return 1;
	111	}
	112
	113	last if ++$count > 5; # try max 5 time
	114
	115	my $delay = $haenv->get_time() - $starttime;
	116	last if $delay > 5; # for max 5 seconds
	117
	118	$haenv->sleep(1);
	119	}
	120
	121	return 0;
	122	}
	123
546e2f1f DM	124	sub active_service_count {
	125	my ($self) = @_;
	126
	127	my $haenv = $self->{haenv};
	128
	129	my $nodename = $haenv->nodename();
	130
	131	my $ss = $self->{service_status};
	132
	133	my $count = 0;
	134
	135	foreach my $sid (keys %$ss) {
	136	my $sd = $ss->{$sid};
	137	next if !$sd->{node};
	138	next if $sd->{node} ne $nodename;
	139	my $req_state = $sd->{state};
	140	next if !defined($req_state);
	141	next if $req_state eq 'stopped';
9c7d068b	142	next if $req_state eq 'freeze';
546e2f1f DM	143
	144	$count++;
	145	}
	146
	147	return $count;
	148	}
067cdf33	149
5f095798 DM	150	sub do_one_iteration {
	151	my ($self) = @_;
	152
	153	my $haenv = $self->{haenv};
	154
	155	my $status = $self->get_local_status();
	156	my $state = $status->{state};
	157
067cdf33 DM	158	my $ms = $haenv->read_manager_status();
	159	$self->{service_status} = $ms->{service_status} \|\| {};
	160
49777d09	161	my $fence_request = PVE::HA::Tools::count_fenced_services($self->{service_status}, $haenv->nodename());
067cdf33	162
5f095798 DM	163	# do state changes first
	164
	165	my $ctime = $haenv->get_time();
	166
b0bf08a9	167	if ($state eq 'wait_for_agent_lock') {
5f095798	168
546e2f1f	169	my $service_count = $self->active_service_count();
5f095798	170
067cdf33	171	if (!$fence_request && $service_count && $haenv->quorate()) {
0bba8f60 DM	172	if ($self->get_protected_ha_agent_lock()) {
0bba8f60 DM	173	$self->set_local_status({ state => 'active' });
5f095798 DM	174	}
	175	}
	176
	177	} elsif ($state eq 'lost_agent_lock') {
	178
067cdf33	179	if (!$fence_request && $haenv->quorate()) {
0bba8f60 DM	180	if ($self->get_protected_ha_agent_lock()) {
0bba8f60 DM	181	$self->set_local_status({ state => 'active' });
5f095798 DM	182	}
	183	}
	184
0bba8f60	185	} elsif ($state eq 'active') {
5f095798	186
067cdf33 DM	187	if ($fence_request) {
	188	$haenv->log('err', "node need to be fenced - releasing agent_lock\n");
	189	$self->set_local_status({ state => 'lost_agent_lock'});
	190	} elsif (!$self->get_protected_ha_agent_lock()) {
5f095798 DM	191	$self->set_local_status({ state => 'lost_agent_lock'});
	192	}
	193	}
	194
	195	$status = $self->get_local_status();
	196	$state = $status->{state};
	197
	198	# do work
	199
	200	if ($state eq 'wait_for_agent_lock') {
	201
	202	return 0 if $self->{shutdown_request};
	203
	204	$haenv->sleep(5);
	205
0bba8f60	206	} elsif ($state eq 'active') {
5f095798 DM	207
	208	my $startime = $haenv->get_time();
	209
	210	my $max_time = 10;
	211
	212	my $shutdown = 0;
	213
	214	# do work (max_time seconds)
	215	eval {
	216	# fixme: set alert timer
	217
	218	if ($self->{shutdown_request}) {
	219
	220	# fixme: request service stop or relocate ?
	221
546e2f1f	222	my $service_count = $self->active_service_count();
5f095798 DM	223
	224	if ($service_count == 0) {
	225
	226	if ($self->{ha_agent_wd}) {
	227	$haenv->watchdog_close($self->{ha_agent_wd});
	228	delete $self->{ha_agent_wd};
	229	}
	230
	231	$shutdown = 1;
	232	}
c4a221bc	233	} else {
c4a221bc DM	234
c4a221bc DM	235	$self->manage_resources();
067cdf33	236
5f095798 DM	237	}
	238	};
	239	if (my $err = $@) {
	240	$haenv->log('err', "got unexpected error - $err");
	241	}
	242
	243	return 0 if $shutdown;
	244
	245	$haenv->sleep_until($startime + $max_time);
	246
	247	} elsif ($state eq 'lost_agent_lock') {
	248
	249	# Note: watchdog is active an will triger soon!
	250
	251	# so we hope to get the lock back soon!
	252
	253	if ($self->{shutdown_request}) {
	254
546e2f1f	255	my $service_count = $self->active_service_count();
5f095798	256
546e2f1f	257	if ($service_count > 0) {
5f095798	258	$haenv->log('err', "get shutdown request in state 'lost_agent_lock' - " .
546e2f1f	259	"detected $service_count running services");
5f095798	260
546e2f1f	261	} else {
5f095798	262
546e2f1f	263	# all services are stopped, so we can close the watchdog
5f095798	264
546e2f1f DM	265	if ($self->{ha_agent_wd}) {
	266	$haenv->watchdog_close($self->{ha_agent_wd});
	267	delete $self->{ha_agent_wd};
	268	}
	269
	270	return 0;
5f095798	271	}
5f095798 DM	272	}
5f095798 DM	273
b0bf08a9 DM	274	$haenv->sleep(5);
b0bf08a9 DM	275
5f095798 DM	276	} else {
	277
	278	die "got unexpected status '$state'\n";
	279
	280	}
	281
	282	return 1;
	283	}
	284
c4a221bc DM	285	sub manage_resources {
	286	my ($self) = @_;
	287
	288	my $haenv = $self->{haenv};
	289
	290	my $nodename = $haenv->nodename();
	291
c4a221bc DM	292	my $ss = $self->{service_status};
	293
	294	foreach my $sid (keys %$ss) {
	295	my $sd = $ss->{$sid};
	296	next if !$sd->{node};
	297	next if !$sd->{uid};
	298	next if $sd->{node} ne $nodename;
	299	my $req_state = $sd->{state};
	300	next if !defined($req_state);
9c7d068b	301	next if $req_state eq 'freeze';
c4a221bc	302	eval {
e88469ba	303	$self->queue_resource_command($sid, $sd->{uid}, $req_state, $sd->{target});
c4a221bc DM	304	};
c4a221bc DM	305	if (my $err = $@) {
f31b7e94	306	$haenv->log('err', "unable to run resource agent for '$sid' - $err"); # fixme
c4a221bc DM	307	}
	308	}
	309
f31b7e94	310	my $starttime = $haenv->get_time();
c4a221bc DM	311
	312	# start workers
	313	my $max_workers = 4;
	314
6dbf93a0	315	my $sc = $haenv->read_service_config();
f31b7e94 DM	316
f31b7e94 DM	317	while (($haenv->get_time() - $starttime) < 5) {
c4a221bc DM	318	my $count = $self->check_active_workers();
	319
	320	foreach my $sid (keys %{$self->{workers}}) {
	321	last if $count >= $max_workers;
	322	my $w = $self->{workers}->{$sid};
6dbf93a0 DM	323	my $cd = $sc->{$sid};
6dbf93a0 DM	324	if (!$cd) {
f31b7e94	325	$haenv->log('err', "missing resource configuration for '$sid'");
6dbf93a0 DM	326	next;
6dbf93a0 DM	327	}
c4a221bc	328	if (!$w->{pid}) {
f31b7e94 DM	329	if ($haenv->can_fork()) {
	330	my $pid = fork();
	331	if (!defined($pid)) {
	332	$haenv->log('err', "fork worker failed");
	333	$count = 0; last; # abort, try later
	334	} elsif ($pid == 0) {
	335	# do work
	336	my $res = -1;
	337	eval {
	338	$res = $haenv->exec_resource_agent($sid, $cd, $w->{state}, $w->{target});
	339	};
	340	if (my $err = $@) {
	341	$haenv->log('err', $err);
	342	POSIX::_exit(-1);
	343	}
	344	POSIX::_exit($res);
	345	} else {
	346	$count++;
	347	$w->{pid} = $pid;
	348	}
	349	} else {
c4a221bc DM	350	my $res = -1;
c4a221bc DM	351	eval {
6dbf93a0	352	$res = $haenv->exec_resource_agent($sid, $cd, $w->{state}, $w->{target});
c4a221bc DM	353	};
c4a221bc DM	354	if (my $err = $@) {
f31b7e94 DM	355	$haenv->log('err', $err);
	356	}
	357	$self->resource_command_finished($sid, $w->{uid}, $res);
c4a221bc DM	358	}
	359	}
	360	}
	361
	362	last if !$count;
	363
f31b7e94	364	$haenv->sleep(1);
c4a221bc DM	365	}
	366	}
	367
	368	# fixme: use a queue an limit number of parallel workers?
	369	sub queue_resource_command {
e88469ba	370	my ($self, $sid, $uid, $state, $target) = @_;
c4a221bc DM	371
	372	if (my $w = $self->{workers}->{$sid}) {
	373	return if $w->{pid}; # already started
	374	# else, delete and overwrite queue entry with new command
	375	delete $self->{workers}->{$sid};
	376	}
	377
	378	$self->{workers}->{$sid} = {
	379	sid => $sid,
	380	uid => $uid,
	381	state => $state,
	382	};
e88469ba DM	383
e88469ba DM	384	$self->{workers}->{$sid}->{target} = $target if $target;
c4a221bc DM	385	}
	386
	387	sub check_active_workers {
	388	my ($self) = @_;
	389
	390	# finish/count workers
	391	my $count = 0;
	392	foreach my $sid (keys %{$self->{workers}}) {
	393	my $w = $self->{workers}->{$sid};
	394	if (my $pid = $w->{pid}) {
	395	# check status
	396	my $waitpid = waitpid($pid, WNOHANG);
	397	if (defined($waitpid) && ($waitpid == $pid)) {
	398	$self->resource_command_finished($sid, $w->{uid}, $?);
	399	} else {
	400	$count++;
	401	}
	402	}
	403	}
	404
	405	return $count;
	406	}
	407
	408	sub resource_command_finished {
	409	my ($self, $sid, $uid, $status) = @_;
	410
	411	my $haenv = $self->{haenv};
	412
	413	my $w = delete $self->{workers}->{$sid};
	414	return if !$w; # should not happen
	415
	416	my $exit_code = -1;
	417
	418	if ($status == -1) {
0f70400d	419	$haenv->log('err', "resource agent $sid finished - failed to execute");
c4a221bc	420	} elsif (my $sig = ($status & 127)) {
0f70400d	421	$haenv->log('err', "resource agent $sid finished - got signal $sig");
c4a221bc DM	422	} else {
c4a221bc DM	423	$exit_code = ($status >> 8);
c4a221bc DM	424	}
	425
	426	$self->{results}->{$uid} = {
	427	sid => $w->{sid},
	428	state => $w->{state},
	429	exit_code => $exit_code,
	430	};
	431
	432	my $ss = $self->{service_status};
	433
	434	# compute hash of valid/existing uids
	435	my $valid_uids = {};
	436	foreach my $sid (keys %$ss) {
	437	my $sd = $ss->{$sid};
	438	next if !$sd->{uid};
	439	$valid_uids->{$sd->{uid}} = 1;
	440	}
	441
	442	my $results = {};
	443	foreach my $id (keys %{$self->{results}}) {
	444	next if !$valid_uids->{$id};
	445	$results->{$id} = $self->{results}->{$id};
	446	}
	447	$self->{results} = $results;
	448
9c7d068b	449	$self->update_lrm_status();
c4a221bc DM	450	}
c4a221bc DM	451
5f095798	452	1;