3 # Local Resource Manager
8 use POSIX
qw(:sys_wait_h);
12 use PVE
::HA
::Tools
':exit_codes';
13 use PVE
::HA
::Resources
;
15 # Server can have several states:
18 wait_for_agent_lock
=> "waiting for agent lock",
19 active
=> "got agent_lock",
20 lost_agent_lock
=> "lost agent_lock",
24 my ($this, $haenv) = @_;
26 my $class = ref($this) || $this;
30 status
=> { state => 'startup' },
34 shutdown_request
=> 0,
36 # mode can be: active, reboot, shutdown, restart
40 $self->set_local_status({ state => 'wait_for_agent_lock' });
45 sub shutdown_request
{
48 return if $self->{shutdown_request
}; # already in shutdown mode
50 my $haenv = $self->{haenv
};
52 my $nodename = $haenv->nodename();
54 my $shutdown = $haenv->is_node_shutdown();
57 $haenv->log('info', "shutdown LRM, stop all services");
58 $self->{mode
} = 'shutdown';
60 # queue stop jobs for all services
62 my $ss = $self->{service_status
};
64 foreach my $sid (keys %$ss) {
67 next if $sd->{node
} ne $nodename;
68 # Note: use undef uid to mark shutdown/stop jobs
69 $self->queue_resource_command($sid, undef, 'request_stop');
73 $haenv->log('info', "restart LRM, freeze all services");
74 $self->{mode
} = 'restart';
77 $self->{shutdown_request
} = 1;
79 eval { $self->update_lrm_status(); };
81 $self->log('err', "unable to update lrm status file - $err");
85 sub get_local_status
{
88 return $self->{status
};
91 sub set_local_status
{
92 my ($self, $new) = @_;
94 die "invalid state '$new->{state}'" if !$valid_states->{$new->{state}};
96 my $haenv = $self->{haenv
};
98 my $old = $self->{status
};
100 # important: only update if if really changed
101 return if $old->{state} eq $new->{state};
103 $haenv->log('info', "status change $old->{state} => $new->{state}");
105 $new->{state_change_time
} = $haenv->get_time();
107 $self->{status
} = $new;
110 sub update_lrm_status
{
113 my $haenv = $self->{haenv
};
115 return 0 if !$haenv->quorate();
118 state => $self->{status
}->{state},
119 mode
=> $self->{mode
},
120 results
=> $self->{results
},
121 timestamp
=> $haenv->get_time(),
124 eval { $haenv->write_lrm_status($lrm_status); };
126 $haenv->log('err', "unable to write lrm status file - $err");
133 sub get_protected_ha_agent_lock
{
136 my $haenv = $self->{haenv
};
139 my $starttime = $haenv->get_time();
143 if ($haenv->get_ha_agent_lock()) {
144 if ($self->{ha_agent_wd
}) {
145 $haenv->watchdog_update($self->{ha_agent_wd
});
147 my $wfh = $haenv->watchdog_open();
148 $self->{ha_agent_wd
} = $wfh;
153 last if ++$count > 5; # try max 5 time
155 my $delay = $haenv->get_time() - $starttime;
156 last if $delay > 5; # for max 5 seconds
164 sub active_service_count
{
167 my $haenv = $self->{haenv
};
169 my $nodename = $haenv->nodename();
171 my $ss = $self->{service_status
};
175 foreach my $sid (keys %$ss) {
176 my $sd = $ss->{$sid};
177 next if !$sd->{node
};
178 next if $sd->{node
} ne $nodename;
179 my $req_state = $sd->{state};
180 next if !defined($req_state);
181 next if $req_state eq 'stopped';
182 next if $req_state eq 'freeze';
190 my $wrote_lrm_status_at_startup = 0;
192 sub do_one_iteration
{
195 my $haenv = $self->{haenv
};
197 if (!$wrote_lrm_status_at_startup) {
198 if ($self->update_lrm_status()) {
199 $wrote_lrm_status_at_startup = 1;
203 return $self->{shutdown_request
} ?
0 : 1;
207 my $status = $self->get_local_status();
208 my $state = $status->{state};
210 my $ms = $haenv->read_manager_status();
211 $self->{service_status
} = $ms->{service_status
} || {};
213 my $fence_request = PVE
::HA
::Tools
::count_fenced_services
($self->{service_status
}, $haenv->nodename());
215 # do state changes first
217 my $ctime = $haenv->get_time();
219 if ($state eq 'wait_for_agent_lock') {
221 my $service_count = $self->active_service_count();
223 if (!$fence_request && $service_count && $haenv->quorate()) {
224 if ($self->get_protected_ha_agent_lock()) {
225 $self->set_local_status({ state => 'active' });
229 } elsif ($state eq 'lost_agent_lock') {
231 if (!$fence_request && $haenv->quorate()) {
232 if ($self->get_protected_ha_agent_lock()) {
233 $self->set_local_status({ state => 'active' });
237 } elsif ($state eq 'active') {
239 if ($fence_request) {
240 $haenv->log('err', "node need to be fenced - releasing agent_lock\n");
241 $self->set_local_status({ state => 'lost_agent_lock'});
242 } elsif (!$self->get_protected_ha_agent_lock()) {
243 $self->set_local_status({ state => 'lost_agent_lock'});
247 $status = $self->get_local_status();
248 $state = $status->{state};
252 if ($state eq 'wait_for_agent_lock') {
254 return 0 if $self->{shutdown_request
};
256 $self->update_lrm_status();
260 } elsif ($state eq 'active') {
262 my $startime = $haenv->get_time();
268 # do work (max_time seconds)
270 # fixme: set alert timer
272 if ($self->{shutdown_request
}) {
274 if ($self->{mode
} eq 'restart') {
276 my $service_count = $self->active_service_count();
278 if ($service_count == 0) {
280 if ($self->run_workers() == 0) {
281 if ($self->{ha_agent_wd
}) {
282 $haenv->watchdog_close($self->{ha_agent_wd
});
283 delete $self->{ha_agent_wd
};
288 # restart with no or freezed services, release the lock
289 $haenv->release_ha_agent_lock();
294 if ($self->run_workers() == 0) {
295 if ($self->{shutdown_errors
} == 0) {
296 if ($self->{ha_agent_wd
}) {
297 $haenv->watchdog_close($self->{ha_agent_wd
});
298 delete $self->{ha_agent_wd
};
301 # shutdown with all services stopped thus release the lock
302 $haenv->release_ha_agent_lock();
310 $self->manage_resources();
315 $haenv->log('err', "got unexpected error - $err");
318 $self->update_lrm_status();
320 return 0 if $shutdown;
322 $haenv->sleep_until($startime + $max_time);
324 } elsif ($state eq 'lost_agent_lock') {
326 # Note: watchdog is active an will triger soon!
328 # so we hope to get the lock back soon!
330 if ($self->{shutdown_request
}) {
332 my $service_count = $self->active_service_count();
334 if ($service_count > 0) {
335 $haenv->log('err', "get shutdown request in state 'lost_agent_lock' - " .
336 "detected $service_count running services");
340 # all services are stopped, so we can close the watchdog
342 if ($self->{ha_agent_wd
}) {
343 $haenv->watchdog_close($self->{ha_agent_wd
});
344 delete $self->{ha_agent_wd
};
355 die "got unexpected status '$state'\n";
365 my $haenv = $self->{haenv
};
367 my $starttime = $haenv->get_time();
369 # number of workers to start, if 0 we exec the command directly witouth forking
370 my $max_workers = $haenv->get_max_workers();
372 my $sc = $haenv->read_service_config();
374 while (($haenv->get_time() - $starttime) < 5) {
375 my $count = $self->check_active_workers();
377 foreach my $sid (keys %{$self->{workers
}}) {
378 last if $count >= $max_workers && $max_workers > 0;
380 my $w = $self->{workers
}->{$sid};
382 # only fork if we may else call exec_resource_agent
383 # directly (e.g. for regression tests)
384 if ($max_workers > 0) {
386 if (!defined($pid)) {
387 $haenv->log('err', "fork worker failed");
388 $count = 0; last; # abort, try later
389 } elsif ($pid == 0) {
390 $haenv->after_fork(); # cleanup
395 $res = $self->exec_resource_agent($sid, $sc->{$sid}, $w->{state}, $w->{target
});
398 $haenv->log('err', $err);
409 $res = $self->exec_resource_agent($sid, $sc->{$sid}, $w->{state}, $w->{target
});
410 $res = $res << 8 if $res > 0;
413 $haenv->log('err', $err);
415 if (defined($w->{uid
})) {
416 $self->resource_command_finished($sid, $w->{uid
}, $res);
418 $self->stop_command_finished($sid, $res);
429 return scalar(keys %{$self->{workers
}});
432 sub manage_resources
{
435 my $haenv = $self->{haenv
};
437 my $nodename = $haenv->nodename();
439 my $ss = $self->{service_status
};
441 foreach my $sid (keys %$ss) {
442 my $sd = $ss->{$sid};
443 next if !$sd->{node
};
445 next if $sd->{node
} ne $nodename;
446 my $req_state = $sd->{state};
447 next if !defined($req_state);
448 next if $req_state eq 'freeze';
449 $self->queue_resource_command($sid, $sd->{uid
}, $req_state, $sd->{target
});
452 return $self->run_workers();
455 sub queue_resource_command
{
456 my ($self, $sid, $uid, $state, $target) = @_;
458 # do not queue the excatly same command twice as this may lead to
459 # an inconsistent HA state when the first command fails but the CRM
460 # does not process its failure right away and the LRM starts a second
461 # try, without the CRM knowing of it (race condition)
462 # The 'stopped' command is an exception as we do not process its result
463 # in the CRM and we want to execute it always (even with no active CRM)
464 return if $state ne 'stopped' && $uid && defined($self->{results
}->{$uid});
466 if (my $w = $self->{workers
}->{$sid}) {
467 return if $w->{pid
}; # already started
468 # else, delete and overwrite queue entry with new command
469 delete $self->{workers
}->{$sid};
472 $self->{workers
}->{$sid} = {
478 $self->{workers
}->{$sid}->{target
} = $target if $target;
481 sub check_active_workers
{
484 # finish/count workers
486 foreach my $sid (keys %{$self->{workers
}}) {
487 my $w = $self->{workers
}->{$sid};
488 if (my $pid = $w->{pid
}) {
490 my $waitpid = waitpid($pid, WNOHANG
);
491 if (defined($waitpid) && ($waitpid == $pid)) {
492 if (defined($w->{uid
})) {
493 $self->resource_command_finished($sid, $w->{uid
}, $?);
495 $self->stop_command_finished($sid, $?);
506 sub stop_command_finished
{
507 my ($self, $sid, $status) = @_;
509 my $haenv = $self->{haenv
};
511 my $w = delete $self->{workers
}->{$sid};
512 return if !$w; # should not happen
517 $haenv->log('err', "resource agent $sid finished - failed to execute");
518 } elsif (my $sig = ($status & 127)) {
519 $haenv->log('err', "resource agent $sid finished - got signal $sig");
521 $exit_code = ($status >> 8);
524 if ($exit_code != 0) {
525 $self->{shutdown_errors
}++;
529 sub resource_command_finished
{
530 my ($self, $sid, $uid, $status) = @_;
532 my $haenv = $self->{haenv
};
534 my $w = delete $self->{workers
}->{$sid};
535 return if !$w; # should not happen
540 $haenv->log('err', "resource agent $sid finished - failed to execute");
541 } elsif (my $sig = ($status & 127)) {
542 $haenv->log('err', "resource agent $sid finished - got signal $sig");
544 $exit_code = ($status >> 8);
547 $exit_code = $self->handle_service_exitcode($sid, $w->{state}, $exit_code);
549 return if $exit_code == ETRY_AGAIN
; # tell nobody, simply retry
551 $self->{results
}->{$uid} = {
553 state => $w->{state},
554 exit_code
=> $exit_code,
557 my $ss = $self->{service_status
};
559 # compute hash of valid/existing uids
561 foreach my $sid (keys %$ss) {
562 my $sd = $ss->{$sid};
564 $valid_uids->{$sd->{uid
}} = 1;
568 foreach my $id (keys %{$self->{results
}}) {
569 next if !$valid_uids->{$id};
570 $results->{$id} = $self->{results
}->{$id};
572 $self->{results
} = $results;
575 # processes the exit code from a finished resource agent, so that the CRM knows
576 # if the LRM wants to retry an action based on the current recovery policies for
577 # the failed service, or the CRM itself must try to recover from the failure.
578 sub handle_service_exitcode
{
579 my ($self, $sid, $cmd, $exit_code) = @_;
581 my $haenv = $self->{haenv
};
582 my $tries = $self->{restart_tries
};
584 my $sc = $haenv->read_service_config();
588 if (my $cd = $sc->{$sid}) {
589 $max_restart = $cd->{max_restart
};
592 if ($cmd eq 'started') {
594 if ($exit_code == SUCCESS
) {
600 } elsif ($exit_code == ERROR
) {
602 $tries->{$sid} = 0 if !defined($tries->{$sid});
604 if ($tries->{$sid} >= $max_restart) {
605 $haenv->log('err', "unable to start service $sid on local node".
606 " after $tries->{$sid} retries");
613 $haenv->log('warning', "restart policy: retry number $tries->{$sid}" .
614 " for service '$sid'");
615 # tell CRM that we retry the start
624 sub exec_resource_agent
{
625 my ($self, $sid, $service_config, $cmd, @params) = @_;
627 # setup execution environment
629 $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
631 my $haenv = $self->{haenv
};
633 my $nodename = $haenv->nodename();
635 my (undef, $service_type, $service_name) = PVE
::HA
::Tools
::parse_sid
($sid);
637 my $plugin = PVE
::HA
::Resources-
>lookup($service_type);
639 $haenv->log('err', "service type '$service_type' not implemented");
640 return EUNKNOWN_SERVICE_TYPE
;
643 if (!$service_config) {
644 $haenv->log('err', "missing resource configuration for '$sid'");
645 return EUNKNOWN_SERVICE
;
648 if ($service_config->{node
} ne $nodename) {
649 $haenv->log('err', "service '$sid' not on this node");
653 my $id = $service_name;
655 my $running = $plugin->check_running($haenv, $id);
657 if ($cmd eq 'started') {
659 return SUCCESS
if $running;
661 $haenv->log("info", "starting service $sid");
663 $plugin->start($haenv, $id);
665 $running = $plugin->check_running($haenv, $id);
668 $haenv->log("info", "service status $sid started");
671 $haenv->log("warning", "unable to start service $sid");
675 } elsif ($cmd eq 'request_stop' || $cmd eq 'stopped') {
677 return SUCCESS
if !$running;
679 $haenv->log("info", "stopping service $sid");
681 $plugin->shutdown($haenv, $id);
683 $running = $plugin->check_running($haenv, $id);
686 $haenv->log("info", "service status $sid stopped");
689 $haenv->log("info", "unable to stop stop service $sid (still running)");
693 } elsif ($cmd eq 'migrate' || $cmd eq 'relocate') {
695 my $target = $params[0];
696 if (!defined($target)) {
697 die "$cmd '$sid' failed - missing target\n" if !defined($target);
698 return EINVALID_PARAMETER
;
701 if ($service_config->{node
} eq $target) {
706 my $online = ($cmd eq 'migrate') ?
1 : 0;
708 my $res = $plugin->migrate($haenv, $id, $target, $online);
710 # something went wrong if service is still on this node
712 $haenv->log("err", "service $sid not moved (migration error)");
718 } elsif ($cmd eq 'error') {
721 $haenv->log("err", "service $sid is in an error state while running");
723 $haenv->log("warning", "service $sid is not running and in an error state");
725 return SUCCESS
; # error always succeeds
729 $haenv->log("err", "implement me (cmd '$cmd')");
730 return EUNKNOWN_COMMAND
;