]> git.proxmox.com Git - pve-ha-manager.git/blame - src/PVE/HA/LRM.pm
bump version to 3.3-1
[pve-ha-manager.git] / src / PVE / HA / LRM.pm
CommitLineData
5f095798
DM
1package PVE::HA::LRM;
2
3# Local Resource Manager
4
5use strict;
6use warnings;
c4a221bc 7use POSIX qw(:sys_wait_h);
5f095798
DM
8
9use PVE::SafeSyslog;
10use PVE::Tools;
a89ff919 11use PVE::HA::Tools ':exit_codes';
2a045f55 12use PVE::HA::Resources;
5f095798
DM
13
14# Server can have several states:
15
16my $valid_states = {
ec911edd 17 wait_for_agent_lock => "waiting for agent lock",
0bba8f60 18 active => "got agent_lock",
99278e06 19 maintenance => "going into maintenance",
5f095798
DM
20 lost_agent_lock => "lost agent_lock",
21};
22
21051707
TL
23# we sleep ~10s per 'active' round, so if no services is available for >= 10 min we'd go in wait
24# state givining up the watchdog and the LRM lock acquire voluntary, ensuring the WD can do no harm
25my $max_active_idle_rounds = 60;
26
5f095798
DM
27sub new {
28 my ($this, $haenv) = @_;
29
30 my $class = ref($this) || $this;
31
32 my $self = bless {
33 haenv => $haenv,
34 status => { state => 'startup' },
c4a221bc
DM
35 workers => {},
36 results => {},
ea4443cc 37 restart_tries => {},
067cdf33 38 shutdown_request => 0,
116dea30 39 shutdown_errors => 0,
9c7d068b
DM
40 # mode can be: active, reboot, shutdown, restart
41 mode => 'active',
3df15380 42 cluster_state_update => 0,
21051707 43 active_idle_rounds => 0,
5f095798
DM
44 }, $class;
45
289e4784 46 $self->set_local_status({ state => 'wait_for_agent_lock' });
9c7d068b 47
5f095798
DM
48 return $self;
49}
50
51sub shutdown_request {
52 my ($self) = @_;
53
f1be5b3a
DM
54 return if $self->{shutdown_request}; # already in shutdown mode
55
499f06e3
DM
56 my $haenv = $self->{haenv};
57
116dea30
DM
58 my $nodename = $haenv->nodename();
59
f65f41b9 60 my ($shutdown, $reboot) = $haenv->is_node_shutdown();
499f06e3 61
ba15a9b9
TL
62 my $dc_ha_cfg = $haenv->get_ha_settings();
63 my $shutdown_policy = $dc_ha_cfg->{shutdown_policy} // 'conditional';
64
7a20d688
TL
65 if ($shutdown) { # don't log this on service restart, only on node shutdown
66 $haenv->log('info', "got shutdown request with shutdown policy '$shutdown_policy'");
67 }
68
d2236278 69 my $freeze_all;
99278e06 70 my $maintenance;
ba15a9b9
TL
71 if ($shutdown_policy eq 'conditional') {
72 $freeze_all = $reboot;
73 } elsif ($shutdown_policy eq 'freeze') {
74 $freeze_all = 1;
75 } elsif ($shutdown_policy eq 'failover') {
76 $freeze_all = 0;
99278e06
TL
77 } elsif ($shutdown_policy eq 'migrate') {
78 $maintenance = 1;
ba15a9b9 79 } else {
d2236278
TL
80 $haenv->log('err', "unknown shutdown policy '$shutdown_policy', fall back to conditional");
81 $freeze_all = $reboot;
ba15a9b9
TL
82 }
83
99278e06
TL
84 if ($maintenance) {
85 # we get marked as unaivalable by the manager, then all services will
86 # be migrated away, we'll still have the same "can we exit" clause than
87 # a normal shutdown -> no running service on this node
88 # FIXME: after X minutes, add shutdown command for remaining services,
89 # e.g., if they have no alternative node???
90 } elsif ($shutdown) {
f65f41b9
TL
91 # *always* queue stop jobs for all services if the node shuts down,
92 # independent if it's a reboot or a poweroff, else we may corrupt
93 # services or hinder node shutdown
116dea30
DM
94 my $ss = $self->{service_status};
95
96 foreach my $sid (keys %$ss) {
97 my $sd = $ss->{$sid};
98 next if !$sd->{node};
99 next if $sd->{node} ne $nodename;
c0edbd7e 100 # Note: use undef uid to mark shutdown/stop jobs
116dea30
DM
101 $self->queue_resource_command($sid, undef, 'request_stop');
102 }
f65f41b9 103 }
116dea30 104
f65f41b9 105 if ($shutdown) {
41236dcf 106 my $shutdown_type = $reboot ? 'reboot' : 'shutdown';
99278e06
TL
107 if ($maintenance) {
108 $haenv->log('info', "$shutdown_type LRM, doing maintenance, removing this node from active list");
109 $self->{mode} = 'maintenance';
110 } elsif ($freeze_all) {
41236dcf 111 $haenv->log('info', "$shutdown_type LRM, stop and freeze all services");
f65f41b9
TL
112 $self->{mode} = 'restart';
113 } else {
114 $haenv->log('info', "shutdown LRM, stop all services");
115 $self->{mode} = 'shutdown';
116 }
499f06e3
DM
117 } else {
118 $haenv->log('info', "restart LRM, freeze all services");
119 $self->{mode} = 'restart';
120 }
9c7d068b 121
99278e06 122 $self->{shutdown_request} = $haenv->get_time();
9c7d068b 123
a19f2576 124 eval { $self->update_lrm_status() or die "not quorate?\n"; };
9c7d068b 125 if (my $err = $@) {
5bd7aa54 126 $self->log('err', "unable to update lrm status file - $err");
9c7d068b 127 }
5f095798
DM
128}
129
130sub get_local_status {
131 my ($self) = @_;
132
133 return $self->{status};
134}
135
136sub set_local_status {
137 my ($self, $new) = @_;
138
139 die "invalid state '$new->{state}'" if !$valid_states->{$new->{state}};
140
141 my $haenv = $self->{haenv};
142
143 my $old = $self->{status};
144
289e4784 145 # important: only update if if really changed
5f095798
DM
146 return if $old->{state} eq $new->{state};
147
0bba8f60 148 $haenv->log('info', "status change $old->{state} => $new->{state}");
5f095798
DM
149
150 $new->{state_change_time} = $haenv->get_time();
151
152 $self->{status} = $new;
153}
154
9c7d068b
DM
155sub update_lrm_status {
156 my ($self) = @_;
157
5bd7aa54
DM
158 my $haenv = $self->{haenv};
159
79829202 160 return 0 if !$haenv->quorate();
289e4784
TL
161
162 my $lrm_status = {
331a9f00 163 state => $self->{status}->{state},
9c7d068b
DM
164 mode => $self->{mode},
165 results => $self->{results},
aa330d1c 166 timestamp => $haenv->get_time(),
9c7d068b 167 };
289e4784 168
5bd7aa54
DM
169 eval { $haenv->write_lrm_status($lrm_status); };
170 if (my $err = $@) {
171 $haenv->log('err', "unable to write lrm status file - $err");
172 return 0;
173 }
174
175 return 1;
9c7d068b
DM
176}
177
8e940b68
TL
178sub update_service_status {
179 my ($self) = @_;
180
181 my $haenv = $self->{haenv};
182
183 my $ms = eval { $haenv->read_manager_status(); };
184 if (my $err = $@) {
185 $haenv->log('err', "updating service status from manager failed: $err");
186 return undef;
187 } else {
188 $self->{service_status} = $ms->{service_status} || {};
189 return 1;
190 }
191}
192
5f095798
DM
193sub get_protected_ha_agent_lock {
194 my ($self) = @_;
195
196 my $haenv = $self->{haenv};
197
198 my $count = 0;
199 my $starttime = $haenv->get_time();
200
201 for (;;) {
289e4784 202
5f095798
DM
203 if ($haenv->get_ha_agent_lock()) {
204 if ($self->{ha_agent_wd}) {
205 $haenv->watchdog_update($self->{ha_agent_wd});
206 } else {
207 my $wfh = $haenv->watchdog_open();
208 $self->{ha_agent_wd} = $wfh;
209 }
210 return 1;
211 }
289e4784 212
5f095798
DM
213 last if ++$count > 5; # try max 5 time
214
215 my $delay = $haenv->get_time() - $starttime;
216 last if $delay > 5; # for max 5 seconds
217
218 $haenv->sleep(1);
219 }
289e4784 220
5f095798
DM
221 return 0;
222}
223
21051707
TL
224# only cares if any service has the local node as their node, independent of which req.state it is
225sub has_configured_service_on_local_node {
226 my ($self) = @_;
227
228 my $haenv = $self->{haenv};
229 my $nodename = $haenv->nodename();
230
231 my $ss = $self->{service_status};
232 foreach my $sid (keys %$ss) {
233 my $sd = $ss->{$sid};
234 next if !$sd->{node} || $sd->{node} ne $nodename;
235
236 return 1;
237 }
238 return 0;
239}
240
546e2f1f
DM
241sub active_service_count {
242 my ($self) = @_;
289e4784 243
546e2f1f 244 my $haenv = $self->{haenv};
546e2f1f
DM
245 my $nodename = $haenv->nodename();
246
247 my $ss = $self->{service_status};
248
249 my $count = 0;
546e2f1f
DM
250 foreach my $sid (keys %$ss) {
251 my $sd = $ss->{$sid};
252 next if !$sd->{node};
253 next if $sd->{node} ne $nodename;
254 my $req_state = $sd->{state};
255 next if !defined($req_state);
256 next if $req_state eq 'stopped';
d54c04bd 257 # NOTE: 'ignored' ones are already dropped by the manager from service_status
9c7d068b 258 next if $req_state eq 'freeze';
38545741
TL
259 # erroneous services are not managed by HA, don't count them as active
260 next if $req_state eq 'error';
546e2f1f
DM
261
262 $count++;
263 }
289e4784 264
546e2f1f
DM
265 return $count;
266}
5bd7aa54
DM
267
268my $wrote_lrm_status_at_startup = 0;
269
5f095798
DM
270sub do_one_iteration {
271 my ($self) = @_;
272
273 my $haenv = $self->{haenv};
274
da6f0416
TL
275 $haenv->loop_start_hook();
276
3df15380
TL
277 $self->{cluster_state_update} = $haenv->cluster_state_update();
278
da6f0416
TL
279 my $res = $self->work();
280
281 $haenv->loop_end_hook();
282
283 return $res;
284}
285
abc1499b
TL
286# NOTE: this is disabling the self-fence mechanism, so it must NOT be called with active services
287# It's normally *only* OK on graceful shutdown (with no services, or all services frozen)
288my sub give_up_watchdog_protection {
289 my ($self) = @_;
290
291 if ($self->{ha_agent_wd}) {
292 $self->{haenv}->watchdog_close($self->{ha_agent_wd});
293 delete $self->{ha_agent_wd}; # only delete after close!
294 }
295}
296
da6f0416
TL
297sub work {
298 my ($self) = @_;
299
300 my $haenv = $self->{haenv};
301
c5ec095f 302 if (!$wrote_lrm_status_at_startup) {
79829202 303 if ($self->update_lrm_status()) {
c5ec095f
DM
304 $wrote_lrm_status_at_startup = 1;
305 } else {
306 # do nothing
307 $haenv->sleep(5);
308 return $self->{shutdown_request} ? 0 : 1;
309 }
5bd7aa54 310 }
289e4784 311
5f095798
DM
312 my $status = $self->get_local_status();
313 my $state = $status->{state};
314
8e940b68 315 $self->update_service_status();
067cdf33 316
49777d09 317 my $fence_request = PVE::HA::Tools::count_fenced_services($self->{service_status}, $haenv->nodename());
289e4784
TL
318
319 # do state changes first
5f095798
DM
320
321 my $ctime = $haenv->get_time();
322
b0bf08a9 323 if ($state eq 'wait_for_agent_lock') {
5f095798 324
546e2f1f 325 my $service_count = $self->active_service_count();
5f095798 326
067cdf33 327 if (!$fence_request && $service_count && $haenv->quorate()) {
0bba8f60
DM
328 if ($self->get_protected_ha_agent_lock()) {
329 $self->set_local_status({ state => 'active' });
5f095798
DM
330 }
331 }
289e4784 332
5f095798
DM
333 } elsif ($state eq 'lost_agent_lock') {
334
067cdf33 335 if (!$fence_request && $haenv->quorate()) {
0bba8f60
DM
336 if ($self->get_protected_ha_agent_lock()) {
337 $self->set_local_status({ state => 'active' });
5f095798
DM
338 }
339 }
340
0bba8f60 341 } elsif ($state eq 'active') {
5f095798 342
289e4784 343 if ($fence_request) {
067cdf33 344 $haenv->log('err', "node need to be fenced - releasing agent_lock\n");
289e4784 345 $self->set_local_status({ state => 'lost_agent_lock'});
067cdf33 346 } elsif (!$self->get_protected_ha_agent_lock()) {
5f095798 347 $self->set_local_status({ state => 'lost_agent_lock'});
99278e06
TL
348 } elsif ($self->{mode} eq 'maintenance') {
349 $self->set_local_status({ state => 'maintenance'});
21051707
TL
350 } else {
351 if (!$self->has_configured_service_on_local_node() && !$self->run_workers()) {
352 # no active service configured for this node and all (old) workers are done
353 $self->{active_idle_rounds}++;
354 if ($self->{active_idle_rounds} > $max_active_idle_rounds) {
355 $haenv->log('info', "node had no service configured for $max_active_idle_rounds rounds, going idle.\n");
356 # safety: no active service & no running worker for quite some time -> OK
357 $haenv->release_ha_agent_lock();
358 give_up_watchdog_protection($self);
359 $self->set_local_status({ state => 'wait_for_agent_lock'});
360 $self->{active_idle_rounds} = 0;
361 }
362 } elsif ($self->{active_idle_rounds}) {
363 $self->{active_idle_rounds} = 0;
364 }
99278e06
TL
365 }
366 } elsif ($state eq 'maintenance') {
367
368 if ($fence_request) {
369 $haenv->log('err', "node need to be fenced during maintenance mode - releasing agent_lock\n");
370 $self->set_local_status({ state => 'lost_agent_lock'});
371 } elsif (!$self->get_protected_ha_agent_lock()) {
372 $self->set_local_status({ state => 'lost_agent_lock'});
5f095798
DM
373 }
374 }
375
376 $status = $self->get_local_status();
377 $state = $status->{state};
378
379 # do work
380
381 if ($state eq 'wait_for_agent_lock') {
382
383 return 0 if $self->{shutdown_request};
289e4784 384
79829202 385 $self->update_lrm_status();
289e4784 386
5f095798 387 $haenv->sleep(5);
289e4784 388
0bba8f60 389 } elsif ($state eq 'active') {
5f095798
DM
390
391 my $startime = $haenv->get_time();
392
393 my $max_time = 10;
394
395 my $shutdown = 0;
396
397 # do work (max_time seconds)
398 eval {
399 # fixme: set alert timer
400
8e940b68
TL
401 # if we could not get the current service status there's no point
402 # in doing anything, try again next round.
403 return if !$self->update_service_status();
404
5f095798
DM
405 if ($self->{shutdown_request}) {
406
499f06e3 407 if ($self->{mode} eq 'restart') {
5f095798 408
499f06e3 409 my $service_count = $self->active_service_count();
5f095798 410
499f06e3 411 if ($service_count == 0) {
116dea30 412 if ($self->run_workers() == 0) {
abc1499b
TL
413 # safety: no active services or workers -> OK
414 give_up_watchdog_protection($self);
116dea30 415 $shutdown = 1;
e23f674c
TL
416
417 # restart with no or freezed services, release the lock
418 $haenv->release_ha_agent_lock();
116dea30
DM
419 }
420 }
421 } else {
422
423 if ($self->run_workers() == 0) {
424 if ($self->{shutdown_errors} == 0) {
abc1499b
TL
425 # safety: no active services and LRM shutdown -> OK
426 give_up_watchdog_protection($self);
0e5b1a43
TL
427
428 # shutdown with all services stopped thus release the lock
429 $haenv->release_ha_agent_lock();
499f06e3 430 }
5f095798 431
499f06e3
DM
432 $shutdown = 1;
433 }
5f095798 434 }
c4a221bc 435 } else {
724bd3f3
TL
436 if (!$self->{cluster_state_update}) {
437 # update failed but we could still renew our lock (cfs restart?),
438 # safely skip manage and expect to update just fine next round
439 $haenv->log('notice', "temporary inconsistent cluster state " .
440 "(cfs restart?), skip round");
441 return;
442 }
c4a221bc
DM
443
444 $self->manage_resources();
067cdf33 445
5f095798
DM
446 }
447 };
448 if (my $err = $@) {
449 $haenv->log('err', "got unexpected error - $err");
450 }
451
79829202 452 $self->update_lrm_status();
289e4784 453
5f095798
DM
454 return 0 if $shutdown;
455
456 $haenv->sleep_until($startime + $max_time);
457
458 } elsif ($state eq 'lost_agent_lock') {
289e4784 459
abc1499b 460 # NOTE: watchdog is active an will trigger soon!
5f095798 461 # so we hope to get the lock back soon!
5f095798
DM
462 if ($self->{shutdown_request}) {
463
546e2f1f 464 my $service_count = $self->active_service_count();
5f095798 465
546e2f1f 466 if ($service_count > 0) {
289e4784 467 $haenv->log('err', "get shutdown request in state 'lost_agent_lock' - " .
546e2f1f 468 "detected $service_count running services");
5f095798 469
c5c7faf6
TL
470 if ($self->{mode} eq 'restart') {
471 my $state_mt = $self->{status}->{state_change_time};
472
473 # watchdog should have already triggered, so either it's set
474 # set to noboot or it failed. As we are in restart mode, and
475 # have infinity stoptimeout -> exit now - we don't touch services
476 # or change state, so this is save, relatively speaking
477 if (($haenv->get_time() - $state_mt) > 90) {
478 $haenv->log('err', "lost agent lock and restart request for over 90 seconds - giving up!");
479 return 0;
480 }
481 }
546e2f1f 482 } else {
abc1499b
TL
483 # safety: all services are stopped, so we can close the watchdog
484 give_up_watchdog_protection($self);
289e4784 485
546e2f1f 486 return 0;
5f095798 487 }
5f095798
DM
488 }
489
b0bf08a9
DM
490 $haenv->sleep(5);
491
99278e06
TL
492 } elsif ($state eq 'maintenance') {
493
494 my $startime = $haenv->get_time();
495 return if !$self->update_service_status();
496
497 # wait until all active services moved away
498 my $service_count = $self->active_service_count();
499
500 my $exit_lrm = 0;
501
502 if ($self->{shutdown_request}) {
503 if ($service_count == 0 && $self->run_workers() == 0) {
abc1499b
TL
504 # safety: going into maintenance and all active services got moved -> OK
505 give_up_watchdog_protection($self);
99278e06
TL
506
507 $exit_lrm = 1;
508
509 # restart with no or freezed services, release the lock
510 $haenv->release_ha_agent_lock();
511 }
512 }
513
514 $self->manage_resources() if !$exit_lrm;
515
516 $self->update_lrm_status();
517
518 return 0 if $exit_lrm;
519
520 $haenv->sleep_until($startime + 5);
521
5f095798
DM
522 } else {
523
524 die "got unexpected status '$state'\n";
525
526 }
527
528 return 1;
529}
530
116dea30 531sub run_workers {
c4a221bc
DM
532 my ($self) = @_;
533
534 my $haenv = $self->{haenv};
535
f31b7e94 536 my $starttime = $haenv->get_time();
c4a221bc 537
a28fa330
TL
538 # number of workers to start, if 0 we exec the command directly witouth forking
539 my $max_workers = $haenv->get_max_workers();
c4a221bc 540
6dbf93a0 541 my $sc = $haenv->read_service_config();
f31b7e94
DM
542
543 while (($haenv->get_time() - $starttime) < 5) {
c4a221bc
DM
544 my $count = $self->check_active_workers();
545
a5e4bef4 546 foreach my $sid (sort keys %{$self->{workers}}) {
a28fa330
TL
547 last if $count >= $max_workers && $max_workers > 0;
548
c4a221bc
DM
549 my $w = $self->{workers}->{$sid};
550 if (!$w->{pid}) {
a28fa330
TL
551 # only fork if we may else call exec_resource_agent
552 # directly (e.g. for regression tests)
553 if ($max_workers > 0) {
f31b7e94
DM
554 my $pid = fork();
555 if (!defined($pid)) {
556 $haenv->log('err', "fork worker failed");
557 $count = 0; last; # abort, try later
558 } elsif ($pid == 0) {
a2aae08a
TL
559 $haenv->after_fork(); # cleanup
560
f31b7e94
DM
561 # do work
562 my $res = -1;
563 eval {
3ac1ee6b 564 $res = $self->exec_resource_agent($sid, $sc->{$sid}, $w->{state}, $w->{params});
f31b7e94
DM
565 };
566 if (my $err = $@) {
567 $haenv->log('err', $err);
568 POSIX::_exit(-1);
289e4784
TL
569 }
570 POSIX::_exit($res);
f31b7e94
DM
571 } else {
572 $count++;
573 $w->{pid} = $pid;
574 }
575 } else {
c4a221bc
DM
576 my $res = -1;
577 eval {
3ac1ee6b 578 $res = $self->exec_resource_agent($sid, $sc->{$sid}, $w->{state}, $w->{params});
b33b5743 579 $res = $res << 8 if $res > 0;
c4a221bc
DM
580 };
581 if (my $err = $@) {
f31b7e94 582 $haenv->log('err', $err);
116dea30
DM
583 }
584 if (defined($w->{uid})) {
585 $self->resource_command_finished($sid, $w->{uid}, $res);
586 } else {
587 $self->stop_command_finished($sid, $res);
588 }
c4a221bc
DM
589 }
590 }
591 }
592
593 last if !$count;
594
f31b7e94 595 $haenv->sleep(1);
c4a221bc 596 }
116dea30
DM
597
598 return scalar(keys %{$self->{workers}});
599}
600
601sub manage_resources {
602 my ($self) = @_;
603
604 my $haenv = $self->{haenv};
605
606 my $nodename = $haenv->nodename();
607
608 my $ss = $self->{service_status};
609
5a28da91
TL
610 foreach my $sid (keys %{$self->{restart_tries}}) {
611 delete $self->{restart_tries}->{$sid} if !$ss->{$sid};
612 }
613
116dea30
DM
614 foreach my $sid (keys %$ss) {
615 my $sd = $ss->{$sid};
616 next if !$sd->{node};
617 next if !$sd->{uid};
618 next if $sd->{node} ne $nodename;
619 my $req_state = $sd->{state};
620 next if !defined($req_state);
90a24755
TL
621 # can only happen for restricted groups where the failed node itself needs to be the
622 # reocvery target. Always let the master first do so, it will then marked as 'stopped' and
623 # we can just continue normally. But we must NOT do anything with it while still in recovery
624 next if $req_state eq 'recovery';
116dea30 625 next if $req_state eq 'freeze';
90a24755
TL
626
627 $self->queue_resource_command($sid, $sd->{uid}, $req_state, {
628 'target' => $sd->{target},
629 'timeout' => $sd->{timeout},
630 });
116dea30
DM
631 }
632
633 return $self->run_workers();
c4a221bc
DM
634}
635
c4a221bc 636sub queue_resource_command {
3ac1ee6b 637 my ($self, $sid, $uid, $state, $params) = @_;
c4a221bc 638
35cbb764
TL
639 # do not queue the excatly same command twice as this may lead to
640 # an inconsistent HA state when the first command fails but the CRM
641 # does not process its failure right away and the LRM starts a second
642 # try, without the CRM knowing of it (race condition)
643 # The 'stopped' command is an exception as we do not process its result
644 # in the CRM and we want to execute it always (even with no active CRM)
645 return if $state ne 'stopped' && $uid && defined($self->{results}->{$uid});
646
c4a221bc
DM
647 if (my $w = $self->{workers}->{$sid}) {
648 return if $w->{pid}; # already started
649 # else, delete and overwrite queue entry with new command
650 delete $self->{workers}->{$sid};
651 }
652
653 $self->{workers}->{$sid} = {
654 sid => $sid,
655 uid => $uid,
656 state => $state,
657 };
e88469ba 658
3ac1ee6b 659 $self->{workers}->{$sid}->{params} = $params if $params;
c4a221bc
DM
660}
661
662sub check_active_workers {
663 my ($self) = @_;
664
665 # finish/count workers
666 my $count = 0;
667 foreach my $sid (keys %{$self->{workers}}) {
668 my $w = $self->{workers}->{$sid};
669 if (my $pid = $w->{pid}) {
670 # check status
671 my $waitpid = waitpid($pid, WNOHANG);
672 if (defined($waitpid) && ($waitpid == $pid)) {
c0edbd7e 673 if (defined($w->{uid})) {
116dea30
DM
674 $self->resource_command_finished($sid, $w->{uid}, $?);
675 } else {
676 $self->stop_command_finished($sid, $?);
677 }
c4a221bc
DM
678 } else {
679 $count++;
680 }
681 }
682 }
289e4784 683
c4a221bc
DM
684 return $count;
685}
686
116dea30
DM
687sub stop_command_finished {
688 my ($self, $sid, $status) = @_;
689
690 my $haenv = $self->{haenv};
691
692 my $w = delete $self->{workers}->{$sid};
693 return if !$w; # should not happen
694
695 my $exit_code = -1;
696
697 if ($status == -1) {
698 $haenv->log('err', "resource agent $sid finished - failed to execute");
699 } elsif (my $sig = ($status & 127)) {
700 $haenv->log('err', "resource agent $sid finished - got signal $sig");
701 } else {
702 $exit_code = ($status >> 8);
703 }
704
705 if ($exit_code != 0) {
706 $self->{shutdown_errors}++;
707 }
708}
709
c4a221bc
DM
710sub resource_command_finished {
711 my ($self, $sid, $uid, $status) = @_;
712
713 my $haenv = $self->{haenv};
714
715 my $w = delete $self->{workers}->{$sid};
716 return if !$w; # should not happen
717
718 my $exit_code = -1;
719
720 if ($status == -1) {
289e4784 721 $haenv->log('err', "resource agent $sid finished - failed to execute");
c4a221bc 722 } elsif (my $sig = ($status & 127)) {
0f70400d 723 $haenv->log('err', "resource agent $sid finished - got signal $sig");
c4a221bc
DM
724 } else {
725 $exit_code = ($status >> 8);
c4a221bc
DM
726 }
727
ea4443cc
TL
728 $exit_code = $self->handle_service_exitcode($sid, $w->{state}, $exit_code);
729
280ee5d5
DM
730 return if $exit_code == ETRY_AGAIN; # tell nobody, simply retry
731
c4a221bc
DM
732 $self->{results}->{$uid} = {
733 sid => $w->{sid},
734 state => $w->{state},
735 exit_code => $exit_code,
736 };
737
738 my $ss = $self->{service_status};
739
740 # compute hash of valid/existing uids
741 my $valid_uids = {};
742 foreach my $sid (keys %$ss) {
743 my $sd = $ss->{$sid};
744 next if !$sd->{uid};
745 $valid_uids->{$sd->{uid}} = 1;
746 }
747
748 my $results = {};
749 foreach my $id (keys %{$self->{results}}) {
750 next if !$valid_uids->{$id};
751 $results->{$id} = $self->{results}->{$id};
752 }
753 $self->{results} = $results;
c4a221bc
DM
754}
755
ea4443cc
TL
756# processes the exit code from a finished resource agent, so that the CRM knows
757# if the LRM wants to retry an action based on the current recovery policies for
758# the failed service, or the CRM itself must try to recover from the failure.
759sub handle_service_exitcode {
760 my ($self, $sid, $cmd, $exit_code) = @_;
761
762 my $haenv = $self->{haenv};
763 my $tries = $self->{restart_tries};
764
765 my $sc = $haenv->read_service_config();
aaabde6a
DM
766
767 my $max_restart = 0;
768
769 if (my $cd = $sc->{$sid}) {
770 $max_restart = $cd->{max_restart};
771 }
ea4443cc
TL
772
773 if ($cmd eq 'started') {
774
a89ff919 775 if ($exit_code == SUCCESS) {
ea4443cc
TL
776
777 $tries->{$sid} = 0;
778
779 return $exit_code;
780
a89ff919 781 } elsif ($exit_code == ERROR) {
ea4443cc
TL
782
783 $tries->{$sid} = 0 if !defined($tries->{$sid});
784
aaabde6a 785 if ($tries->{$sid} >= $max_restart) {
ea4443cc
TL
786 $haenv->log('err', "unable to start service $sid on local node".
787 " after $tries->{$sid} retries");
788 $tries->{$sid} = 0;
a89ff919 789 return ERROR;
ea4443cc
TL
790 }
791
e9e1cd68
TL
792 $tries->{$sid}++;
793
794 $haenv->log('warning', "restart policy: retry number $tries->{$sid}" .
795 " for service '$sid'");
a89ff919
TL
796 # tell CRM that we retry the start
797 return ETRY_AGAIN;
ea4443cc
TL
798 }
799 }
800
801 return $exit_code;
802
803}
804
2a045f55 805sub exec_resource_agent {
3ac1ee6b 806 my ($self, $sid, $service_config, $cmd, $params) = @_;
2a045f55
TL
807
808 # setup execution environment
809
810 $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
811
2a045f55
TL
812 my $haenv = $self->{haenv};
813
814 my $nodename = $haenv->nodename();
815
0087839a 816 my (undef, $service_type, $service_name) = $haenv->parse_sid($sid);
2a045f55
TL
817
818 my $plugin = PVE::HA::Resources->lookup($service_type);
819 if (!$plugin) {
820 $haenv->log('err', "service type '$service_type' not implemented");
821 return EUNKNOWN_SERVICE_TYPE;
822 }
823
aaabde6a
DM
824 if (!$service_config) {
825 $haenv->log('err', "missing resource configuration for '$sid'");
826 return EUNKNOWN_SERVICE;
827 }
828
d338a56f
TL
829 # process error state early
830 if ($cmd eq 'error') {
d338a56f
TL
831 $haenv->log('err', "service $sid is in an error state and needs manual " .
832 "intervention. Look up 'ERROR RECOVERY' in the documentation.");
833
834 return SUCCESS; # error always succeeds
835 }
836
2a045f55
TL
837 if ($service_config->{node} ne $nodename) {
838 $haenv->log('err', "service '$sid' not on this node");
839 return EWRONG_NODE;
840 }
841
842 my $id = $service_name;
843
844 my $running = $plugin->check_running($haenv, $id);
845
846 if ($cmd eq 'started') {
847
848 return SUCCESS if $running;
849
850 $haenv->log("info", "starting service $sid");
851
852 $plugin->start($haenv, $id);
853
854 $running = $plugin->check_running($haenv, $id);
855
856 if ($running) {
857 $haenv->log("info", "service status $sid started");
858 return SUCCESS;
859 } else {
860 $haenv->log("warning", "unable to start service $sid");
861 return ERROR;
862 }
863
864 } elsif ($cmd eq 'request_stop' || $cmd eq 'stopped') {
865
866 return SUCCESS if !$running;
867
e4ef317d
FE
868 if (defined($params->{timeout})) {
869 $haenv->log("info", "stopping service $sid (timeout=$params->{timeout})");
870 } else {
871 $haenv->log("info", "stopping service $sid");
872 }
2a045f55 873
e4ef317d 874 $plugin->shutdown($haenv, $id, $params->{timeout});
2a045f55
TL
875
876 $running = $plugin->check_running($haenv, $id);
877
878 if (!$running) {
879 $haenv->log("info", "service status $sid stopped");
880 return SUCCESS;
881 } else {
882 $haenv->log("info", "unable to stop stop service $sid (still running)");
883 return ERROR;
884 }
885
886 } elsif ($cmd eq 'migrate' || $cmd eq 'relocate') {
887
3ac1ee6b 888 my $target = $params->{target};
2a045f55
TL
889 if (!defined($target)) {
890 die "$cmd '$sid' failed - missing target\n" if !defined($target);
891 return EINVALID_PARAMETER;
892 }
893
894 if ($service_config->{node} eq $target) {
895 # already there
896 return SUCCESS;
897 }
898
899 my $online = ($cmd eq 'migrate') ? 1 : 0;
900
ea28f873 901 my $res = $plugin->migrate($haenv, $id, $target, $online);
2a045f55
TL
902
903 # something went wrong if service is still on this node
ea28f873 904 if (!$res) {
2a045f55
TL
905 $haenv->log("err", "service $sid not moved (migration error)");
906 return ERROR;
907 }
908
909 return SUCCESS;
910
2a045f55
TL
911 }
912
913 $haenv->log("err", "implement me (cmd '$cmd')");
914 return EUNKNOWN_COMMAND;
915}
916
917
5f095798 9181;