Daemon: also terminate old_workers on stop
[pve-common.git] / data / PVE / Daemon.pm
1 package PVE::Daemon;
2
3 # Abstract class to implement Daemons
4 #
5 # Features:
6 # * lock and write PID file /var/run/$name.pid to make sure onyl
7 # one instance is running.
8 # * keep lock open during restart
9 # * correctly daemonize (redirect STDIN/STDOUT)
10 # * restart by stop/start, exec, or signal HUP
11 # * daemon restart on error (option 'restart_on_error')
12 # * handle worker processes (option 'max_workers')
13 # * allow to restart while workers are still runningl
14 # (option 'leave_children_open_on_reload')
15
16 use strict;
17 use warnings;
18 use PVE::SafeSyslog;
19 use PVE::INotify;
20
21 use POSIX ":sys_wait_h";
22 use Fcntl ':flock';
23 use Getopt::Long;
24 use Time::HiRes qw (gettimeofday);
25
26 use base qw(PVE::CLIHandler);
27
28 $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
29
30 my $daemon_initialized = 0; # we only allow one instance
31
32 my $close_daemon_lock = sub {
33 my ($self) = @_;
34
35 return if !$self->{daemon_lock_fh};
36
37 close $self->{daemon_lock_fh};
38 delete $self->{daemon_lock_fh};
39 };
40
41 my $log_err = sub {
42 my ($msg) = @_;
43 chomp $msg;
44 print STDERR "$msg\n";
45 syslog('err', "%s", $msg);
46 };
47
48 # call this if you fork() from child
49 # Note: we already call this for workers, so it is only required
50 # if you fork inside a simple daemon (max_workers == 0).
51 sub after_fork_cleanup {
52 my ($self) = @_;
53
54 &$close_daemon_lock($self);
55
56 PVE::INotify::inotify_close();
57
58 for my $sig (qw(CHLD HUP INT TERM QUIT)) {
59 $SIG{$sig} = 'DEFAULT'; # restore default handler
60 # AnyEvent signals only works if $SIG{XX} is
61 # undefined (perl event loop)
62 delete $SIG{$sig}; # so that we can handle events with AnyEvent
63 }
64 }
65
66 my $lockpidfile = sub {
67 my ($self) = @_;
68
69 my $lkfn = $self->{pidfile} . ".lock";
70
71 my $waittime = 0;
72
73 if (my $fd = $self->{env_pve_lock_fd}) {
74
75 $self->{daemon_lock_fh} = IO::Handle->new_from_fd($fd, "a");
76
77 } else {
78
79 $waittime = 5;
80 $self->{daemon_lock_fh} = IO::File->new(">>$lkfn");
81 }
82
83 if (!$self->{daemon_lock_fh}) {
84 die "can't open lock '$lkfn' - $!\n";
85 }
86
87 for (my $i = 0; $i < $waittime; $i ++) {
88 return if flock ($self->{daemon_lock_fh}, LOCK_EX|LOCK_NB);
89 sleep(1);
90 }
91
92 if (!flock ($self->{daemon_lock_fh}, LOCK_EX|LOCK_NB)) {
93 &$close_daemon_lock($self);
94 my $err = $!;
95
96 my ($running, $pid) = $self->running();
97 if ($running) {
98 die "can't aquire lock '$lkfn' - daemon already started (pid = $pid)\n";
99 } else {
100 die "can't aquire lock '$lkfn' - $err\n";
101 }
102 }
103 };
104
105 my $writepidfile = sub {
106 my ($self) = @_;
107
108 my $pidfile = $self->{pidfile};
109
110 die "can't open pid file '$pidfile' - $!\n" if !open (PIDFH, ">$pidfile");
111
112 print PIDFH "$$\n";
113 close (PIDFH);
114 };
115
116 my $server_cleanup = sub {
117 my ($self) = @_;
118
119 unlink $self->{pidfile} . ".lock";
120 unlink $self->{pidfile};
121 };
122
123 my $finish_workers = sub {
124 my ($self) = @_;
125
126 foreach my $id (qw(workers old_workers)) {
127 foreach my $cpid (keys %{$self->{$id}}) {
128 my $waitpid = waitpid($cpid, WNOHANG);
129 if (defined($waitpid) && ($waitpid == $cpid)) {
130 delete ($self->{$id}->{$cpid});
131 syslog('info', "worker $cpid finished");
132 }
133 }
134 }
135 };
136
137 my $start_workers = sub {
138 my ($self) = @_;
139
140 return if $self->{terminate};
141
142 my $count = 0;
143 foreach my $cpid (keys %{$self->{workers}}) {
144 $count++;
145 }
146
147 my $need = $self->{max_workers} - $count;
148
149 return if $need <= 0;
150
151 syslog('info', "starting $need worker(s)");
152
153 while ($need > 0) {
154 my $pid = fork;
155
156 if (!defined ($pid)) {
157 syslog('err', "can't fork worker");
158 sleep (1);
159 } elsif ($pid) { # parent
160 $self->{workers}->{$pid} = 1;
161 syslog('info', "worker $pid started");
162 $need--;
163 } else {
164 $0 = "$self->{name} worker";
165
166 $self->after_fork_cleanup();
167
168 eval { $self->run(); };
169 if (my $err = $@) {
170 syslog('err', $err);
171 sleep(5); # avoid fast restarts
172 }
173
174 syslog('info', "worker exit");
175 exit (0);
176 }
177 }
178 };
179
180 my $terminate_server = sub {
181 my ($self) = @_;
182
183 $self->{terminate} = 1; # set flag to avoid worker restart
184
185 if (!$self->{max_workers}) {
186 eval { $self->shutdown(); };
187 warn $@ if $@;
188 return;
189 }
190
191 eval { $self->shutdown(); };
192 warn $@ if $@;
193
194 # we have workers - send TERM signal
195
196 foreach my $cpid (keys %{$self->{workers}}) {
197 kill(15, $cpid); # TERM childs
198 }
199
200 # if configured, leave children running on HUP
201 return if $self->{got_hup_signal} &&
202 $self->{leave_children_open_on_reload};
203
204 # else, send TERM to old workers
205 foreach my $cpid (keys %{$self->{old_workers}}) {
206 kill(15, $cpid); # TERM childs
207 }
208
209 # nicely shutdown childs (give them max 10 seconds to shut down)
210 my $previous_alarm = alarm(10);
211 eval {
212 local $SIG{ALRM} = sub { die "timeout\n" };
213
214 while ((my $pid = waitpid (-1, 0)) > 0) {
215 foreach my $id (qw(workers old_workers)) {
216 if (defined($self->{$id}->{$pid})) {
217 delete($self->{$id}->{$pid});
218 syslog('info', "worker $pid finished");
219 }
220 }
221 }
222 alarm(0); # avoid race condition
223 };
224 my $err = $@;
225
226 alarm ($previous_alarm);
227
228 if ($err) {
229 syslog('err', "error stopping workers (will kill them now) - $err");
230 foreach my $id (qw(workers old_workers)) {
231 foreach my $cpid (keys %{$self->{$id}}) {
232 # KILL childs still alive!
233 if (kill (0, $cpid)) {
234 delete($self->{$id}->{$cpid});
235 syslog("err", "kill worker $cpid");
236 kill(9, $cpid);
237 # fixme: waitpid?
238 }
239 }
240 }
241 }
242 };
243
244 my $server_run = sub {
245 my ($self, $debug) = @_;
246
247 # fixme: handle restart lockfd
248 &$lockpidfile($self);
249
250 # remove FD_CLOEXEC bit to reuse on exec
251 $self->{daemon_lock_fh}->fcntl(Fcntl::F_SETFD(), 0);
252
253 $ENV{PVE_DAEMON_LOCK_FD} = $self->{daemon_lock_fh}->fileno;
254
255 # run in background
256 my $spid;
257
258 $self->{debug} = 1 if $debug;
259
260 $self->init();
261
262 if (!$debug) {
263 open STDIN, '</dev/null' || die "can't read /dev/null";
264 open STDOUT, '>/dev/null' || die "can't write /dev/null";
265 }
266
267 if (!$self->{env_restart_pve_daemon} && !$debug) {
268 PVE::INotify::inotify_close();
269 $spid = fork();
270 if (!defined ($spid)) {
271 die "can't put server into background - fork failed";
272 } elsif ($spid) { # parent
273 exit (0);
274 }
275 PVE::INotify::inotify_init();
276 }
277
278 if ($self->{env_restart_pve_daemon}) {
279 syslog('info' , "restarting server");
280 } else {
281 &$writepidfile($self);
282 syslog('info' , "starting server");
283 }
284
285 POSIX::setsid();
286
287 open STDERR, '>&STDOUT' || die "can't close STDERR\n";
288
289 my $old_sig_term = $SIG{TERM};
290 local $SIG{TERM} = sub {
291 local ($@, $!, $?); # do not overwrite error vars
292 syslog('info', "received signal TERM");
293 &$terminate_server($self);
294 &$server_cleanup($self);
295 &$old_sig_term(@_) if $old_sig_term;
296 };
297
298 my $old_sig_quit = $SIG{QUIT};
299 local $SIG{QUIT} = sub {
300 local ($@, $!, $?); # do not overwrite error vars
301 syslog('info', "received signal QUIT");
302 &$terminate_server($self);
303 &$server_cleanup($self);
304 &$old_sig_quit(@_) if $old_sig_quit;
305 };
306
307 my $old_sig_int = $SIG{INT};
308 local $SIG{INT} = sub {
309 local ($@, $!, $?); # do not overwrite error vars
310 syslog('info', "received signal INT");
311 $SIG{INT} = 'DEFAULT'; # allow to terminate now
312 &$terminate_server($self);
313 &$server_cleanup($self);
314 &$old_sig_int(@_) if $old_sig_int;
315 };
316
317 $SIG{HUP} = sub {
318 local ($@, $!, $?); # do not overwrite error vars
319 syslog('info', "received signal HUP");
320 $self->{got_hup_signal} = 1;
321 if ($self->{max_workers}) {
322 &$terminate_server($self);
323 } elsif ($self->can('hup')) {
324 eval { $self->hup() };
325 warn $@ if $@;
326 }
327 };
328
329 eval {
330 if ($self->{max_workers}) {
331 my $old_sig_chld = $SIG{CHLD};
332 local $SIG{CHLD} = sub {
333 local ($@, $!, $?); # do not overwrite error vars
334 &$finish_workers($self);
335 &$old_sig_chld(@_) if $old_sig_chld;
336 };
337
338 for (;;) { # forever
339 &$start_workers($self);
340 sleep(5);
341 &$finish_workers($self);
342 last if $self->{terminate};
343 }
344
345 } else {
346 $self->run();
347 }
348 };
349 my $err = $@;
350
351 if ($err) {
352 syslog ('err', "ERROR: $err");
353
354 &$terminate_server($self);
355
356 if (my $wait_time = $self->{restart_on_error}) {
357 $self->restart_daemon($wait_time);
358 } else {
359 $self->exit_daemon(-1);
360 }
361 }
362
363 if ($self->{got_hup_signal}) {
364 $self->restart_daemon();
365 } else {
366 $self->exit_daemon(0);
367 }
368 };
369
370 sub new {
371 my ($this, $name, $cmdline, %params) = @_;
372
373 $name = 'daemon' if !$name; # should not happen
374
375 initlog($name);
376
377 my $self;
378
379 eval {
380
381 my $restart = $ENV{RESTART_PVE_DAEMON};
382 delete $ENV{RESTART_PVE_DAEMON};
383
384 my $lockfd = $ENV{PVE_DAEMON_LOCK_FD};
385 delete $ENV{PVE_DAEMON_LOCK_FD};
386
387 if (defined($lockfd)) {
388 die "unable to parse lock fd '$lockfd'\n"
389 if $lockfd !~ m/^(\d+)$/;
390 $lockfd = $1; # untaint
391 }
392
393 die "please run as root\n" if !$restart && ($> != 0);
394
395 die "can't create more that one PVE::Daemon" if $daemon_initialized;
396 $daemon_initialized = 1;
397
398 PVE::INotify::inotify_init();
399
400 my $class = ref($this) || $this;
401
402 $self = bless {
403 name => $name,
404 run_dir => '/var/run',
405 env_restart_pve_daemon => $restart,
406 env_pve_lock_fd => $lockfd,
407 workers => {},
408 old_workers => {},
409 }, $class;
410
411 foreach my $opt (keys %params) {
412 my $value = $params{$opt};
413 if ($opt eq 'restart_on_error') {
414 $self->{$opt} = $value;
415 } elsif ($opt eq 'stop_wait_time') {
416 $self->{$opt} = $value;
417 } elsif ($opt eq 'run_dir') {
418 $self->{$opt} = $value;
419 } elsif ($opt eq 'max_workers') {
420 $self->{$opt} = $value;
421 } elsif ($opt eq 'leave_children_open_on_reload') {
422 $self->{$opt} = $value;
423 } else {
424 die "unknown daemon option '$opt'\n";
425 }
426 }
427
428 if ($restart && $self->{max_workers}) {
429 if (my $wpids = $ENV{PVE_DAEMON_WORKER_PIDS}) {
430 foreach my $pid (split(':', $wpids)) {
431 if ($pid =~ m/^(\d+)$/) {
432 $self->{old_workers}->{$1} = 1;
433 }
434 }
435 }
436 }
437
438 $self->{pidfile} = "$self->{run_dir}/${name}.pid";
439
440 $self->{nodename} = PVE::INotify::nodename();
441
442 $self->{cmdline} = [];
443
444 foreach my $el (@$cmdline) {
445 $el =~ m/^(.*)$/; # untaint
446 push @{$self->{cmdline}}, $1;
447 }
448
449 $0 = $name;
450 };
451 if (my $err = $@) {
452 &$log_err($err);
453 exit(-1);
454 }
455
456 return $self;
457 }
458
459 sub exit_daemon {
460 my ($self, $status) = @_;
461
462 syslog("info", "server stopped");
463
464 &$server_cleanup($self);
465
466 exit($status);
467 }
468
469 sub restart_daemon {
470 my ($self, $waittime) = @_;
471
472 syslog('info', "server shutdown (restart)");
473
474 $ENV{RESTART_PVE_DAEMON} = 1;
475
476 if ($self->{max_workers}) {
477 my @workers = keys %{$self->{workers}};
478 push @workers, keys %{$self->{old_workers}};
479 $ENV{PVE_DAEMON_WORKER_PIDS} = join(':', @workers);
480 }
481
482 sleep($waittime) if $waittime; # avoid high server load due to restarts
483
484 PVE::INotify::inotify_close();
485
486 exec (@{$self->{cmdline}});
487
488 exit (-1); # never reached?
489 }
490
491 # please overwrite in subclass
492 # this is called at startup - before forking
493 sub init {
494 my ($self) = @_;
495
496 }
497
498 # please overwrite in subclass
499 sub shutdown {
500 my ($self) = @_;
501
502 syslog('info' , "server closing");
503
504 if (!$self->{max_workers}) {
505 # wait for children
506 1 while (waitpid(-1, POSIX::WNOHANG()) > 0);
507 }
508 }
509
510 # please define in subclass
511 #sub hup {
512 # my ($self) = @_;
513 #
514 # syslog('info' , "received signal HUP (restart)");
515 #}
516
517 # please overwrite in subclass
518 sub run {
519 my ($self) = @_;
520
521 for (;;) { # forever
522 syslog('info' , "server is running");
523 sleep(5);
524 }
525 }
526
527 sub start {
528 my ($self, $debug) = @_;
529
530 eval { &$server_run($self, $debug); };
531 if (my $err = $@) {
532 &$log_err("start failed - $err");
533 exit(-1);
534 }
535 }
536
537 my $read_pid = sub {
538 my ($self) = @_;
539
540 my $pid_str = PVE::Tools::file_read_firstline($self->{pidfile});
541
542 return 0 if !$pid_str;
543
544 return 0 if $pid_str !~ m/^(\d+)$/; # untaint
545
546 my $pid = int($1);
547
548 return $pid;
549 };
550
551 sub running {
552 my ($self) = @_;
553
554 my $pid = &$read_pid($self);
555
556 if ($pid) {
557 my $res = PVE::ProcFSTools::check_process_running($pid) ? 1 : 0;
558 return wantarray ? ($res, $pid) : $res;
559 }
560
561 return wantarray ? (0, 0) : 0;
562 }
563
564 sub stop {
565 my ($self) = @_;
566
567 my $pid = &$read_pid($self);
568
569 return if !$pid;
570
571 if (PVE::ProcFSTools::check_process_running($pid)) {
572 kill(15, $pid); # send TERM signal
573 # give some time
574 my $wait_time = $self->{stop_wait_time} || 5;
575 my $running = 1;
576 for (my $i = 0; $i < $wait_time; $i++) {
577 $running = PVE::ProcFSTools::check_process_running($pid);
578 last if !$running;
579 sleep (1);
580 }
581
582 syslog('err', "server still running - send KILL") if $running;
583
584 # to be sure
585 kill(9, $pid);
586 waitpid($pid, 0);
587 }
588
589 if (-f $self->{pidfile}) {
590 eval {
591 # try to get the lock
592 &$lockpidfile($self);
593 &$server_cleanup($self);
594 };
595 if (my $err = $@) {
596 &$log_err("cleanup failed - $err");
597 }
598 }
599 }
600
601 sub register_start_command {
602 my ($self, $description) = @_;
603
604 my $class = ref($self);
605
606 $class->register_method({
607 name => 'start',
608 path => 'start',
609 method => 'POST',
610 description => $description || "Start the daemon.",
611 parameters => {
612 additionalProperties => 0,
613 properties => {
614 debug => {
615 description => "Debug mode - stay in foreground",
616 type => "boolean",
617 optional => 1,
618 default => 0,
619 },
620 },
621 },
622 returns => { type => 'null' },
623
624 code => sub {
625 my ($param) = @_;
626
627 $self->start($param->{debug});
628
629 return undef;
630 }});
631 }
632
633 my $reload_daemon = sub {
634 my ($self, $use_hup) = @_;
635
636 if ($self->{env_restart_pve_daemon}) {
637 $self->start();
638 } else {
639 my ($running, $pid) = $self->running();
640 if (!$running) {
641 $self->start();
642 } else {
643 if ($use_hup) {
644 syslog('info', "send HUP to $pid");
645 kill 1, $pid;
646 } else {
647 $self->stop();
648 $self->start();
649 }
650 }
651 }
652 };
653
654 sub register_restart_command {
655 my ($self, $use_hup, $description) = @_;
656
657 my $class = ref($self);
658
659 $class->register_method({
660 name => 'restart',
661 path => 'restart',
662 method => 'POST',
663 description => $description || "Restart the daemon (or start if not running).",
664 parameters => {
665 additionalProperties => 0,
666 properties => {},
667 },
668 returns => { type => 'null' },
669
670 code => sub {
671 my ($param) = @_;
672
673 &$reload_daemon($self, $use_hup);
674
675 return undef;
676 }});
677 }
678
679 sub register_reload_command {
680 my ($self, $description) = @_;
681
682 my $class = ref($self);
683
684 $class->register_method({
685 name => 'reload',
686 path => 'reload',
687 method => 'POST',
688 description => $description || "Reload daemon configuration (or start if not running).",
689 parameters => {
690 additionalProperties => 0,
691 properties => {},
692 },
693 returns => { type => 'null' },
694
695 code => sub {
696 my ($param) = @_;
697
698 &$reload_daemon($self, 1);
699
700 return undef;
701 }});
702 }
703
704 sub register_stop_command {
705 my ($self, $description) = @_;
706
707 my $class = ref($self);
708
709 $class->register_method({
710 name => 'stop',
711 path => 'stop',
712 method => 'POST',
713 description => $description || "Stop the daemon.",
714 parameters => {
715 additionalProperties => 0,
716 properties => {},
717 },
718 returns => { type => 'null' },
719
720 code => sub {
721 my ($param) = @_;
722
723 $self->stop();
724
725 return undef;
726 }});
727 }
728
729 sub register_status_command {
730 my ($self, $description) = @_;
731
732 my $class = ref($self);
733
734 $class->register_method({
735 name => 'status',
736 path => 'status',
737 method => 'GET',
738 description => "Get daemon status.",
739 parameters => {
740 additionalProperties => 0,
741 properties => {},
742 },
743 returns => {
744 type => 'string',
745 enum => ['stopped', 'running'],
746 },
747 code => sub {
748 my ($param) = @_;
749
750 return $self->running() ? 'running' : 'stopped';
751 }});
752 }
753
754 1;
755