fix #1819: fork_worker: ensure sync'ed workers control terminal
[pve-common.git] / src / PVE / Daemon.pm
1 package PVE::Daemon;
2
3 # Abstract class to implement Daemons
4 #
5 # Features:
6 # * lock and write PID file /var/run/$name.pid to make sure onyl
7 # one instance is running.
8 # * keep lock open during restart
9 # * correctly daemonize (redirect STDIN/STDOUT)
10 # * restart by stop/start, exec, or signal HUP
11 # * daemon restart on error (option 'restart_on_error')
12 # * handle worker processes (option 'max_workers')
13 # * allow to restart while workers are still runningl
14 # (option 'leave_children_open_on_reload')
15 # * run as different user using setuid/setgid
16
17 use strict;
18 use warnings;
19 use English;
20
21 use PVE::SafeSyslog;
22 use PVE::INotify;
23
24 use POSIX ":sys_wait_h";
25 use Fcntl ':flock';
26 use Socket qw(IPPROTO_TCP TCP_NODELAY SOMAXCONN);
27 use IO::Socket::IP;
28
29 use Getopt::Long;
30 use Time::HiRes qw (gettimeofday);
31
32 use base qw(PVE::CLIHandler);
33
34 $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
35
36 my $daemon_initialized = 0; # we only allow one instance
37 my $daemon_sockets = [];
38
39 my $close_daemon_lock = sub {
40 my ($self) = @_;
41
42 return if !$self->{daemon_lock_fh};
43
44 close $self->{daemon_lock_fh};
45 delete $self->{daemon_lock_fh};
46 };
47
48 my $log_err = sub {
49 my ($msg) = @_;
50 chomp $msg;
51 print STDERR "$msg\n";
52 syslog('err', "%s", $msg);
53 };
54
55 # call this if you fork() from child
56 # Note: we already call this for workers, so it is only required
57 # if you fork inside a simple daemon (max_workers == 0).
58 sub after_fork_cleanup {
59 my ($self) = @_;
60
61 &$close_daemon_lock($self);
62
63 PVE::INotify::inotify_close();
64
65 for my $sig (qw(CHLD HUP INT TERM QUIT)) {
66 $SIG{$sig} = 'DEFAULT'; # restore default handler
67 # AnyEvent signals only works if $SIG{XX} is
68 # undefined (perl event loop)
69 delete $SIG{$sig}; # so that we can handle events with AnyEvent
70 }
71 }
72
73 my $lockpidfile = sub {
74 my ($self) = @_;
75
76 my $lkfn = $self->{pidfile} . ".lock";
77
78 my $waittime = 0;
79
80 if (my $fd = $self->{env_pve_lock_fd}) {
81
82 $self->{daemon_lock_fh} = IO::Handle->new_from_fd($fd, "a");
83
84 } else {
85
86 $waittime = 5;
87 $self->{daemon_lock_fh} = IO::File->new(">>$lkfn");
88 }
89
90 if (!$self->{daemon_lock_fh}) {
91 die "can't open lock '$lkfn' - $!\n";
92 }
93
94 for (my $i = 0; $i < $waittime; $i ++) {
95 return if flock ($self->{daemon_lock_fh}, LOCK_EX|LOCK_NB);
96 sleep(1);
97 }
98
99 if (!flock ($self->{daemon_lock_fh}, LOCK_EX|LOCK_NB)) {
100 &$close_daemon_lock($self);
101 my $err = $!;
102
103 my ($running, $pid) = $self->running();
104 if ($running) {
105 die "can't aquire lock '$lkfn' - daemon already started (pid = $pid)\n";
106 } else {
107 die "can't aquire lock '$lkfn' - $err\n";
108 }
109 }
110 };
111
112 my $writepidfile = sub {
113 my ($self) = @_;
114
115 my $pidfile = $self->{pidfile};
116
117 die "can't open pid file '$pidfile' - $!\n" if !open (PIDFH, ">$pidfile");
118
119 print PIDFH "$$\n";
120 close (PIDFH);
121 };
122
123 my $server_cleanup = sub {
124 my ($self) = @_;
125
126 unlink $self->{pidfile} . ".lock";
127 unlink $self->{pidfile};
128 };
129
130 my $finish_workers = sub {
131 my ($self) = @_;
132
133 foreach my $id (qw(workers old_workers)) {
134 foreach my $cpid (keys %{$self->{$id}}) {
135 my $waitpid = waitpid($cpid, WNOHANG);
136 if (defined($waitpid) && ($waitpid == $cpid)) {
137 delete ($self->{$id}->{$cpid});
138 syslog('info', "worker $cpid finished");
139 }
140 }
141 }
142 };
143
144 my $start_workers = sub {
145 my ($self) = @_;
146
147 return if $self->{terminate};
148
149 my $count = scalar keys %{$self->{workers}};
150 my $need = $self->{max_workers} - $count;
151
152 return if $need <= 0;
153
154 syslog('info', "starting $need worker(s)");
155
156 while ($need > 0) {
157 my $pid = fork;
158
159 if (!defined ($pid)) {
160 syslog('err', "can't fork worker");
161 sleep (1);
162 } elsif ($pid) { # parent
163 $self->{workers}->{$pid} = 1;
164 syslog('info', "worker $pid started");
165 $need--;
166 } else {
167 $0 = "$self->{name} worker";
168
169 $self->after_fork_cleanup();
170
171 eval { $self->run(); };
172 if (my $err = $@) {
173 syslog('err', $err);
174 sleep(5); # avoid fast restarts
175 }
176
177 syslog('info', "worker exit");
178 exit (0);
179 }
180 }
181 };
182
183 my $terminate_old_workers = sub {
184 my ($self) = @_;
185
186 # if list is empty kill sends no signal, so no checks needed
187 kill 15, keys %{$self->{old_workers}};
188 };
189
190 my $terminate_server = sub {
191 my ($self, $allow_open_children) = @_;
192
193 $self->{terminate} = 1; # set flag to avoid worker restart
194
195 eval { $self->shutdown(); };
196 warn $@ if $@;
197
198 return if !$self->{max_workers}; # if we have no workers we're done here
199
200 # if configured, leave children running on HUP
201 return if $allow_open_children && $self->{leave_children_open_on_reload};
202
203 # else send TERM to all (old and current) child workers
204 kill 15, (keys %{$self->{workers}}, keys %{$self->{old_workers}});
205
206 # nicely shutdown childs (give them max 10 seconds to shut down)
207 my $previous_alarm = alarm(10);
208 eval {
209 local $SIG{ALRM} = sub { die "timeout\n" };
210
211 while ((my $pid = waitpid (-1, 0)) > 0) {
212 foreach my $id (qw(workers old_workers)) {
213 if (defined($self->{$id}->{$pid})) {
214 delete($self->{$id}->{$pid});
215 syslog('info', "worker $pid finished");
216 }
217 }
218 }
219 alarm(0); # avoid race condition
220 };
221 my $err = $@;
222
223 alarm ($previous_alarm);
224
225 if ($err) {
226 syslog('err', "error stopping workers (will kill them now) - $err");
227 foreach my $id (qw(workers old_workers)) {
228 foreach my $cpid (keys %{$self->{$id}}) {
229 # KILL childs still alive!
230 if (kill (0, $cpid)) {
231 delete($self->{$id}->{$cpid});
232 syslog("err", "kill worker $cpid");
233 kill(9, $cpid);
234 # fixme: waitpid?
235 }
236 }
237 }
238 }
239 };
240
241 sub setup {
242 my ($self) = @_;
243
244 initlog($self->{name});
245
246 my $restart = $ENV{RESTART_PVE_DAEMON};
247 delete $ENV{RESTART_PVE_DAEMON};
248 $self->{env_restart_pve_daemon} = $restart;
249
250 my $lockfd = $ENV{PVE_DAEMON_LOCK_FD};
251 delete $ENV{PVE_DAEMON_LOCK_FD};
252 if (defined($lockfd)) {
253 die "unable to parse lock fd '$lockfd'\n"
254 if $lockfd !~ m/^(\d+)$/;
255 $lockfd = $1; # untaint
256 }
257 $self->{env_pve_lock_fd} = $lockfd;
258
259 die "please run as root\n" if !$restart && ($> != 0);
260
261 die "can't create more that one PVE::Daemon" if $daemon_initialized;
262 $daemon_initialized = 1;
263
264 PVE::INotify::inotify_init();
265
266 if (my $gidstr = $self->{setgid}) {
267 my $gid = getgrnam($gidstr) || die "getgrnam failed - $!\n";
268 POSIX::setgid($gid) || die "setgid $gid failed - $!\n";
269 $EGID = "$gid $gid"; # this calls setgroups
270 # just to be sure
271 die "detected strange gid\n" if !($GID eq "$gid $gid" && $EGID eq "$gid $gid");
272 }
273
274 if (my $uidstr = $self->{setuid}) {
275 my $uid = getpwnam($uidstr) || die "getpwnam failed - $!\n";
276 POSIX::setuid($uid) || die "setuid $uid failed - $!\n";
277 # just to be sure
278 die "detected strange uid\n" if !($UID == $uid && $EUID == $uid);
279 }
280
281 if ($restart && $self->{max_workers}) {
282 if (my $wpids = $ENV{PVE_DAEMON_WORKER_PIDS}) {
283 foreach my $pid (split(':', $wpids)) {
284 # check & untaint
285 if ($pid =~ m/^(\d+)$/) {
286 $self->{old_workers}->{$1} = 1;
287 }
288 }
289 }
290 }
291
292 $self->{nodename} = PVE::INotify::nodename();
293 }
294
295 my $server_run = sub {
296 my ($self, $debug) = @_;
297
298 # fixme: handle restart lockfd
299 &$lockpidfile($self);
300
301 # remove FD_CLOEXEC bit to reuse on exec
302 $self->{daemon_lock_fh}->fcntl(Fcntl::F_SETFD(), 0);
303
304 $ENV{PVE_DAEMON_LOCK_FD} = $self->{daemon_lock_fh}->fileno;
305
306 # run in background
307 my $spid;
308
309 $self->{debug} = 1 if $debug;
310
311 $self->init();
312
313 if (!$debug) {
314 open STDIN, '</dev/null' || die "can't read /dev/null";
315 open STDOUT, '>/dev/null' || die "can't write /dev/null";
316 }
317
318 if (!$self->{env_restart_pve_daemon} && !$debug) {
319 PVE::INotify::inotify_close();
320 $spid = fork();
321 if (!defined ($spid)) {
322 die "can't put server into background - fork failed";
323 } elsif ($spid) { # parent
324 exit (0);
325 }
326 PVE::INotify::inotify_init();
327 }
328
329 if ($self->{env_restart_pve_daemon}) {
330 syslog('info' , "restarting server");
331 } else {
332 &$writepidfile($self);
333 syslog('info' , "starting server");
334 }
335
336 POSIX::setsid();
337
338 open STDERR, '>&STDOUT' || die "can't close STDERR\n";
339
340 my $old_sig_term = $SIG{TERM};
341 local $SIG{TERM} = sub {
342 local ($@, $!, $?); # do not overwrite error vars
343 syslog('info', "received signal TERM");
344 &$terminate_server($self, 0);
345 &$server_cleanup($self);
346 &$old_sig_term(@_) if $old_sig_term;
347 };
348
349 my $old_sig_quit = $SIG{QUIT};
350 local $SIG{QUIT} = sub {
351 local ($@, $!, $?); # do not overwrite error vars
352 syslog('info', "received signal QUIT");
353 &$terminate_server($self, 0);
354 &$server_cleanup($self);
355 &$old_sig_quit(@_) if $old_sig_quit;
356 };
357
358 my $old_sig_int = $SIG{INT};
359 local $SIG{INT} = sub {
360 local ($@, $!, $?); # do not overwrite error vars
361 syslog('info', "received signal INT");
362 $SIG{INT} = 'DEFAULT'; # allow to terminate now
363 &$terminate_server($self, 0);
364 &$server_cleanup($self);
365 &$old_sig_int(@_) if $old_sig_int;
366 };
367
368 $SIG{HUP} = sub {
369 local ($@, $!, $?); # do not overwrite error vars
370 syslog('info', "received signal HUP");
371 $self->{got_hup_signal} = 1;
372 if ($self->{max_workers}) {
373 &$terminate_server($self, 1);
374 } elsif ($self->can('hup')) {
375 eval { $self->hup() };
376 warn $@ if $@;
377 }
378 };
379
380 eval {
381 if ($self->{max_workers}) {
382 my $old_sig_chld = $SIG{CHLD};
383 local $SIG{CHLD} = sub {
384 local ($@, $!, $?); # do not overwrite error vars
385 &$finish_workers($self);
386 &$old_sig_chld(@_) if $old_sig_chld;
387 };
388
389 # now loop forever (until we receive terminate signal)
390 for (;;) {
391 &$start_workers($self);
392 sleep(5);
393 &$terminate_old_workers($self);
394 &$finish_workers($self);
395 last if $self->{terminate};
396 }
397
398 } else {
399 $self->run();
400 }
401 };
402 my $err = $@;
403
404 if ($err) {
405 syslog ('err', "ERROR: $err");
406
407 &$terminate_server($self, 1);
408
409 if (my $wait_time = $self->{restart_on_error}) {
410 $self->restart_daemon($wait_time);
411 } else {
412 $self->exit_daemon(-1);
413 }
414 }
415
416 if ($self->{got_hup_signal}) {
417 $self->restart_daemon();
418 } else {
419 $self->exit_daemon(0);
420 }
421 };
422
423 sub new {
424 my ($this, $name, $cmdline, %params) = @_;
425
426 $name = 'daemon' if !$name; # should not happen
427
428 my $self;
429
430 eval {
431 my $class = ref($this) || $this;
432
433 $self = bless {
434 name => $name,
435 pidfile => "/var/run/${name}.pid",
436 workers => {},
437 old_workers => {},
438 }, $class;
439
440
441 foreach my $opt (keys %params) {
442 my $value = $params{$opt};
443 if ($opt eq 'restart_on_error') {
444 $self->{$opt} = $value;
445 } elsif ($opt eq 'stop_wait_time') {
446 $self->{$opt} = $value;
447 } elsif ($opt eq 'pidfile') {
448 $self->{$opt} = $value;
449 } elsif ($opt eq 'max_workers') {
450 $self->{$opt} = $value;
451 } elsif ($opt eq 'leave_children_open_on_reload') {
452 $self->{$opt} = $value;
453 } elsif ($opt eq 'setgid') {
454 $self->{$opt} = $value;
455 } elsif ($opt eq 'setuid') {
456 $self->{$opt} = $value;
457 } else {
458 die "unknown daemon option '$opt'\n";
459 }
460 }
461
462
463 # untaint
464 $self->{cmdline} = [map { /^(.*)$/ } @$cmdline];
465
466 $0 = $name;
467 };
468 if (my $err = $@) {
469 &$log_err($err);
470 exit(-1);
471 }
472
473 return $self;
474 }
475
476 sub exit_daemon {
477 my ($self, $status) = @_;
478
479 syslog("info", "server stopped");
480
481 &$server_cleanup($self);
482
483 exit($status);
484 }
485
486 sub restart_daemon {
487 my ($self, $waittime) = @_;
488
489 syslog('info', "server shutdown (restart)");
490
491 $ENV{RESTART_PVE_DAEMON} = 1;
492
493 foreach my $ds (@$daemon_sockets) {
494 $ds->fcntl(Fcntl::F_SETFD(), 0);
495 }
496
497 if ($self->{max_workers}) {
498 my @workers = (keys %{$self->{workers}}, keys %{$self->{old_workers}});
499 $ENV{PVE_DAEMON_WORKER_PIDS} = join(':', @workers);
500 }
501
502 sleep($waittime) if $waittime; # avoid high server load due to restarts
503
504 PVE::INotify::inotify_close();
505
506 exec (@{$self->{cmdline}});
507
508 exit (-1); # never reached?
509 }
510
511 # please overwrite in subclass
512 # this is called at startup - before forking
513 sub init {
514 my ($self) = @_;
515
516 }
517
518 # please overwrite in subclass
519 sub shutdown {
520 my ($self) = @_;
521
522 syslog('info' , "server closing");
523
524 if (!$self->{max_workers}) {
525 # wait for children
526 1 while (waitpid(-1, POSIX::WNOHANG()) > 0);
527 }
528 }
529
530 # please define in subclass
531 #sub hup {
532 # my ($self) = @_;
533 #
534 # syslog('info' , "received signal HUP (restart)");
535 #}
536
537 # please overwrite in subclass
538 sub run {
539 my ($self) = @_;
540
541 for (;;) { # forever
542 syslog('info' , "server is running");
543 sleep(5);
544 }
545 }
546
547 sub start {
548 my ($self, $debug) = @_;
549
550 eval {
551 $self->setup();
552 &$server_run($self, $debug);
553 };
554 if (my $err = $@) {
555 &$log_err("start failed - $err");
556 exit(-1);
557 }
558 }
559
560 my $read_pid = sub {
561 my ($self) = @_;
562
563 my $pid_str = PVE::Tools::file_read_firstline($self->{pidfile});
564
565 return 0 if !$pid_str;
566
567 return 0 if $pid_str !~ m/^(\d+)$/; # untaint
568
569 my $pid = int($1);
570
571 return $pid;
572 };
573
574 # checks if the process was started by systemd
575 my $init_ppid = sub {
576
577 if (getppid() == 1) {
578 return 1;
579 } else {
580 return 0;
581 }
582 };
583
584 sub running {
585 my ($self) = @_;
586
587 my $pid = &$read_pid($self);
588
589 if ($pid) {
590 my $res = PVE::ProcFSTools::check_process_running($pid) ? 1 : 0;
591 return wantarray ? ($res, $pid) : $res;
592 }
593
594 return wantarray ? (0, 0) : 0;
595 }
596
597 sub stop {
598 my ($self) = @_;
599
600 my $pid = &$read_pid($self);
601
602 return if !$pid;
603
604 if (PVE::ProcFSTools::check_process_running($pid)) {
605 kill(15, $pid); # send TERM signal
606 # give some time
607 my $wait_time = $self->{stop_wait_time} || 5;
608 my $running = 1;
609 for (my $i = 0; $i < $wait_time; $i++) {
610 $running = PVE::ProcFSTools::check_process_running($pid);
611 last if !$running;
612 sleep (1);
613 }
614
615 syslog('err', "server still running - send KILL") if $running;
616
617 # to be sure
618 kill(9, $pid);
619 waitpid($pid, 0);
620 }
621
622 if (-f $self->{pidfile}) {
623 eval {
624 # try to get the lock
625 &$lockpidfile($self);
626 &$server_cleanup($self);
627 };
628 if (my $err = $@) {
629 &$log_err("cleanup failed - $err");
630 }
631 }
632 }
633
634 sub register_start_command {
635 my ($self, $description) = @_;
636
637 my $class = ref($self);
638
639 $class->register_method({
640 name => 'start',
641 path => 'start',
642 method => 'POST',
643 description => $description || "Start the daemon.",
644 parameters => {
645 additionalProperties => 0,
646 properties => {
647 debug => {
648 description => "Debug mode - stay in foreground",
649 type => "boolean",
650 optional => 1,
651 default => 0,
652 },
653 },
654 },
655 returns => { type => 'null' },
656
657 code => sub {
658 my ($param) = @_;
659
660 if (&$init_ppid() || $param->{debug}) {
661 $self->start($param->{debug});
662 } else {
663 PVE::Tools::run_command(['systemctl', 'start', $self->{name}]);
664 }
665
666 return undef;
667 }});
668 }
669
670 my $reload_daemon = sub {
671 my ($self, $use_hup) = @_;
672
673 if ($self->{env_restart_pve_daemon}) {
674 $self->start();
675 } else {
676 my ($running, $pid) = $self->running();
677 if (!$running) {
678 $self->start();
679 } else {
680 if ($use_hup) {
681 syslog('info', "send HUP to $pid");
682 kill 1, $pid;
683 } else {
684 $self->stop();
685 $self->start();
686 }
687 }
688 }
689 };
690
691 sub register_restart_command {
692 my ($self, $use_hup, $description) = @_;
693
694 my $class = ref($self);
695
696 $class->register_method({
697 name => 'restart',
698 path => 'restart',
699 method => 'POST',
700 description => $description || "Restart the daemon (or start if not running).",
701 parameters => {
702 additionalProperties => 0,
703 properties => {},
704 },
705 returns => { type => 'null' },
706
707 code => sub {
708 my ($param) = @_;
709
710 if (&$init_ppid()) {
711 &$reload_daemon($self, $use_hup);
712 } else {
713 PVE::Tools::run_command(['systemctl', $use_hup ? 'reload-or-restart' : 'restart', $self->{name}]);
714 }
715
716 return undef;
717 }});
718 }
719
720 sub register_reload_command {
721 my ($self, $description) = @_;
722
723 my $class = ref($self);
724
725 $class->register_method({
726 name => 'reload',
727 path => 'reload',
728 method => 'POST',
729 description => $description || "Reload daemon configuration (or start if not running).",
730 parameters => {
731 additionalProperties => 0,
732 properties => {},
733 },
734 returns => { type => 'null' },
735
736 code => sub {
737 my ($param) = @_;
738
739 &$reload_daemon($self, 1);
740
741 return undef;
742 }});
743 }
744
745 sub register_stop_command {
746 my ($self, $description) = @_;
747
748 my $class = ref($self);
749
750 $class->register_method({
751 name => 'stop',
752 path => 'stop',
753 method => 'POST',
754 description => $description || "Stop the daemon.",
755 parameters => {
756 additionalProperties => 0,
757 properties => {},
758 },
759 returns => { type => 'null' },
760
761 code => sub {
762 my ($param) = @_;
763
764 if (&$init_ppid()) {
765 $self->stop();
766 } else {
767 PVE::Tools::run_command(['systemctl', 'stop', $self->{name}]);
768 }
769
770 return undef;
771 }});
772 }
773
774 sub register_status_command {
775 my ($self, $description) = @_;
776
777 my $class = ref($self);
778
779 $class->register_method({
780 name => 'status',
781 path => 'status',
782 method => 'GET',
783 description => "Get daemon status.",
784 parameters => {
785 additionalProperties => 0,
786 properties => {},
787 },
788 returns => {
789 type => 'string',
790 enum => ['stopped', 'running'],
791 },
792 code => sub {
793 my ($param) = @_;
794
795 return $self->running() ? 'running' : 'stopped';
796 }});
797 }
798
799 # some useful helper
800
801 sub create_reusable_socket {
802 my ($self, $port, $host, $family) = @_;
803
804 die "no port specifed" if !$port;
805
806 my ($socket, $sockfd);
807
808 if (defined($sockfd = $ENV{"PVE_DAEMON_SOCKET_$port"}) &&
809 $self->{env_restart_pve_daemon}) {
810
811 die "unable to parse socket fd '$sockfd'\n"
812 if $sockfd !~ m/^(\d+)$/;
813 $sockfd = $1; # untaint
814
815 $socket = IO::Socket::IP->new;
816 $socket->fdopen($sockfd, 'w') ||
817 die "cannot fdopen file descriptor '$sockfd' - $!\n";
818
819 $socket->fcntl(Fcntl::F_SETFD(), Fcntl::FD_CLOEXEC);
820 } else {
821
822 $socket = IO::Socket::IP->new(
823 LocalAddr => $host,
824 LocalPort => $port,
825 Listen => SOMAXCONN,
826 Family => $family,
827 Proto => 'tcp',
828 GetAddrInfoFlags => 0,
829 ReuseAddr => 1) ||
830 die "unable to create socket - $@\n";
831
832 # we often observe delays when using Nagle algorithm,
833 # so we disable that to maximize performance
834 setsockopt($socket, IPPROTO_TCP, TCP_NODELAY, 1);
835
836 $ENV{"PVE_DAEMON_SOCKET_$port"} = $socket->fileno;
837 }
838
839 push @$daemon_sockets, $socket;
840
841 return $socket;
842 }
843
844
845 1;
846