X-Git-Url: https://git.proxmox.com/?p=pve-common.git;a=blobdiff_plain;f=data%2FPVE%2FDaemon.pm;h=264f8be5e2a840aa7969a47ef95fbda017d7cbc8;hp=3d214015ecbd0b329bd40d554abbd7709b706d80;hb=309d99e71818b7af5092152d02dff39894ecf432;hpb=1ea9e6df4542abcb17c6396bd0ea39b5693f6dc0 diff --git a/data/PVE/Daemon.pm b/data/PVE/Daemon.pm index 3d21401..264f8be 100644 --- a/data/PVE/Daemon.pm +++ b/data/PVE/Daemon.pm @@ -5,18 +5,27 @@ package PVE::Daemon; # Features: # * lock and write PID file /var/run/$name.pid to make sure onyl # one instance is running. +# * keep lock open during restart # * correctly daemonize (redirect STDIN/STDOUT) # * restart by stop/start, exec, or signal HUP # * daemon restart on error (option 'restart_on_error') # * handle worker processes (option 'max_workers') - +# * allow to restart while workers are still runningl +# (option 'leave_children_open_on_reload') +# * run as different user using setuid/setgid + use strict; use warnings; +use English; + use PVE::SafeSyslog; use PVE::INotify; use POSIX ":sys_wait_h"; use Fcntl ':flock'; +use Socket qw(IPPROTO_TCP TCP_NODELAY SOMAXCONN); +use IO::Socket::INET; + use Getopt::Long; use Time::HiRes qw (gettimeofday); @@ -120,11 +129,13 @@ my $server_cleanup = sub { my $finish_workers = sub { my ($self) = @_; - foreach my $cpid (keys %{$self->{workers}}) { - my $waitpid = waitpid($cpid, WNOHANG); - if (defined($waitpid) && ($waitpid == $cpid)) { - delete ($self->{workers}->{$cpid}); - syslog('info', "worker $cpid finished"); + foreach my $id (qw(workers old_workers)) { + foreach my $cpid (keys %{$self->{$id}}) { + my $waitpid = waitpid($cpid, WNOHANG); + if (defined($waitpid) && ($waitpid == $cpid)) { + delete ($self->{$id}->{$cpid}); + syslog('info', "worker $cpid finished"); + } } } }; @@ -173,7 +184,7 @@ my $start_workers = sub { }; my $terminate_server = sub { - my ($self) = @_; + my ($self, $allow_open_children) = @_; $self->{terminate} = 1; # set flag to avoid worker restart @@ -186,21 +197,32 @@ my $terminate_server = sub { eval { $self->shutdown(); }; warn $@ if $@; - # we have workers - terminate them + # we have workers - send TERM signal foreach my $cpid (keys %{$self->{workers}}) { kill(15, $cpid); # TERM childs } + # if configured, leave children running on HUP + return if $allow_open_children && + $self->{leave_children_open_on_reload}; + + # else, send TERM to old workers + foreach my $cpid (keys %{$self->{old_workers}}) { + kill(15, $cpid); # TERM childs + } + # nicely shutdown childs (give them max 10 seconds to shut down) my $previous_alarm = alarm(10); eval { local $SIG{ALRM} = sub { die "timeout\n" }; while ((my $pid = waitpid (-1, 0)) > 0) { - if (defined($self->{workers}->{$pid})) { - delete($self->{workers}->{$pid}); - syslog('info', "worker $pid finished"); + foreach my $id (qw(workers old_workers)) { + if (defined($self->{$id}->{$pid})) { + delete($self->{$id}->{$pid}); + syslog('info', "worker $pid finished"); + } } } alarm(0); # avoid race condition @@ -211,13 +233,15 @@ my $terminate_server = sub { if ($err) { syslog('err', "error stopping workers (will kill them now) - $err"); - foreach my $cpid (keys %{$self->{workers}}) { - # KILL childs still alive! - if (kill (0, $cpid)) { - delete($self->{workers}->{$cpid}); - syslog("err", "kill worker $cpid"); - kill(9, $cpid); - # fixme: waitpid? + foreach my $id (qw(workers old_workers)) { + foreach my $cpid (keys %{$self->{$id}}) { + # KILL childs still alive! + if (kill (0, $cpid)) { + delete($self->{$id}->{$cpid}); + syslog("err", "kill worker $cpid"); + kill(9, $cpid); + # fixme: waitpid? + } } } } @@ -272,7 +296,7 @@ my $server_run = sub { local $SIG{TERM} = sub { local ($@, $!, $?); # do not overwrite error vars syslog('info', "received signal TERM"); - &$terminate_server($self); + &$terminate_server($self, 0); &$server_cleanup($self); &$old_sig_term(@_) if $old_sig_term; }; @@ -281,7 +305,7 @@ my $server_run = sub { local $SIG{QUIT} = sub { local ($@, $!, $?); # do not overwrite error vars syslog('info', "received signal QUIT"); - &$terminate_server($self); + &$terminate_server($self, 0); &$server_cleanup($self); &$old_sig_quit(@_) if $old_sig_quit; }; @@ -291,7 +315,7 @@ my $server_run = sub { local ($@, $!, $?); # do not overwrite error vars syslog('info', "received signal INT"); $SIG{INT} = 'DEFAULT'; # allow to terminate now - &$terminate_server($self); + &$terminate_server($self, 0); &$server_cleanup($self); &$old_sig_int(@_) if $old_sig_int; }; @@ -299,9 +323,9 @@ my $server_run = sub { $SIG{HUP} = sub { local ($@, $!, $?); # do not overwrite error vars syslog('info', "received signal HUP"); + $self->{got_hup_signal} = 1; if ($self->{max_workers}) { - &$terminate_server($self); - $self->{got_hup_signal} = 1; + &$terminate_server($self, 1); } elsif ($self->can('hup')) { eval { $self->hup() }; warn $@ if $@; @@ -317,7 +341,11 @@ my $server_run = sub { &$old_sig_chld(@_) if $old_sig_chld; }; - for (;;) { # forever + # catch worker finished during restart phase + &$finish_workers($self); + + # now loop forever (until we receive terminate signal) + for (;;) { &$start_workers($self); sleep(5); &$finish_workers($self); @@ -333,7 +361,7 @@ my $server_run = sub { if ($err) { syslog ('err', "ERROR: $err"); - # fixme: kill all workers + &$terminate_server($self, 1); if (my $wait_time = $self->{restart_on_error}) { $self->restart_daemon($wait_time); @@ -366,6 +394,12 @@ sub new { my $lockfd = $ENV{PVE_DAEMON_LOCK_FD}; delete $ENV{PVE_DAEMON_LOCK_FD}; + if (defined($lockfd)) { + die "unable to parse lock fd '$lockfd'\n" + if $lockfd !~ m/^(\d+)$/; + $lockfd = $1; # untaint + } + die "please run as root\n" if !$restart && ($> != 0); die "can't create more that one PVE::Daemon" if $daemon_initialized; @@ -377,28 +411,59 @@ sub new { $self = bless { name => $name, - run_dir => '/var/run', + pidfile => "/var/run/${name}.pid", env_restart_pve_daemon => $restart, env_pve_lock_fd => $lockfd, workers => {}, + old_workers => {}, }, $class; + foreach my $opt (keys %params) { my $value = $params{$opt}; if ($opt eq 'restart_on_error') { $self->{$opt} = $value; } elsif ($opt eq 'stop_wait_time') { $self->{$opt} = $value; - } elsif ($opt eq 'run_dir') { + } elsif ($opt eq 'pidfile') { $self->{$opt} = $value; } elsif ($opt eq 'max_workers') { $self->{$opt} = $value; + } elsif ($opt eq 'leave_children_open_on_reload') { + $self->{$opt} = $value; + } elsif ($opt eq 'setgid') { + $self->{$opt} = $value; + } elsif ($opt eq 'setuid') { + $self->{$opt} = $value; } else { die "unknown daemon option '$opt'\n"; } } + + if (my $gidstr = $self->{setgid}) { + my $gid = getgrnam($gidstr) || die "getgrnam failed - $!\n"; + POSIX::setgid($gid) || die "setgid $gid failed - $!\n"; + $EGID = "$gid $gid"; # this calls setgroups + # just to be sure + die "detected strange gid\n" if !($GID eq "$gid $gid" && $EGID eq "$gid $gid"); + } + + if (my $uidstr = $self->{setuid}) { + my $uid = getpwnam($uidstr) || die "getpwnam failed - $!\n"; + POSIX::setuid($uid) || die "setuid $uid failed - $!\n"; + # just to be sure + die "detected strange uid\n" if !($UID == $uid && $EUID == $uid); + } - $self->{pidfile} = "$self->{run_dir}/${name}.pid"; + if ($restart && $self->{max_workers}) { + if (my $wpids = $ENV{PVE_DAEMON_WORKER_PIDS}) { + foreach my $pid (split(':', $wpids)) { + if ($pid =~ m/^(\d+)$/) { + $self->{old_workers}->{$1} = 1; + } + } + } + } $self->{nodename} = PVE::INotify::nodename(); @@ -436,6 +501,12 @@ sub restart_daemon { $ENV{RESTART_PVE_DAEMON} = 1; + if ($self->{max_workers}) { + my @workers = keys %{$self->{workers}}; + push @workers, keys %{$self->{old_workers}}; + $ENV{PVE_DAEMON_WORKER_PIDS} = join(':', @workers); + } + sleep($waittime) if $waittime; # avoid high server load due to restarts PVE::INotify::inotify_close(); @@ -556,7 +627,9 @@ sub stop { } sub register_start_command { - my ($self, $class, $description) = @_; + my ($self, $description) = @_; + + my $class = ref($self); $class->register_method({ name => 'start', @@ -607,7 +680,9 @@ my $reload_daemon = sub { }; sub register_restart_command { - my ($self, $class, $use_hup, $description) = @_; + my ($self, $use_hup, $description) = @_; + + my $class = ref($self); $class->register_method({ name => 'restart', @@ -630,7 +705,9 @@ sub register_restart_command { } sub register_reload_command { - my ($self, $class, $description) = @_; + my ($self, $description) = @_; + + my $class = ref($self); $class->register_method({ name => 'reload', @@ -653,7 +730,9 @@ sub register_reload_command { } sub register_stop_command { - my ($self, $class, $description) = @_; + my ($self, $description) = @_; + + my $class = ref($self); $class->register_method({ name => 'stop', @@ -676,7 +755,9 @@ sub register_stop_command { } sub register_status_command { - my ($self, $class, $description) = @_; + my ($self, $description) = @_; + + my $class = ref($self); $class->register_method({ name => 'status', @@ -698,5 +779,49 @@ sub register_status_command { }}); } +# some useful helper + +sub create_reusable_socket { + my ($self, $port, $host) = @_; + + die "no port specifed" if !$port; + + my ($socket, $sockfd); + + if (defined($sockfd = $ENV{"PVE_DAEMON_SOCKET_$port"}) && + $self->{env_restart_pve_daemon}) { + + die "unable to parse socket fd '$sockfd'\n" + if $sockfd !~ m/^(\d+)$/; + $sockfd = $1; # untaint + + $socket = IO::Socket::INET->new; + $socket->fdopen($sockfd, 'w') || + die "cannot fdopen file descriptor '$sockfd' - $!\n"; + + } else { + + $socket = IO::Socket::INET->new( + LocalAddr => $host, + LocalPort => $port, + Listen => SOMAXCONN, + Proto => 'tcp', + ReuseAddr => 1) || + die "unable to create socket - $@\n"; + + # we often observe delays when using Nagle algorithm, + # so we disable that to maximize performance + setsockopt($socket, IPPROTO_TCP, TCP_NODELAY, 1); + + $ENV{"PVE_DAEMON_SOCKET_$port"} = $socket->fileno; + } + + # remove FD_CLOEXEC bit to reuse on exec + $socket->fcntl(Fcntl::F_SETFD(), 0); + + return $socket; +} + + 1;