X-Git-Url: https://git.proxmox.com/?p=pve-common.git;a=blobdiff_plain;f=data%2FPVE%2FDaemon.pm;h=264f8be5e2a840aa7969a47ef95fbda017d7cbc8;hp=6bd9c2966fd268af70ad76e5de8c954e03111bc6;hb=309d99e71818b7af5092152d02dff39894ecf432;hpb=c56111950d19a322262e0ea7b08856e2f0f9c35a diff --git a/data/PVE/Daemon.pm b/data/PVE/Daemon.pm index 6bd9c29..264f8be 100644 --- a/data/PVE/Daemon.pm +++ b/data/PVE/Daemon.pm @@ -5,18 +5,27 @@ package PVE::Daemon; # Features: # * lock and write PID file /var/run/$name.pid to make sure onyl # one instance is running. +# * keep lock open during restart # * correctly daemonize (redirect STDIN/STDOUT) # * restart by stop/start, exec, or signal HUP # * daemon restart on error (option 'restart_on_error') # * handle worker processes (option 'max_workers') - +# * allow to restart while workers are still runningl +# (option 'leave_children_open_on_reload') +# * run as different user using setuid/setgid + use strict; use warnings; +use English; + use PVE::SafeSyslog; use PVE::INotify; use POSIX ":sys_wait_h"; use Fcntl ':flock'; +use Socket qw(IPPROTO_TCP TCP_NODELAY SOMAXCONN); +use IO::Socket::INET; + use Getopt::Long; use Time::HiRes qw (gettimeofday); @@ -65,12 +74,15 @@ my $lockpidfile = sub { my $lkfn = $self->{pidfile} . ".lock"; + my $waittime = 0; + if (my $fd = $self->{env_pve_lock_fd}) { $self->{daemon_lock_fh} = IO::Handle->new_from_fd($fd, "a"); - + } else { + $waittime = 5; $self->{daemon_lock_fh} = IO::File->new(">>$lkfn"); } @@ -78,7 +90,7 @@ my $lockpidfile = sub { die "can't open lock '$lkfn' - $!\n"; } - for (my $i = 0; $i < 5; $i ++) { + for (my $i = 0; $i < $waittime; $i ++) { return if flock ($self->{daemon_lock_fh}, LOCK_EX|LOCK_NB); sleep(1); } @@ -117,11 +129,13 @@ my $server_cleanup = sub { my $finish_workers = sub { my ($self) = @_; - foreach my $cpid (keys %{$self->{workers}}) { - my $waitpid = waitpid($cpid, WNOHANG); - if (defined($waitpid) && ($waitpid == $cpid)) { - delete ($self->{workers}->{$cpid}); - syslog('info', "worker $cpid finished"); + foreach my $id (qw(workers old_workers)) { + foreach my $cpid (keys %{$self->{$id}}) { + my $waitpid = waitpid($cpid, WNOHANG); + if (defined($waitpid) && ($waitpid == $cpid)) { + delete ($self->{$id}->{$cpid}); + syslog('info', "worker $cpid finished"); + } } } }; @@ -170,7 +184,7 @@ my $start_workers = sub { }; my $terminate_server = sub { - my ($self) = @_; + my ($self, $allow_open_children) = @_; $self->{terminate} = 1; # set flag to avoid worker restart @@ -183,21 +197,32 @@ my $terminate_server = sub { eval { $self->shutdown(); }; warn $@ if $@; - # we have workers - terminate them + # we have workers - send TERM signal foreach my $cpid (keys %{$self->{workers}}) { kill(15, $cpid); # TERM childs } + # if configured, leave children running on HUP + return if $allow_open_children && + $self->{leave_children_open_on_reload}; + + # else, send TERM to old workers + foreach my $cpid (keys %{$self->{old_workers}}) { + kill(15, $cpid); # TERM childs + } + # nicely shutdown childs (give them max 10 seconds to shut down) my $previous_alarm = alarm(10); eval { local $SIG{ALRM} = sub { die "timeout\n" }; while ((my $pid = waitpid (-1, 0)) > 0) { - if (defined($self->{workers}->{$pid})) { - delete($self->{workers}->{$pid}); - syslog('info', "worker $pid finished"); + foreach my $id (qw(workers old_workers)) { + if (defined($self->{$id}->{$pid})) { + delete($self->{$id}->{$pid}); + syslog('info', "worker $pid finished"); + } } } alarm(0); # avoid race condition @@ -208,13 +233,15 @@ my $terminate_server = sub { if ($err) { syslog('err', "error stopping workers (will kill them now) - $err"); - foreach my $cpid (keys %{$self->{workers}}) { - # KILL childs still alive! - if (kill (0, $cpid)) { - delete($self->{workers}->{$cpid}); - syslog("err", "kill worker $cpid"); - kill(9, $cpid); - # fixme: waitpid? + foreach my $id (qw(workers old_workers)) { + foreach my $cpid (keys %{$self->{$id}}) { + # KILL childs still alive! + if (kill (0, $cpid)) { + delete($self->{$id}->{$cpid}); + syslog("err", "kill worker $cpid"); + kill(9, $cpid); + # fixme: waitpid? + } } } } @@ -269,7 +296,7 @@ my $server_run = sub { local $SIG{TERM} = sub { local ($@, $!, $?); # do not overwrite error vars syslog('info', "received signal TERM"); - &$terminate_server($self); + &$terminate_server($self, 0); &$server_cleanup($self); &$old_sig_term(@_) if $old_sig_term; }; @@ -278,7 +305,7 @@ my $server_run = sub { local $SIG{QUIT} = sub { local ($@, $!, $?); # do not overwrite error vars syslog('info', "received signal QUIT"); - &$terminate_server($self); + &$terminate_server($self, 0); &$server_cleanup($self); &$old_sig_quit(@_) if $old_sig_quit; }; @@ -288,7 +315,7 @@ my $server_run = sub { local ($@, $!, $?); # do not overwrite error vars syslog('info', "received signal INT"); $SIG{INT} = 'DEFAULT'; # allow to terminate now - &$terminate_server($self); + &$terminate_server($self, 0); &$server_cleanup($self); &$old_sig_int(@_) if $old_sig_int; }; @@ -296,9 +323,9 @@ my $server_run = sub { $SIG{HUP} = sub { local ($@, $!, $?); # do not overwrite error vars syslog('info', "received signal HUP"); + $self->{got_hup_signal} = 1; if ($self->{max_workers}) { - &$terminate_server($self); - $self->{got_hup_signal} = 1; + &$terminate_server($self, 1); } elsif ($self->can('hup')) { eval { $self->hup() }; warn $@ if $@; @@ -314,7 +341,11 @@ my $server_run = sub { &$old_sig_chld(@_) if $old_sig_chld; }; - for (;;) { # forever + # catch worker finished during restart phase + &$finish_workers($self); + + # now loop forever (until we receive terminate signal) + for (;;) { &$start_workers($self); sleep(5); &$finish_workers($self); @@ -330,7 +361,7 @@ my $server_run = sub { if ($err) { syslog ('err', "ERROR: $err"); - # fixme: kill all workers + &$terminate_server($self, 1); if (my $wait_time = $self->{restart_on_error}) { $self->restart_daemon($wait_time); @@ -349,60 +380,106 @@ my $server_run = sub { sub new { my ($this, $name, $cmdline, %params) = @_; - die "missing name" if !$name; + $name = 'daemon' if !$name; # should not happen initlog($name); - my $restart = $ENV{RESTART_PVE_DAEMON}; - delete $ENV{RESTART_PVE_DAEMON}; + my $self; - my $lockfd = $ENV{PVE_DAEMON_LOCK_FD}; - delete $ENV{PVE_DAEMON_LOCK_FD}; + eval { - die "please run as root\n" if !$restart && ($> != 0); + my $restart = $ENV{RESTART_PVE_DAEMON}; + delete $ENV{RESTART_PVE_DAEMON}; - die "can't create more that one PVE::Daemon" if $daemon_initialized; - $daemon_initialized = 1; + my $lockfd = $ENV{PVE_DAEMON_LOCK_FD}; + delete $ENV{PVE_DAEMON_LOCK_FD}; - PVE::INotify::inotify_init(); + if (defined($lockfd)) { + die "unable to parse lock fd '$lockfd'\n" + if $lockfd !~ m/^(\d+)$/; + $lockfd = $1; # untaint + } - my $class = ref($this) || $this; + die "please run as root\n" if !$restart && ($> != 0); - my $self = bless { - name => $name, - run_dir => '/var/run', - env_restart_pve_daemon => $restart, - env_pve_lock_fd => $lockfd, - workers => {}, - }, $class; + die "can't create more that one PVE::Daemon" if $daemon_initialized; + $daemon_initialized = 1; - foreach my $opt (keys %params) { - my $value = $params{$opt}; - if ($opt eq 'restart_on_error') { - $self->{$opt} = $value; - } elsif ($opt eq 'stop_wait_time') { - $self->{$opt} = $value; - } elsif ($opt eq 'run_dir') { - $self->{$opt} = $value; - } elsif ($opt eq 'max_workers') { - $self->{$opt} = $value; - } else { - die "unknown option '$opt'"; + PVE::INotify::inotify_init(); + + my $class = ref($this) || $this; + + $self = bless { + name => $name, + pidfile => "/var/run/${name}.pid", + env_restart_pve_daemon => $restart, + env_pve_lock_fd => $lockfd, + workers => {}, + old_workers => {}, + }, $class; + + + foreach my $opt (keys %params) { + my $value = $params{$opt}; + if ($opt eq 'restart_on_error') { + $self->{$opt} = $value; + } elsif ($opt eq 'stop_wait_time') { + $self->{$opt} = $value; + } elsif ($opt eq 'pidfile') { + $self->{$opt} = $value; + } elsif ($opt eq 'max_workers') { + $self->{$opt} = $value; + } elsif ($opt eq 'leave_children_open_on_reload') { + $self->{$opt} = $value; + } elsif ($opt eq 'setgid') { + $self->{$opt} = $value; + } elsif ($opt eq 'setuid') { + $self->{$opt} = $value; + } else { + die "unknown daemon option '$opt'\n"; + } + } + + if (my $gidstr = $self->{setgid}) { + my $gid = getgrnam($gidstr) || die "getgrnam failed - $!\n"; + POSIX::setgid($gid) || die "setgid $gid failed - $!\n"; + $EGID = "$gid $gid"; # this calls setgroups + # just to be sure + die "detected strange gid\n" if !($GID eq "$gid $gid" && $EGID eq "$gid $gid"); } - } - $self->{pidfile} = "$self->{run_dir}/${name}.pid"; + if (my $uidstr = $self->{setuid}) { + my $uid = getpwnam($uidstr) || die "getpwnam failed - $!\n"; + POSIX::setuid($uid) || die "setuid $uid failed - $!\n"; + # just to be sure + die "detected strange uid\n" if !($UID == $uid && $EUID == $uid); + } - $self->{nodename} = PVE::INotify::nodename(); + if ($restart && $self->{max_workers}) { + if (my $wpids = $ENV{PVE_DAEMON_WORKER_PIDS}) { + foreach my $pid (split(':', $wpids)) { + if ($pid =~ m/^(\d+)$/) { + $self->{old_workers}->{$1} = 1; + } + } + } + } - $self->{cmdline} = []; + $self->{nodename} = PVE::INotify::nodename(); - foreach my $el (@$cmdline) { - $el =~ m/^(.*)$/; # untaint - push @{$self->{cmdline}}, $1; - } + $self->{cmdline} = []; + + foreach my $el (@$cmdline) { + $el =~ m/^(.*)$/; # untaint + push @{$self->{cmdline}}, $1; + } - $0 = $name; + $0 = $name; + }; + if (my $err = $@) { + &$log_err($err); + exit(-1); + } return $self; } @@ -424,6 +501,12 @@ sub restart_daemon { $ENV{RESTART_PVE_DAEMON} = 1; + if ($self->{max_workers}) { + my @workers = keys %{$self->{workers}}; + push @workers, keys %{$self->{old_workers}}; + $ENV{PVE_DAEMON_WORKER_PIDS} = join(':', @workers); + } + sleep($waittime) if $waittime; # avoid high server load due to restarts PVE::INotify::inotify_close(); @@ -544,7 +627,9 @@ sub stop { } sub register_start_command { - my ($self, $class, $description) = @_; + my ($self, $description) = @_; + + my $class = ref($self); $class->register_method({ name => 'start', @@ -595,7 +680,9 @@ my $reload_daemon = sub { }; sub register_restart_command { - my ($self, $class, $use_hup, $description) = @_; + my ($self, $use_hup, $description) = @_; + + my $class = ref($self); $class->register_method({ name => 'restart', @@ -618,7 +705,9 @@ sub register_restart_command { } sub register_reload_command { - my ($self, $class, $description) = @_; + my ($self, $description) = @_; + + my $class = ref($self); $class->register_method({ name => 'reload', @@ -641,7 +730,9 @@ sub register_reload_command { } sub register_stop_command { - my ($self, $class, $description) = @_; + my ($self, $description) = @_; + + my $class = ref($self); $class->register_method({ name => 'stop', @@ -664,7 +755,9 @@ sub register_stop_command { } sub register_status_command { - my ($self, $class, $description) = @_; + my ($self, $description) = @_; + + my $class = ref($self); $class->register_method({ name => 'status', @@ -686,5 +779,49 @@ sub register_status_command { }}); } +# some useful helper + +sub create_reusable_socket { + my ($self, $port, $host) = @_; + + die "no port specifed" if !$port; + + my ($socket, $sockfd); + + if (defined($sockfd = $ENV{"PVE_DAEMON_SOCKET_$port"}) && + $self->{env_restart_pve_daemon}) { + + die "unable to parse socket fd '$sockfd'\n" + if $sockfd !~ m/^(\d+)$/; + $sockfd = $1; # untaint + + $socket = IO::Socket::INET->new; + $socket->fdopen($sockfd, 'w') || + die "cannot fdopen file descriptor '$sockfd' - $!\n"; + + } else { + + $socket = IO::Socket::INET->new( + LocalAddr => $host, + LocalPort => $port, + Listen => SOMAXCONN, + Proto => 'tcp', + ReuseAddr => 1) || + die "unable to create socket - $@\n"; + + # we often observe delays when using Nagle algorithm, + # so we disable that to maximize performance + setsockopt($socket, IPPROTO_TCP, TCP_NODELAY, 1); + + $ENV{"PVE_DAEMON_SOCKET_$port"} = $socket->fileno; + } + + # remove FD_CLOEXEC bit to reuse on exec + $socket->fcntl(Fcntl::F_SETFD(), 0); + + return $socket; +} + + 1;