# Features:
# * lock and write PID file /var/run/$name.pid to make sure onyl
# one instance is running.
+# * keep lock open during restart
# * correctly daemonize (redirect STDIN/STDOUT)
# * restart by stop/start, exec, or signal HUP
# * daemon restart on error (option 'restart_on_error')
# * handle worker processes (option 'max_workers')
-
+# * allow to restart while workers are still runningl
+# (option 'leave_children_open_on_reload')
+
use strict;
use warnings;
use PVE::SafeSyslog;
use POSIX ":sys_wait_h";
use Fcntl ':flock';
+use Socket qw(IPPROTO_TCP TCP_NODELAY SOMAXCONN);
+use IO::Socket::INET;
+
use Getopt::Long;
use Time::HiRes qw (gettimeofday);
use base qw(PVE::CLIHandler);
-$SIG{'__WARN__'} = sub {
- my $err = $@;
- my $t = $_[0];
- chomp $t;
- print "$t\n";
- syslog('warning', "WARNING: %s", $t);
- $@ = $err;
-};
-
$ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
my $daemon_initialized = 0; # we only allow one instance
+my $close_daemon_lock = sub {
+ my ($self) = @_;
+
+ return if !$self->{daemon_lock_fh};
+
+ close $self->{daemon_lock_fh};
+ delete $self->{daemon_lock_fh};
+};
+
+my $log_err = sub {
+ my ($msg) = @_;
+ chomp $msg;
+ print STDERR "$msg\n";
+ syslog('err', "%s", $msg);
+};
+
+# call this if you fork() from child
+# Note: we already call this for workers, so it is only required
+# if you fork inside a simple daemon (max_workers == 0).
+sub after_fork_cleanup {
+ my ($self) = @_;
+
+ &$close_daemon_lock($self);
+
+ PVE::INotify::inotify_close();
+
+ for my $sig (qw(CHLD HUP INT TERM QUIT)) {
+ $SIG{$sig} = 'DEFAULT'; # restore default handler
+ # AnyEvent signals only works if $SIG{XX} is
+ # undefined (perl event loop)
+ delete $SIG{$sig}; # so that we can handle events with AnyEvent
+ }
+}
+
my $lockpidfile = sub {
my ($self) = @_;
my $lkfn = $self->{pidfile} . ".lock";
- if (!open (FLCK, ">>$lkfn")) {
- my $msg = "can't aquire lock on file '$lkfn' - $!";
- syslog ('err', $msg);
- die "ERROR: $msg\n";
+ my $waittime = 0;
+
+ if (my $fd = $self->{env_pve_lock_fd}) {
+
+ $self->{daemon_lock_fh} = IO::Handle->new_from_fd($fd, "a");
+
+ } else {
+
+ $waittime = 5;
+ $self->{daemon_lock_fh} = IO::File->new(">>$lkfn");
}
- if (!flock (FLCK, LOCK_EX|LOCK_NB)) {
- close (FLCK);
- my $msg = "can't aquire lock '$lkfn' - $!";
- syslog ('err', $msg);
- die "ERROR: $msg\n";
+ if (!$self->{daemon_lock_fh}) {
+ die "can't open lock '$lkfn' - $!\n";
+ }
+
+ for (my $i = 0; $i < $waittime; $i ++) {
+ return if flock ($self->{daemon_lock_fh}, LOCK_EX|LOCK_NB);
+ sleep(1);
+ }
+
+ if (!flock ($self->{daemon_lock_fh}, LOCK_EX|LOCK_NB)) {
+ &$close_daemon_lock($self);
+ my $err = $!;
+
+ my ($running, $pid) = $self->running();
+ if ($running) {
+ die "can't aquire lock '$lkfn' - daemon already started (pid = $pid)\n";
+ } else {
+ die "can't aquire lock '$lkfn' - $err\n";
+ }
}
};
my $pidfile = $self->{pidfile};
- if (!open (PIDFH, ">$pidfile")) {
- my $msg = "can't open pid file '$pidfile' - $!";
- syslog ('err', $msg);
- die "ERROR: $msg\n";
- }
+ die "can't open pid file '$pidfile' - $!\n" if !open (PIDFH, ">$pidfile");
+
print PIDFH "$$\n";
close (PIDFH);
};
my $finish_workers = sub {
my ($self) = @_;
- foreach my $cpid (keys %{$self->{workers}}) {
- my $waitpid = waitpid($cpid, WNOHANG);
- if (defined($waitpid) && ($waitpid == $cpid)) {
- delete ($self->{workers}->{$cpid});
- syslog('info', "worker $cpid finished");
+ foreach my $id (qw(workers old_workers)) {
+ foreach my $cpid (keys %{$self->{$id}}) {
+ my $waitpid = waitpid($cpid, WNOHANG);
+ if (defined($waitpid) && ($waitpid == $cpid)) {
+ delete ($self->{$id}->{$cpid});
+ syslog('info', "worker $cpid finished");
+ }
}
}
};
} else {
$0 = "$self->{name} worker";
- close(FLCK);
-
- PVE::INotify::inotify_close();
-
- for my $sig (qw(CHLD HUP INT TERM QUIT)) {
- $SIG{$sig} = 'DEFAULT'; # restore default handler
- # AnyEvent signals only works if $SIG{XX} is
- # undefined (perl event loop)
- delete $SIG{$sig}; # so that we can handle events with AnyEvent
- }
+ $self->after_fork_cleanup();
eval { $self->run(); };
if (my $err = $@) {
eval { $self->shutdown(); };
warn $@ if $@;
- # we have workers - terminate them
+ # we have workers - send TERM signal
foreach my $cpid (keys %{$self->{workers}}) {
kill(15, $cpid); # TERM childs
}
+ # if configured, leave children running on HUP
+ return if $self->{got_hup_signal} &&
+ $self->{leave_children_open_on_reload};
+
+ # else, send TERM to old workers
+ foreach my $cpid (keys %{$self->{old_workers}}) {
+ kill(15, $cpid); # TERM childs
+ }
+
# nicely shutdown childs (give them max 10 seconds to shut down)
my $previous_alarm = alarm(10);
eval {
local $SIG{ALRM} = sub { die "timeout\n" };
while ((my $pid = waitpid (-1, 0)) > 0) {
- if (defined($self->{workers}->{$pid})) {
- delete($self->{workers}->{$pid});
- syslog('info', "worker $pid finished");
+ foreach my $id (qw(workers old_workers)) {
+ if (defined($self->{$id}->{$pid})) {
+ delete($self->{$id}->{$pid});
+ syslog('info', "worker $pid finished");
+ }
}
}
alarm(0); # avoid race condition
if ($err) {
syslog('err', "error stopping workers (will kill them now) - $err");
- foreach my $cpid (keys %{$self->{workers}}) {
- # KILL childs still alive!
- if (kill (0, $cpid)) {
- delete($self->{workers}->{$cpid});
- syslog("err", "kill worker $cpid");
- kill(9, $cpid);
- # fixme: waitpid?
+ foreach my $id (qw(workers old_workers)) {
+ foreach my $cpid (keys %{$self->{$id}}) {
+ # KILL childs still alive!
+ if (kill (0, $cpid)) {
+ delete($self->{$id}->{$cpid});
+ syslog("err", "kill worker $cpid");
+ kill(9, $cpid);
+ # fixme: waitpid?
+ }
}
}
}
my $server_run = sub {
my ($self, $debug) = @_;
+ # fixme: handle restart lockfd
&$lockpidfile($self);
- # run in background
- my $spid;
+ # remove FD_CLOEXEC bit to reuse on exec
+ $self->{daemon_lock_fh}->fcntl(Fcntl::F_SETFD(), 0);
- my $restart = $ENV{RESTART_PVE_DAEMON};
+ $ENV{PVE_DAEMON_LOCK_FD} = $self->{daemon_lock_fh}->fileno;
- delete $ENV{RESTART_PVE_DAEMON};
+ # run in background
+ my $spid;
$self->{debug} = 1 if $debug;
open STDOUT, '>/dev/null' || die "can't write /dev/null";
}
- if (!$restart && !$debug) {
+ if (!$self->{env_restart_pve_daemon} && !$debug) {
PVE::INotify::inotify_close();
$spid = fork();
if (!defined ($spid)) {
- my $msg = "can't put server into background - fork failed";
- syslog('err', $msg);
- die "ERROR: $msg\n";
+ die "can't put server into background - fork failed";
} elsif ($spid) { # parent
exit (0);
}
PVE::INotify::inotify_init();
}
- &$writepidfile($self);
-
- POSIX::setsid();
-
- if ($restart) {
+ if ($self->{env_restart_pve_daemon}) {
syslog('info' , "restarting server");
} else {
+ &$writepidfile($self);
syslog('info' , "starting server");
}
+ POSIX::setsid();
+
open STDERR, '>&STDOUT' || die "can't close STDERR\n";
my $old_sig_term = $SIG{TERM};
$SIG{HUP} = sub {
local ($@, $!, $?); # do not overwrite error vars
syslog('info', "received signal HUP");
+ $self->{got_hup_signal} = 1;
if ($self->{max_workers}) {
&$terminate_server($self);
- $self->{got_hup_signal} = 1;
} elsif ($self->can('hup')) {
eval { $self->hup() };
warn $@ if $@;
&$old_sig_chld(@_) if $old_sig_chld;
};
- for (;;) { # forever
+ # catch worker finished during restart phase
+ &$finish_workers($self);
+
+ # now loop forever (until we receive terminate signal)
+ for (;;) {
&$start_workers($self);
sleep(5);
&$finish_workers($self);
if ($err) {
syslog ('err', "ERROR: $err");
- # fixme: kill all workers
+ &$terminate_server($self);
if (my $wait_time = $self->{restart_on_error}) {
$self->restart_daemon($wait_time);
sub new {
my ($this, $name, $cmdline, %params) = @_;
- die "please run as root\n" if !$ENV{RESTART_PVE_DAEMON} && ($> != 0);
+ $name = 'daemon' if !$name; # should not happen
- die "missing name" if !$name;
+ initlog($name);
- die "can't create more that one PVE::Daemon" if $daemon_initialized;
- $daemon_initialized = 1;
+ my $self;
- PVE::INotify::inotify_init();
+ eval {
- initlog($name);
+ my $restart = $ENV{RESTART_PVE_DAEMON};
+ delete $ENV{RESTART_PVE_DAEMON};
- my $class = ref($this) || $this;
-
- my $self = bless {
- name => $name,
- run_dir => '/var/run',
- workers => {},
- }, $class;
-
- foreach my $opt (keys %params) {
- my $value = $params{$opt};
- if ($opt eq 'restart_on_error') {
- $self->{$opt} = $value;
- } elsif ($opt eq 'stop_wait_time') {
- $self->{$opt} = $value;
- } elsif ($opt eq 'run_dir') {
- $self->{$opt} = $value;
- } elsif ($opt eq 'max_workers') {
- $self->{$opt} = $value;
- } else {
- die "unknown option '$opt'";
+ my $lockfd = $ENV{PVE_DAEMON_LOCK_FD};
+ delete $ENV{PVE_DAEMON_LOCK_FD};
+
+ if (defined($lockfd)) {
+ die "unable to parse lock fd '$lockfd'\n"
+ if $lockfd !~ m/^(\d+)$/;
+ $lockfd = $1; # untaint
}
- }
- $self->{pidfile} = "$self->{run_dir}/${name}.pid";
+ die "please run as root\n" if !$restart && ($> != 0);
- $self->{nodename} = PVE::INotify::nodename();
+ die "can't create more that one PVE::Daemon" if $daemon_initialized;
+ $daemon_initialized = 1;
- $self->{cmdline} = [];
+ PVE::INotify::inotify_init();
- foreach my $el (@$cmdline) {
- $el =~ m/^(.*)$/; # untaint
- push @{$self->{cmdline}}, $1;
- }
+ my $class = ref($this) || $this;
+
+ $self = bless {
+ name => $name,
+ run_dir => '/var/run',
+ env_restart_pve_daemon => $restart,
+ env_pve_lock_fd => $lockfd,
+ workers => {},
+ old_workers => {},
+ }, $class;
+
+ foreach my $opt (keys %params) {
+ my $value = $params{$opt};
+ if ($opt eq 'restart_on_error') {
+ $self->{$opt} = $value;
+ } elsif ($opt eq 'stop_wait_time') {
+ $self->{$opt} = $value;
+ } elsif ($opt eq 'run_dir') {
+ $self->{$opt} = $value;
+ } elsif ($opt eq 'max_workers') {
+ $self->{$opt} = $value;
+ } elsif ($opt eq 'leave_children_open_on_reload') {
+ $self->{$opt} = $value;
+ } else {
+ die "unknown daemon option '$opt'\n";
+ }
+ }
+
+ if ($restart && $self->{max_workers}) {
+ if (my $wpids = $ENV{PVE_DAEMON_WORKER_PIDS}) {
+ foreach my $pid (split(':', $wpids)) {
+ if ($pid =~ m/^(\d+)$/) {
+ $self->{old_workers}->{$1} = 1;
+ }
+ }
+ }
+ }
+
+ $self->{pidfile} = "$self->{run_dir}/${name}.pid";
+
+ $self->{nodename} = PVE::INotify::nodename();
- $0 = $name;
+ $self->{cmdline} = [];
+
+ foreach my $el (@$cmdline) {
+ $el =~ m/^(.*)$/; # untaint
+ push @{$self->{cmdline}}, $1;
+ }
+
+ $0 = $name;
+ };
+ if (my $err = $@) {
+ &$log_err($err);
+ exit(-1);
+ }
return $self;
}
$ENV{RESTART_PVE_DAEMON} = 1;
+ if ($self->{max_workers}) {
+ my @workers = keys %{$self->{workers}};
+ push @workers, keys %{$self->{old_workers}};
+ $ENV{PVE_DAEMON_WORKER_PIDS} = join(':', @workers);
+ }
+
sleep($waittime) if $waittime; # avoid high server load due to restarts
PVE::INotify::inotify_close();
sub start {
my ($self, $debug) = @_;
- &$server_run($self, $debug);
+ eval { &$server_run($self, $debug); };
+ if (my $err = $@) {
+ &$log_err("start failed - $err");
+ exit(-1);
+ }
}
my $read_pid = sub {
}
if (-f $self->{pidfile}) {
- # try to get the lock
- &$lockpidfile($self);
- &$server_cleanup($self);
+ eval {
+ # try to get the lock
+ &$lockpidfile($self);
+ &$server_cleanup($self);
+ };
+ if (my $err = $@) {
+ &$log_err("cleanup failed - $err");
+ }
}
}
sub register_start_command {
- my ($self, $class, $description) = @_;
+ my ($self, $description) = @_;
+
+ my $class = ref($self);
$class->register_method({
name => 'start',
my $reload_daemon = sub {
my ($self, $use_hup) = @_;
- if (my $restart = $ENV{RESTART_PVE_DAEMON}) {
+ if ($self->{env_restart_pve_daemon}) {
$self->start();
} else {
my ($running, $pid) = $self->running();
};
sub register_restart_command {
- my ($self, $class, $use_hup, $description) = @_;
+ my ($self, $use_hup, $description) = @_;
+
+ my $class = ref($self);
$class->register_method({
name => 'restart',
}
sub register_reload_command {
- my ($self, $class, $description) = @_;
+ my ($self, $description) = @_;
+
+ my $class = ref($self);
$class->register_method({
name => 'reload',
}
sub register_stop_command {
- my ($self, $class, $description) = @_;
+ my ($self, $description) = @_;
+
+ my $class = ref($self);
$class->register_method({
name => 'stop',
}
sub register_status_command {
- my ($self, $class, $description) = @_;
+ my ($self, $description) = @_;
+
+ my $class = ref($self);
$class->register_method({
name => 'status',
}});
}
+# some useful helper
+
+sub create_reusable_socket {
+ my ($self, $port, $host) = @_;
+
+ die "no port specifed" if !$port;
+
+ my ($socket, $sockfd);
+
+ if (defined($sockfd = $ENV{"PVE_DAEMON_SOCKET_$port"}) &&
+ $self->{env_restart_pve_daemon}) {
+
+ die "unable to parse socket fd '$sockfd'\n"
+ if $sockfd !~ m/^(\d+)$/;
+ $sockfd = $1; # untaint
+
+ $socket = IO::Socket::INET->new;
+ $socket->fdopen($sockfd, 'w') ||
+ die "cannot fdopen file descriptor '$sockfd' - $!\n";
+
+ } else {
+
+ $socket = IO::Socket::INET->new(
+ LocalAddr => $host,
+ LocalPort => $port,
+ Listen => SOMAXCONN,
+ Proto => 'tcp',
+ ReuseAddr => 1) ||
+ die "unable to create socket - $@\n";
+
+ # we often observe delays when using Nagle algorithm,
+ # so we disable that to maximize performance
+ setsockopt($socket, IPPROTO_TCP, TCP_NODELAY, 1);
+
+ $ENV{"PVE_DAEMON_SOCKET_$port"} = $socket->fileno;
+ }
+
+ # remove FD_CLOEXEC bit to reuse on exec
+ $socket->fcntl(Fcntl::F_SETFD(), 0);
+
+ return $socket;
+}
+
+
1;