X-Git-Url: https://git.proxmox.com/?p=pve-common.git;a=blobdiff_plain;f=data%2FPVE%2FDaemon.pm;h=264f8be5e2a840aa7969a47ef95fbda017d7cbc8;hp=279a78c1f36fce16394c9b06b3b9bba6963061c2;hb=309d99e71818b7af5092152d02dff39894ecf432;hpb=4fc691765746d74422f840ebfa4e3391de615244

diff --git a/data/PVE/Daemon.pm b/data/PVE/Daemon.pm
index 279a78c..264f8be 100644
--- a/data/PVE/Daemon.pm
+++ b/data/PVE/Daemon.pm
@@ -5,52 +5,106 @@ package PVE::Daemon;
 # Features:
 # * lock and write PID file /var/run/$name.pid to make sure onyl
 #   one instance is running.
+# * keep lock open during restart
 # * correctly daemonize (redirect STDIN/STDOUT)
 # * restart by stop/start, exec, or signal HUP
 # * daemon restart on error (option 'restart_on_error')
-#
-
+# * handle worker processes (option 'max_workers')
+# * allow to restart while workers are still runningl
+#   (option 'leave_children_open_on_reload')
+# * run as different user using setuid/setgid
+ 
 use strict;
 use warnings;
+use English;
+
 use PVE::SafeSyslog;
 use PVE::INotify;
 
 use POSIX ":sys_wait_h";
 use Fcntl ':flock';
+use Socket qw(IPPROTO_TCP TCP_NODELAY SOMAXCONN);
+use IO::Socket::INET;
+
 use Getopt::Long;
 use Time::HiRes qw (gettimeofday);
 
 use base qw(PVE::CLIHandler);
 
-$SIG{'__WARN__'} = sub {
-    my $err = $@;
-    my $t = $_[0];
-    chomp $t;
-    print "$t\n";
-    syslog('warning', "WARNING: %s", $t);
-    $@ = $err;
-};
-
 $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
 
 my $daemon_initialized = 0; # we only allow one instance
 
+my $close_daemon_lock = sub {
+    my ($self) = @_;
+
+    return if !$self->{daemon_lock_fh};
+
+    close $self->{daemon_lock_fh};
+    delete $self->{daemon_lock_fh};
+};
+
+my $log_err = sub {
+    my ($msg) = @_;
+    chomp $msg;
+    print STDERR "$msg\n";
+    syslog('err', "%s", $msg);
+};
+
+# call this if you fork() from child
+# Note: we already call this for workers, so it is only required
+# if you fork inside a simple daemon (max_workers == 0).
+sub after_fork_cleanup {
+    my ($self) = @_;
+
+    &$close_daemon_lock($self);
+
+    PVE::INotify::inotify_close();
+
+    for my $sig (qw(CHLD HUP INT TERM QUIT)) {
+	$SIG{$sig} = 'DEFAULT'; # restore default handler
+	# AnyEvent signals only works if $SIG{XX} is 
+	# undefined (perl event loop)
+	delete $SIG{$sig}; # so that we can handle events with AnyEvent
+    }
+}
+
 my $lockpidfile = sub {
     my ($self) = @_;
 
     my $lkfn = $self->{pidfile} . ".lock";
 
-    if (!open (FLCK, ">>$lkfn")) {
-	my $msg = "can't aquire lock on file '$lkfn' - $!";
-	syslog ('err', $msg);
-	die "ERROR: $msg\n";
+    my $waittime = 0;
+
+    if (my $fd = $self->{env_pve_lock_fd}) {
+
+	$self->{daemon_lock_fh} = IO::Handle->new_from_fd($fd, "a");
+	
+    } else {
+
+	$waittime = 5;
+	$self->{daemon_lock_fh} = IO::File->new(">>$lkfn");
+    }
+
+    if (!$self->{daemon_lock_fh}) {
+	die "can't open lock '$lkfn' - $!\n";
+    }
+
+    for (my $i = 0; $i < $waittime; $i ++) {
+	return if flock ($self->{daemon_lock_fh}, LOCK_EX|LOCK_NB);
+	sleep(1);
     }
 
-    if (!flock (FLCK, LOCK_EX|LOCK_NB)) {
-	close (FLCK);
-        my $msg = "can't aquire lock '$lkfn' - $!";
-	syslog ('err', $msg);
-	die "ERROR: $msg\n";
+    if (!flock ($self->{daemon_lock_fh}, LOCK_EX|LOCK_NB)) {
+	&$close_daemon_lock($self);
+	my $err = $!;
+
+	my ($running, $pid) = $self->running();
+	if ($running) {
+	    die "can't aquire lock '$lkfn' - daemon already started (pid = $pid)\n";
+	} else {
+	    die "can't aquire lock '$lkfn' - $err\n";
+	}
     }
 };
 
@@ -59,11 +113,8 @@ my $writepidfile = sub {
 
     my $pidfile = $self->{pidfile};
 
-    if (!open (PIDFH, ">$pidfile")) {
-	my $msg = "can't open pid file '$pidfile' - $!";
-	syslog ('err', $msg);
-	die "ERROR: $msg\n";
-    }
+    die "can't open pid file '$pidfile' - $!\n" if !open (PIDFH, ">$pidfile");
+
     print PIDFH "$$\n";
     close (PIDFH);
 };
@@ -75,17 +126,140 @@ my $server_cleanup = sub {
     unlink $self->{pidfile};
 };
 
+my $finish_workers = sub {
+    my ($self) = @_;
+
+    foreach my $id (qw(workers old_workers)) {
+	foreach my $cpid (keys %{$self->{$id}}) {
+	    my $waitpid = waitpid($cpid, WNOHANG);
+	    if (defined($waitpid) && ($waitpid == $cpid)) {
+		delete ($self->{$id}->{$cpid});
+		syslog('info', "worker $cpid finished");
+	    }
+	}
+    }
+};
+
+my $start_workers = sub {
+    my ($self) = @_;
+
+    return if $self->{terminate};
+
+    my $count = 0;
+    foreach my $cpid (keys %{$self->{workers}}) {
+	$count++;
+    }
+
+    my $need = $self->{max_workers} - $count;
+
+    return if $need <= 0;
+
+    syslog('info', "starting $need worker(s)");
+
+    while ($need > 0) {
+	my $pid = fork;
+
+	if (!defined ($pid)) {
+	    syslog('err', "can't fork worker");
+	    sleep (1);
+	} elsif ($pid) { # parent
+	    $self->{workers}->{$pid} = 1;
+	    syslog('info', "worker $pid started");
+	    $need--;
+	} else {
+	    $0 = "$self->{name} worker";
+
+	    $self->after_fork_cleanup();
+
+	    eval { $self->run(); };
+	    if (my $err = $@) {
+		syslog('err', $err);
+		sleep(5); # avoid fast restarts
+	    }
+
+	    syslog('info', "worker exit");
+	    exit (0);
+	}
+    }
+};
+
+my $terminate_server = sub {
+    my ($self, $allow_open_children) = @_;
+
+    $self->{terminate} = 1; # set flag to avoid worker restart
+
+    if (!$self->{max_workers}) {
+	eval { $self->shutdown(); };
+	warn $@ if $@;
+	return;
+    }
+
+    eval { $self->shutdown(); };
+    warn $@ if $@;
+
+    # we have workers - send TERM signal
+
+    foreach my $cpid (keys %{$self->{workers}}) {
+	kill(15, $cpid); # TERM childs
+    }
+
+    # if configured, leave children running on HUP
+    return if $allow_open_children &&
+	$self->{leave_children_open_on_reload};
+
+    # else, send TERM to old workers
+    foreach my $cpid (keys %{$self->{old_workers}}) {
+	kill(15, $cpid); # TERM childs
+    }
+
+    # nicely shutdown childs (give them max 10 seconds to shut down)
+    my $previous_alarm = alarm(10);
+    eval {
+	local $SIG{ALRM} = sub { die "timeout\n" };
+
+	while ((my $pid = waitpid (-1, 0)) > 0) {
+	    foreach my $id (qw(workers old_workers)) {
+		if (defined($self->{$id}->{$pid})) {
+		    delete($self->{$id}->{$pid});
+		    syslog('info', "worker $pid finished");
+		}
+	    }
+	}
+	alarm(0); # avoid race condition
+    };
+    my $err = $@;
+
+    alarm ($previous_alarm);
+
+    if ($err) {
+	syslog('err', "error stopping workers (will kill them now) - $err");
+	foreach my $id (qw(workers old_workers)) {
+	    foreach my $cpid (keys %{$self->{$id}}) {
+		# KILL childs still alive!
+		if (kill (0, $cpid)) {
+		    delete($self->{$id}->{$cpid});
+		    syslog("err", "kill worker $cpid");
+		    kill(9, $cpid);
+		    # fixme: waitpid?
+		}
+	    }
+	}
+    }
+};
+
 my $server_run = sub {
     my ($self, $debug) = @_;
 
+    # fixme: handle restart lockfd
     &$lockpidfile($self);
 
-    # run in background
-    my $spid;
+    # remove FD_CLOEXEC bit to reuse on exec
+    $self->{daemon_lock_fh}->fcntl(Fcntl::F_SETFD(), 0);
 
-    my $restart = $ENV{RESTART_PVE_DAEMON};
+    $ENV{PVE_DAEMON_LOCK_FD} = $self->{daemon_lock_fh}->fileno;
 
-    delete $ENV{RESTART_PVE_DAEMON};
+    # run in background
+    my $spid;
 
     $self->{debug} = 1 if $debug;
 
@@ -96,52 +270,99 @@ my $server_run = sub {
 	open STDOUT, '>/dev/null' || die "can't write /dev/null";
     }
 
-    if (!$restart && !$debug) {
+    if (!$self->{env_restart_pve_daemon} && !$debug) {
 	PVE::INotify::inotify_close();
 	$spid = fork();
 	if (!defined ($spid)) {
-	    my $msg =  "can't put server into background - fork failed";
-	    syslog('err', $msg);
-	    die "ERROR: $msg\n";
+	    die "can't put server into background - fork failed";
 	} elsif ($spid) { # parent
 	    exit (0);
 	}
 	PVE::INotify::inotify_init();
     }
 
-    &$writepidfile($self);
-
-    POSIX::setsid(); 
-
-    if ($restart) {
+    if ($self->{env_restart_pve_daemon}) {
 	syslog('info' , "restarting server");
     } else {
+	&$writepidfile($self);
 	syslog('info' , "starting server");
     }
 
+    POSIX::setsid(); 
+
     open STDERR, '>&STDOUT' || die "can't close STDERR\n";
 
-    $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = sub {
-	$SIG{INT} = 'DEFAULT';
+    my $old_sig_term = $SIG{TERM};
+    local $SIG{TERM} = sub {
+	local ($@, $!, $?); # do not overwrite error vars
+	syslog('info', "received signal TERM");
+	&$terminate_server($self, 0);
+	&$server_cleanup($self);
+	&$old_sig_term(@_) if $old_sig_term;
+    };
 
-	eval { $self->shutdown(); };
-	warn $@ if $@;
+    my $old_sig_quit = $SIG{QUIT};
+    local $SIG{QUIT} = sub {
+	local ($@, $!, $?); # do not overwrite error vars
+	syslog('info', "received signal QUIT");
+	&$terminate_server($self, 0);
+	&$server_cleanup($self);
+	&$old_sig_quit(@_) if $old_sig_quit;
+    };
 
+    my $old_sig_int = $SIG{INT};
+    local $SIG{INT} = sub {
+	local ($@, $!, $?); # do not overwrite error vars
+	syslog('info', "received signal INT");
+	$SIG{INT} = 'DEFAULT'; # allow to terminate now
+	&$terminate_server($self, 0);
 	&$server_cleanup($self);
+	&$old_sig_int(@_) if $old_sig_int;
     };
 
-    if ($self->can('hup')) {
-	$SIG{HUP} = sub {
+    $SIG{HUP} = sub {
+	local ($@, $!, $?); # do not overwrite error vars
+	syslog('info', "received signal HUP");
+	$self->{got_hup_signal} = 1;
+	if ($self->{max_workers}) {
+	    &$terminate_server($self, 1);
+	} elsif ($self->can('hup')) {
 	    eval { $self->hup() };
 	    warn $@ if $@;
-	};
-    }
+	}
+    };
+
+    eval { 
+	if ($self->{max_workers}) {
+	    my $old_sig_chld = $SIG{CHLD};
+	    local $SIG{CHLD} = sub {
+		local ($@, $!, $?); # do not overwrite error vars
+		&$finish_workers($self);
+		&$old_sig_chld(@_) if $old_sig_chld;
+	    };
+
+	    # catch worker finished during restart phase 
+	    &$finish_workers($self);
+
+	    # now loop forever (until we receive terminate signal)
+	    for (;;) { 
+		&$start_workers($self);
+		sleep(5);
+		&$finish_workers($self);
+		last if $self->{terminate};
+	    }
 
-    eval { $self->run() };
+	} else {
+	    $self->run();
+	} 
+    };
     my $err = $@;
 
     if ($err) {
 	syslog ('err', "ERROR: $err");
+
+	&$terminate_server($self, 1);
+
 	if (my $wait_time = $self->{restart_on_error}) {
 	    $self->restart_daemon($wait_time);
 	} else {
@@ -149,51 +370,116 @@ my $server_run = sub {
 	}
     }
 
-    $self->exit_daemon(0);
+    if ($self->{got_hup_signal}) {
+	$self->restart_daemon();
+    } else {
+	$self->exit_daemon(0);
+    }
 };
 
 sub new {
     my ($this, $name, $cmdline, %params) = @_;
 
-    die "please run as root\n" if $> != 0;
+    $name = 'daemon' if !$name; # should not happen
 
-    die "missing name" if !$name;
+    initlog($name);
 
-    die "can't create more that one PVE::Daemon" if $daemon_initialized;
-    $daemon_initialized = 1;
+    my $self;
 
-    PVE::INotify::inotify_init();
+    eval {
 
-    initlog($name);
+	my $restart = $ENV{RESTART_PVE_DAEMON};
+	delete $ENV{RESTART_PVE_DAEMON};
 
-    my $class = ref($this) || $this;
-
-    my $self = bless { 
-	name => $name,
-	run_dir => '/var/run',
-    }, $class;
-
-    foreach my $opt (keys %params) {
-	my $value = $params{$opt};
-	if ($opt eq 'restart_on_error') {
-	    $self->{$opt} = $value;
-	} elsif ($opt eq 'stop_wait_time') {
-	    $self->{$opt} = $value;
-	} elsif ($opt eq 'run_dir') {
-	    $self->{$opt} = $value;
-	} else {
-	    die "unknown option '$opt'";
+	my $lockfd = $ENV{PVE_DAEMON_LOCK_FD};
+	delete $ENV{PVE_DAEMON_LOCK_FD};
+
+	if (defined($lockfd)) {
+	    die "unable to parse lock fd '$lockfd'\n"
+		if $lockfd !~ m/^(\d+)$/;
+	    $lockfd = $1; # untaint
 	}
-    }
 
-    $self->{pidfile} = "$self->{run_dir}/${name}.pid";
+	die "please run as root\n" if !$restart && ($> != 0);
+
+	die "can't create more that one PVE::Daemon" if $daemon_initialized;
+	$daemon_initialized = 1;
+
+	PVE::INotify::inotify_init();
+
+	my $class = ref($this) || $this;
+
+	$self = bless { 
+	    name => $name,
+	    pidfile => "/var/run/${name}.pid",
+	    env_restart_pve_daemon => $restart,
+	    env_pve_lock_fd => $lockfd,
+	    workers => {},
+	    old_workers => {},
+	}, $class;
+
+
+	foreach my $opt (keys %params) {
+	    my $value = $params{$opt};
+	    if ($opt eq 'restart_on_error') {
+		$self->{$opt} = $value;
+	    } elsif ($opt eq 'stop_wait_time') {
+		$self->{$opt} = $value;
+	    } elsif ($opt eq 'pidfile') {
+		$self->{$opt} = $value;
+	    } elsif ($opt eq 'max_workers') {
+		$self->{$opt} = $value;
+	    } elsif ($opt eq 'leave_children_open_on_reload') {
+		$self->{$opt} = $value;
+	    } elsif ($opt eq 'setgid') {
+		$self->{$opt} = $value;
+	    } elsif ($opt eq 'setuid') {
+		$self->{$opt} = $value;
+	    } else {
+		die "unknown daemon option '$opt'\n";
+	    }
+	}
+	
+	if (my $gidstr = $self->{setgid}) {
+	    my $gid = getgrnam($gidstr) || die "getgrnam failed - $!\n";
+	    POSIX::setgid($gid) || die "setgid $gid failed - $!\n";
+	    $EGID = "$gid $gid"; # this calls setgroups
+	    # just to be sure
+	    die "detected strange gid\n" if !($GID eq "$gid $gid" && $EGID eq "$gid $gid");
+	}
 
-    $self->{nodename} = PVE::INotify::nodename();
+	if (my $uidstr = $self->{setuid}) {
+	    my $uid = getpwnam($uidstr) || die "getpwnam failed - $!\n";
+	    POSIX::setuid($uid) || die "setuid $uid failed - $!\n";
+	    # just to be sure
+	    die "detected strange uid\n" if !($UID == $uid && $EUID == $uid);
+	}
 
-    $self->{cmdline} = $cmdline;
+	if ($restart && $self->{max_workers}) {
+	    if (my $wpids = $ENV{PVE_DAEMON_WORKER_PIDS}) {
+		foreach my $pid (split(':', $wpids)) {
+		    if ($pid =~ m/^(\d+)$/) {
+			$self->{old_workers}->{$1} = 1;
+		    }
+		}
+	    }
+	}
 
-    $0 = $name;
+	$self->{nodename} = PVE::INotify::nodename();
 
+	$self->{cmdline} = [];
+
+	foreach my $el (@$cmdline) {
+	    $el =~ m/^(.*)$/; # untaint
+	    push @{$self->{cmdline}}, $1;
+	}
+
+	$0 = $name;
+    };
+    if (my $err = $@) {
+	&$log_err($err);
+	exit(-1);
+    }
 
     return $self;
 }
@@ -215,6 +501,12 @@ sub restart_daemon {
 
     $ENV{RESTART_PVE_DAEMON} = 1;
 
+    if ($self->{max_workers}) {
+	my @workers = keys %{$self->{workers}};
+	push @workers, keys %{$self->{old_workers}};
+	$ENV{PVE_DAEMON_WORKER_PIDS} = join(':', @workers);
+    }
+
     sleep($waittime) if $waittime; # avoid high server load due to restarts
 
     PVE::INotify::inotify_close();
@@ -237,8 +529,10 @@ sub shutdown {
 
     syslog('info' , "server closing");
 
-    # wait for children
-    1 while (waitpid(-1, POSIX::WNOHANG()) > 0);
+    if (!$self->{max_workers}) {
+	# wait for children
+	1 while (waitpid(-1, POSIX::WNOHANG()) > 0);
+    }
 }
 
 # please define in subclass
@@ -261,7 +555,11 @@ sub run {
 sub start {
     my ($self, $debug) = @_;
 
-    &$server_run($self, $debug);
+    eval  { &$server_run($self, $debug); };
+    if (my $err = $@) {
+	&$log_err("start failed - $err");
+	exit(-1);
+    }
 }
 
 my $read_pid = sub {
@@ -317,14 +615,21 @@ sub stop {
     }
 
     if (-f $self->{pidfile}) {
-	# try to get the lock
-	&$lockpidfile($self);
-	&$server_cleanup($self);
+	eval {
+	    # try to get the lock
+	    &$lockpidfile($self);
+	    &$server_cleanup($self);
+	};
+	if (my $err = $@) {
+	    &$log_err("cleanup failed - $err");
+	}
     }
 }
 
 sub register_start_command {
-    my ($self, $class, $description) = @_;
+    my ($self, $description) = @_;
+
+    my $class = ref($self);
 
     $class->register_method({
 	name => 'start',
@@ -356,7 +661,7 @@ sub register_start_command {
 my $reload_daemon = sub {
     my ($self, $use_hup) = @_;
 
-    if (my $restart = $ENV{RESTART_PVE_DAEMON}) {
+    if ($self->{env_restart_pve_daemon}) {
 	$self->start();
     } else {
 	my ($running, $pid) = $self->running(); 
@@ -364,7 +669,8 @@ my $reload_daemon = sub {
 	    $self->start();
 	} else {
 	    if ($use_hup) {
-		kill(1, $pid);
+		syslog('info', "send HUP to $pid");
+		kill 1, $pid;
 	    } else {
 		$self->stop();
 		$self->start();
@@ -374,7 +680,9 @@ my $reload_daemon = sub {
 };
 
 sub register_restart_command {
-    my ($self, $class, $use_hup, $description) = @_;
+    my ($self, $use_hup, $description) = @_;
+
+    my $class = ref($self);
 
     $class->register_method({
 	name => 'restart',
@@ -397,7 +705,9 @@ sub register_restart_command {
 }
 
 sub register_reload_command {
-    my ($self, $class, $description) = @_;
+    my ($self, $description) = @_;
+
+    my $class = ref($self);
 
     $class->register_method({
 	name => 'reload',
@@ -420,7 +730,9 @@ sub register_reload_command {
 }
 
 sub register_stop_command {
-    my ($self, $class, $description) = @_;
+    my ($self, $description) = @_;
+
+    my $class = ref($self);
 
     $class->register_method({
 	name => 'stop',
@@ -443,7 +755,9 @@ sub register_stop_command {
 }
 
 sub register_status_command {
-    my ($self, $class, $description) = @_;
+    my ($self, $description) = @_;
+
+    my $class = ref($self);
 
     $class->register_method({
 	name => 'status',
@@ -465,5 +779,49 @@ sub register_status_command {
 	}});
 }
 
+# some useful helper
+
+sub create_reusable_socket {
+    my ($self, $port, $host) = @_;
+
+    die "no port specifed" if !$port;
+
+    my ($socket, $sockfd);
+
+    if (defined($sockfd = $ENV{"PVE_DAEMON_SOCKET_$port"}) &&
+	$self->{env_restart_pve_daemon}) {
+
+	die "unable to parse socket fd '$sockfd'\n" 
+	    if $sockfd !~ m/^(\d+)$/;
+	$sockfd = $1; # untaint
+
+	$socket = IO::Socket::INET->new;
+	$socket->fdopen($sockfd, 'w') || 
+	    die "cannot fdopen file descriptor '$sockfd' - $!\n";
+
+    } else {
+
+	$socket = IO::Socket::INET->new(
+	    LocalAddr => $host,
+	    LocalPort => $port,
+	    Listen => SOMAXCONN,
+	    Proto  => 'tcp',
+	    ReuseAddr => 1) ||
+	    die "unable to create socket - $@\n";
+
+	# we often observe delays when using Nagle algorithm,
+	# so we disable that to maximize performance
+	setsockopt($socket, IPPROTO_TCP, TCP_NODELAY, 1);
+
+	$ENV{"PVE_DAEMON_SOCKET_$port"} = $socket->fileno;
+    }
+
+    # remove FD_CLOEXEC bit to reuse on exec
+    $socket->fcntl(Fcntl::F_SETFD(), 0);
+    
+    return $socket;
+}
+
+
 1;