sen TERM to all pgrp members

[pve-access-control.git] / PVE / RPCEnvironment.pm

diff --git a/PVE/RPCEnvironment.pm b/PVE/RPCEnvironment.pm
index a18ceb1c8fbb568c976449dab23777f6fc1fefe6..5806107d9e7eb4a0546b503809e371dd170db854 100644 (file)
--- a/PVE/RPCEnvironment.pm
+++ b/PVE/RPCEnvironment.pm
@@ -2,7 +2,7 @@ package PVE::RPCEnvironment;
 
 use strict;
 use warnings;
-use POSIX ":sys_wait_h";
+use POSIX qw(:sys_wait_h EINTR);
 use IO::File;
 use Fcntl qw(:flock);
 use PVE::SafeSyslog;
@@ -316,8 +316,10 @@ sub get_user {
 # read/update list of active workers 
 # we move all finished tasks to the archive index,
 # but keep aktive and most recent task in the active file.
+# $nocheck ... consider $new_upid still running (avoid that
+# we try to read the reult to early.
 sub active_workers  {
-    my ($new_upid) = @_;
+    my ($new_upid, $nocheck) = @_;
 
     my $lkfn = "/var/log/pve/tasks/.active.lock";
 
@@ -332,10 +334,9 @@ sub active_workers  {
        my $thash = {}; # only list task once
 
        my $check_task = sub {
-           my ($task) = @_;
+           my ($task, $running) = @_;
 
-           my $pstart = PVE::ProcFSTools::read_proc_starttime($task->{pid});
-           if ($pstart && ($pstart == $task->{pstart})) {
+           if ($running || PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart})) {
                push @$tlist, $task;
            } else {
                delete $task->{pid};
@@ -355,7 +356,7 @@ sub active_workers  {
            $task = PVE::Tools::upid_decode($new_upid);
            $task->{upid} = $new_upid;
            $thash->{$new_upid} = $task;
-           &$check_task($task);
+           &$check_task($task, $nocheck);
        }
 
 
@@ -427,6 +428,42 @@ sub active_workers  {
     return $res;
 }
 
+my $kill_process_group = sub {
+    my ($pid, $pstart) = @_;
+
+    # send kill to process group (negative pid)
+    my $kpid = -$pid;
+
+    # always send signal to all pgrp members
+    kill(15, $kpid); # send TERM signal
+
+    # give max 5 seconds to shut down
+    for (my $i = 0; $i < 5; $i++) {
+       return if !PVE::ProcFSTools::check_process_running($pid, $pstart);
+       sleep (1);
+    }
+       
+    # to be sure
+    kill(9, $kpid); 
+};
+
+sub check_worker {
+    my ($upid, $killit) = @_;
+
+    my $task = PVE::Tools::upid_decode($upid);
+
+    my $running = PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart});
+
+    return 0 if !$running;
+
+    if ($killit) {
+       &$kill_process_group($task->{pid});
+       return 0;
+    }
+
+    return 1;
+}
+
 # start long running workers
 # STDIN is redirected to /dev/null
 # STDOUT,STDERR are redirected to the filename returned by upid_decode
@@ -527,10 +564,10 @@ sub fork_worker {
            POSIX::write($psync[1], $msg, length ($msg));
            POSIX::close($psync[1]);
            POSIX::_exit(1); 
-           kill('KILL', $$); 
+           kill(-9, $$); 
        }
 
-       # sync with parent (signal that we are read)
+       # sync with parent (signal that we are ready)
        if ($sync) {
            print "$upid\n";
        } else {
@@ -553,9 +590,9 @@ sub fork_worker {
            POSIX::_exit(-1); 
        } else {
            print STDERR "TASK OK\n";
-           POSIX::_exit (0);
+           POSIX::_exit(0);
        } 
-       kill('KILL', $$); 
+       kill(-9, $$); 
     }
 
     # parent
@@ -598,13 +635,13 @@ sub fork_worker {
        
     } else {
        POSIX::close($csync[1]);
-       kill (9, $cpid); # make sure it gets killed
+       kill(-9, $cpid); # make sure it gets killed
        die $err;
     }
 
     PVE::Cluster::log_msg('info', $user, "starting task $upid");
 
-    my $tlist = active_workers($upid);
+    my $tlist = active_workers($upid, $sync);
     PVE::Cluster::broadcast_tasklist($tlist);
    
     my $res = 0;
@@ -612,13 +649,27 @@ sub fork_worker {
     if ($sync) {
        my $count;
        my $outbuf = '';
+       my $int_count = 0;
        eval {
-           local $SIG{INT} = 
-               local $SIG{QUIT} = 
-               local $SIG{TERM} = sub { die "got interrupt\n"; };
+           local $SIG{INT} = local $SIG{QUIT} = local $SIG{TERM} = sub { 
+               # always send signal to all pgrp members
+               my $kpid = -$cpid;
+               if ($int_count < 3) {
+                   kill(15, $kpid); # send TERM signal
+               } else {
+                   kill(9, $kpid); # send KILL signal
+               }
+               $int_count++;
+           };
            local $SIG{PIPE} = sub { die "broken pipe\n"; };
        
-           while (($count = POSIX::read($psync[0], $readbuf, 4096)) && ($count > 0)) {
+           while (1) {
+               if (!defined($count = POSIX::read($psync[0], $readbuf, 4096))) {
+                   next if $! == EINTR;
+                   last;
+               }
+               last if $count == 0; # eof
+
                $outbuf .= $readbuf;
                while ($outbuf =~ s/^(([^\010\r\n]*)(\r|\n|(\010)+|\r\n))//s) {
                    my $line = $1;
@@ -632,6 +683,7 @@ sub fork_worker {
                    }
                    if ($outfh) {
                        print $outfh $line;
+                       $outfh->flush();
                    }
                }
            }
@@ -653,15 +705,13 @@ sub fork_worker {
            if ($outfh) {
                print $outfh "TASK ERROR: $err\n";
            }
-           kill (15, $cpid);
-
-       } else {
-           kill (9, $cpid); # make sure it gets killed
        }
 
+       &$kill_process_group($cpid, $pstart); # make sure it gets killed
+
        close($outfh);
 
-       waitpid ($cpid, 0);
+       waitpid($cpid, 0);
        $res = $?;
        &$log_task_result($upid, $user, $res);
     }