]> git.proxmox.com Git - pve-common.git/blob - src/PVE/RESTEnvironment.pm
RESTenv: fork worker: fallback to 'root@pam' for task log user-id
[pve-common.git] / src / PVE / RESTEnvironment.pm
1 package PVE::RESTEnvironment;
2
3 # NOTE: you can/should provide your own specialice class, and
4 # use this a bas class (as example see PVE::RPCEnvironment).
5
6 # we use this singleton class to pass RPC related environment values
7
8 use strict;
9 use warnings;
10
11 use Exporter qw(import);
12 use Fcntl qw(:flock);
13 use IO::File;
14 use IO::Handle;
15 use IO::Select;
16 use POSIX qw(:sys_wait_h EINTR);
17
18 use PVE::Exception qw(raise raise_perm_exc);
19 use PVE::INotify;
20 use PVE::ProcFSTools;
21 use PVE::SafeSyslog;
22 use PVE::Tools;
23
24 our @EXPORT_OK = qw(log_warn);
25
26 my $rest_env;
27
28 # save $SIG{CHLD} handler implementation.
29 # simply set $SIG{CHLD} = $worker_reaper;
30 # and register forked processes with &$register_worker(pid)
31 # Note: using $SIG{CHLD} = 'IGNORE' or $SIG{CHLD} = sub { wait (); } or ...
32 # has serious side effects, because perls built in system() and open()
33 # functions can't get the correct exit status of a child. So we can't use
34 # that (also see perlipc)
35
36 my $WORKER_PIDS;
37 my $WORKER_FLAG = 0;
38
39 my $log_task_result = sub {
40 my ($upid, $user, $status) = @_;
41
42 return if !$rest_env;
43
44 my $msg = 'successful';
45 my $pri = 'info';
46 if ($status != 0) {
47 my $ec = $status >> 8;
48 my $ic = $status & 255;
49 $msg = $ec ? "failed ($ec)" : "interrupted ($ic)";
50 $pri = 'err';
51 }
52
53 my $tlist = $rest_env->active_workers($upid);
54 eval { $rest_env->broadcast_tasklist($tlist); };
55 syslog('err', $@) if $@;
56
57 my $task;
58 foreach my $t (@$tlist) {
59 if ($t->{upid} eq $upid) {
60 $task = $t;
61 last;
62 }
63 }
64 if ($task && $task->{status}) {
65 $msg = $task->{status};
66 }
67
68 $rest_env->log_cluster_msg($pri, $user, "end task $upid $msg");
69 };
70
71 my $worker_reaper = sub {
72 local $!; local $?;
73 foreach my $pid (keys %$WORKER_PIDS) {
74 my $waitpid = waitpid ($pid, WNOHANG);
75 if (defined($waitpid) && ($waitpid == $pid)) {
76 my $info = $WORKER_PIDS->{$pid};
77 if ($info && $info->{upid} && $info->{user}) {
78 &$log_task_result($info->{upid}, $info->{user}, $?);
79 }
80 delete ($WORKER_PIDS->{$pid});
81 }
82 }
83 };
84
85 my $register_worker = sub {
86 my ($pid, $user, $upid) = @_;
87
88 return if !$pid;
89
90 # do not register if already finished
91 my $waitpid = waitpid ($pid, WNOHANG);
92 if (defined($waitpid) && ($waitpid == $pid)) {
93 delete ($WORKER_PIDS->{$pid});
94 return;
95 }
96
97 $WORKER_PIDS->{$pid} = {
98 user => $user,
99 upid => $upid,
100 };
101 };
102
103 # initialize environment - must be called once at program startup
104 sub init {
105 my ($class, $type, %params) = @_;
106
107 $class = ref($class) || $class;
108
109 die "already initialized" if $rest_env;
110
111 die "unknown environment type"
112 if !$type || $type !~ m/^(cli|pub|priv|ha)$/;
113
114 $SIG{CHLD} = $worker_reaper;
115
116 # environment types
117 # cli ... command started fron command line
118 # pub ... access from public server (pveproxy)
119 # priv ... access from private server (pvedaemon)
120 # ha ... access from HA resource manager agent (pve-ha-manager)
121
122 my $self = {
123 type => $type,
124 warning_count => 0,
125 };
126
127 bless $self, $class;
128
129 foreach my $p (keys %params) {
130 if ($p eq 'atfork') {
131 $self->{$p} = $params{$p};
132 } else {
133 die "unknown option '$p'";
134 }
135 }
136
137 $rest_env = $self;
138
139 my ($sysname, $nodename) = POSIX::uname();
140
141 $nodename =~ s/\..*$//; # strip domain part, if any
142
143 $self->{nodename} = $nodename;
144
145 return $self;
146 };
147
148 # convenience function for command line tools
149 sub setup_default_cli_env {
150 my ($class, $username) = @_;
151
152 $class = ref($class) || $class;
153
154 $username //= 'root@pam';
155
156 PVE::INotify::inotify_init();
157
158 my $rpcenv = $class->init('cli');
159 $rpcenv->init_request();
160 $rpcenv->set_language($ENV{LANG});
161 $rpcenv->set_user($username);
162
163 die "please run as root\n"
164 if ($username eq 'root@pam') && ($> != 0);
165 }
166
167 # get the singleton
168 sub get {
169
170 die "REST environment not initialized" if !$rest_env;
171
172 return $rest_env;
173 }
174
175 sub set_client_ip {
176 my ($self, $ip) = @_;
177
178 $self->{client_ip} = $ip;
179 }
180
181 sub get_client_ip {
182 my ($self) = @_;
183
184 return $self->{client_ip};
185 }
186
187 sub set_result_attrib {
188 my ($self, $key, $value) = @_;
189
190 $self->{result_attributes}->{$key} = $value;
191 }
192
193 sub get_result_attrib {
194 my ($self, $key) = @_;
195
196 return $self->{result_attributes}->{$key};
197 }
198
199 sub set_language {
200 my ($self, $lang) = @_;
201
202 # fixme: initialize I18N
203
204 $self->{language} = $lang;
205 }
206
207 sub get_language {
208 my ($self) = @_;
209
210 return $self->{language};
211 }
212
213 sub set_user {
214 my ($self, $user) = @_;
215
216 $self->{user} = $user;
217 }
218
219 sub get_user {
220 my ($self, $noerr) = @_;
221
222 return $self->{user} if defined($self->{user}) || $noerr;
223
224 die "user name not set\n";
225 }
226
227 sub set_u2f_challenge {
228 my ($self, $challenge) = @_;
229
230 $self->{u2f_challenge} = $challenge;
231 }
232
233 sub get_u2f_challenge {
234 my ($self, $noerr) = @_;
235
236 return $self->{u2f_challenge} if defined($self->{u2f_challenge}) || $noerr;
237
238 die "no active u2f challenge\n";
239 }
240
241 sub set_request_host {
242 my ($self, $host) = @_;
243
244 $self->{request_host} = $host;
245 }
246
247 sub get_request_host {
248 my ($self, $noerr) = @_;
249
250 return $self->{request_host} if defined($self->{request_host}) || $noerr;
251
252 die "no hostname available in current environment\n";
253 }
254
255 sub is_worker {
256 my ($class) = @_;
257
258 return $WORKER_FLAG;
259 }
260
261 # read/update list of active workers
262 # we move all finished tasks to the archive index,
263 # but keep aktive and most recent task in the active file.
264 # $nocheck ... consider $new_upid still running (avoid that
265 # we try to read the reult to early.
266 sub active_workers {
267 my ($self, $new_upid, $nocheck) = @_;
268
269 my $lkfn = "/var/log/pve/tasks/.active.lock";
270
271 my $timeout = 10;
272
273 my $code = sub {
274
275 my $tasklist = PVE::INotify::read_file('active');
276
277 my @ta;
278 my $tlist = [];
279 my $thash = {}; # only list task once
280
281 my $check_task = sub {
282 my ($task, $running) = @_;
283
284 if ($running || PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart})) {
285 push @$tlist, $task;
286 } else {
287 delete $task->{pid};
288 push @ta, $task;
289 }
290 delete $task->{pstart};
291 };
292
293 foreach my $task (@$tasklist) {
294 my $upid = $task->{upid};
295 next if $thash->{$upid};
296 $thash->{$upid} = $task;
297 &$check_task($task);
298 }
299
300 if ($new_upid && !(my $task = $thash->{$new_upid})) {
301 $task = PVE::Tools::upid_decode($new_upid);
302 $task->{upid} = $new_upid;
303 $thash->{$new_upid} = $task;
304 &$check_task($task, $nocheck);
305 }
306
307
308 @ta = sort { $b->{starttime} <=> $a->{starttime} } @ta;
309
310 my $save = defined($new_upid);
311
312 foreach my $task (@ta) {
313 next if $task->{endtime};
314 $task->{endtime} = time();
315 $task->{status} = PVE::Tools::upid_read_status($task->{upid});
316 $save = 1;
317 }
318
319 my $archive = '';
320 my @arlist = ();
321 foreach my $task (@ta) {
322 if (!$task->{saved}) {
323 $archive .= sprintf("%s %08X %s\n", $task->{upid}, $task->{endtime}, $task->{status});
324 $save = 1;
325 push @arlist, $task;
326 $task->{saved} = 1;
327 }
328 }
329
330 if ($archive) {
331 my $size = 0;
332 my $filename = "/var/log/pve/tasks/index";
333 eval {
334 my $fh = IO::File->new($filename, '>>', 0644) ||
335 die "unable to open file '$filename' - $!\n";
336 PVE::Tools::safe_print($filename, $fh, $archive);
337 $size = -s $fh;
338 close($fh) ||
339 die "unable to close file '$filename' - $!\n";
340 };
341 my $err = $@;
342 if ($err) {
343 syslog('err', $err);
344 foreach my $task (@arlist) { # mark as not saved
345 $task->{saved} = 0;
346 }
347 }
348 my $maxsize = 50000; # about 1000 entries
349 if ($size > $maxsize) {
350 rename($filename, "$filename.1");
351 }
352 }
353
354 # we try to reduce the amount of data
355 # list all running tasks and task and a few others
356 # try to limit to 25 tasks
357 my $max = 25 - scalar(@$tlist);
358 foreach my $task (@ta) {
359 last if $max <= 0;
360 push @$tlist, $task;
361 $max--;
362 }
363
364 PVE::INotify::write_file('active', $tlist) if $save;
365
366 return $tlist;
367 };
368
369 my $res = PVE::Tools::lock_file($lkfn, $timeout, $code);
370 die $@ if $@;
371
372 return $res;
373 }
374
375 my $kill_process_group = sub {
376 my ($pid, $pstart) = @_;
377
378 # send kill to process group (negative pid)
379 my $kpid = -$pid;
380
381 # always send signal to all pgrp members
382 kill(15, $kpid); # send TERM signal
383
384 # give max 5 seconds to shut down
385 for (my $i = 0; $i < 5; $i++) {
386 return if !PVE::ProcFSTools::check_process_running($pid, $pstart);
387 sleep (1);
388 }
389
390 # to be sure
391 kill(9, $kpid);
392 };
393
394 sub check_worker {
395 my ($self, $upid, $killit) = @_;
396
397 my $task = PVE::Tools::upid_decode($upid);
398
399 my $running = PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart});
400
401 return 0 if !$running;
402
403 if ($killit) {
404 &$kill_process_group($task->{pid});
405 return 0;
406 }
407
408 return 1;
409 }
410
411 # acts almost as tee: writes an output both to STDOUT and a task log,
412 # we differ as we're worker aware and look also at the childs control pipe,
413 # so we know if the function could be executed successfully or not.
414 my $tee_worker = sub {
415 my ($childfd, $ctrlfd, $taskfh, $cpid) = @_;
416
417 eval {
418 my $int_count = 0;
419 local $SIG{INT} = local $SIG{QUIT} = local $SIG{TERM} = sub {
420 # always send signal to all pgrp members
421 my $kpid = -$cpid;
422 if ($int_count < 3) {
423 kill(15, $kpid); # send TERM signal
424 } else {
425 kill(9, $kpid); # send KILL signal
426 }
427 $int_count++;
428 };
429 local $SIG{PIPE} = sub { die "broken pipe\n"; };
430
431 my $select = new IO::Select;
432 my $fh = IO::Handle->new_from_fd($childfd, 'r');
433 $select->add($fh);
434
435 my $readbuf = '';
436 my $count;
437 while ($select->count) {
438 my @handles = $select->can_read(1);
439 if (scalar(@handles)) {
440 my $count = sysread ($handles[0], $readbuf, 4096);
441 if (!defined ($count)) {
442 my $err = $!;
443 die "sync pipe read error: $err\n";
444 }
445 last if $count == 0; # eof
446
447 print $readbuf;
448 select->flush();
449
450 print $taskfh $readbuf;
451 $taskfh->flush();
452 } else {
453 # some commands daemonize without closing stdout
454 last if !PVE::ProcFSTools::check_process_running($cpid);
455 }
456 }
457
458 POSIX::read($ctrlfd, $readbuf, 4096);
459 if ($readbuf =~ m/^TASK OK\n?$/) {
460 # skip printing to stdout
461 print $taskfh $readbuf;
462 } elsif ($readbuf =~ m/^TASK ERROR: (.*)\n?$/) {
463 print STDERR "$1\n";
464 print $taskfh "\n$readbuf"; # ensure start on new line for webUI
465 } elsif ($readbuf =~ m/^TASK WARNINGS: (\d+)\n?$/) {
466 print STDERR "Task finished with $1 warning(s)!\n";
467 print $taskfh "\n$readbuf"; # ensure start on new line for webUI
468 } else {
469 die "got unexpected control message: $readbuf\n";
470 }
471 $taskfh->flush();
472 };
473 my $err = $@;
474
475 POSIX::close($childfd);
476 POSIX::close($ctrlfd);
477
478 if ($err) {
479 $err =~ s/\n/ /mg;
480 print STDERR "$err\n";
481 print $taskfh "TASK ERROR: $err\n";
482 }
483 };
484
485 # start long running workers
486 # STDIN is redirected to /dev/null
487 # STDOUT,STDERR are redirected to the filename returned by upid_decode
488 # NOTE: we simulate running in foreground if ($self->{type} eq 'cli')
489 sub fork_worker {
490 my ($self, $dtype, $id, $user, $function, $background) = @_;
491
492 $dtype = 'unknown' if !defined ($dtype);
493 $id = '' if !defined ($id);
494
495 # note: below is only used for the task log entry
496 $user = $self->get_user(1) // 'root@pam' if !defined($user);
497
498 my $sync = ($self->{type} eq 'cli' && !$background) ? 1 : 0;
499
500 local $SIG{INT} =
501 local $SIG{QUIT} =
502 local $SIG{PIPE} =
503 local $SIG{TERM} = 'IGNORE';
504
505 my $starttime = time ();
506
507 my @psync = POSIX::pipe();
508 my @csync = POSIX::pipe();
509 my @ctrlfd = POSIX::pipe() if $sync;
510
511 my $node = $self->{nodename};
512
513 my $cpid = fork();
514 die "unable to fork worker - $!" if !defined($cpid);
515
516 my $workerpuid = $cpid ? $cpid : $$;
517
518 my $pstart = PVE::ProcFSTools::read_proc_starttime($workerpuid) ||
519 die "unable to read process start time";
520
521 my $upid = PVE::Tools::upid_encode ({
522 node => $node, pid => $workerpuid, pstart => $pstart,
523 starttime => $starttime, type => $dtype, id => $id, user => $user });
524
525 my $outfh;
526
527 if (!$cpid) { # child
528
529 $0 = "task $upid";
530 $WORKER_FLAG = 1;
531
532 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub { die "received interrupt\n"; };
533
534 $SIG{CHLD} = $SIG{PIPE} = 'DEFAULT';
535 $SIG{TTOU} = 'IGNORE';
536
537 my $ppgid;
538 # set session/process group allows to kill the process group
539 if ($sync && -t STDIN) {
540 # some sync'ed workers operate on the tty but setsid sessions lose
541 # the tty, so just create a new pgroup and give it the tty
542 $ppgid = POSIX::getpgrp() or die "failed to get old pgid: $!\n";
543 POSIX::setpgid(0, 0) or die "failed to setpgid: $!\n";
544 POSIX::tcsetpgrp(fileno(STDIN), $$) or die "failed to tcsetpgrp: $!\n";
545 } else {
546 POSIX::setsid();
547 }
548
549 POSIX::close ($psync[0]);
550 POSIX::close ($ctrlfd[0]) if $sync;
551 POSIX::close ($csync[1]);
552
553 $outfh = $sync ? $psync[1] : undef;
554 my $resfh = $sync ? $ctrlfd[1] : undef;
555
556 eval {
557 PVE::INotify::inotify_close();
558
559 if (my $atfork = $self->{atfork}) {
560 &$atfork();
561 }
562
563 # same algorithm as used inside SA
564 # STDIN = /dev/null
565 my $fd = fileno (STDIN);
566
567 if (!$sync) {
568 close STDIN;
569 POSIX::close(0) if $fd != 0;
570
571 die "unable to redirect STDIN - $!"
572 if !open(STDIN, "</dev/null");
573
574 $outfh = PVE::Tools::upid_open($upid);
575 $resfh = fileno($outfh);
576 }
577
578
579 # redirect STDOUT
580 $fd = fileno(STDOUT);
581 close STDOUT;
582 POSIX::close (1) if $fd != 1;
583
584 die "unable to redirect STDOUT - $!"
585 if !open(STDOUT, ">&", $outfh);
586
587 STDOUT->autoflush (1);
588
589 # redirect STDERR to STDOUT
590 $fd = fileno (STDERR);
591 close STDERR;
592 POSIX::close(2) if $fd != 2;
593
594 die "unable to redirect STDERR - $!"
595 if !open(STDERR, ">&1");
596
597 STDERR->autoflush(1);
598 };
599 if (my $err = $@) {
600 my $msg = "ERROR: $err";
601 POSIX::write($psync[1], $msg, length ($msg));
602 POSIX::close($psync[1]);
603 POSIX::_exit(1);
604 kill(-9, $$);
605 }
606
607 # sync with parent (signal that we are ready)
608 POSIX::write($psync[1], $upid, length ($upid));
609 POSIX::close($psync[1]) if !$sync; # don't need output pipe if async
610
611 eval {
612 my $readbuf = '';
613 # sync with parent (wait until parent is ready)
614 POSIX::read($csync[0], $readbuf, 4096);
615 die "parent setup error\n" if $readbuf ne 'OK';
616
617 if ($self->{type} eq 'ha') {
618 print "task started by HA resource agent\n";
619 }
620 &$function($upid);
621 };
622 my ($msg, $exitcode);
623 my $err = $@;
624 if ($err) {
625 chomp $err;
626 $err =~ s/\n/ /mg;
627 syslog('err', $err);
628 $msg = "TASK ERROR: $err\n";
629 $exitcode = -1;
630 } elsif (my $warnings = $self->{warning_count}) {
631 $msg = "TASK WARNINGS: $warnings\n";
632 $exitcode = 0;
633 } else {
634 $msg = "TASK OK\n";
635 $exitcode = 0;
636 }
637 POSIX::write($resfh, $msg, length($msg));
638
639 if ($sync) {
640 POSIX::close($resfh);
641 if ( -t STDIN) {
642 POSIX::tcsetpgrp(fileno(STDIN), $ppgid) or
643 die "failed to tcsetpgrp to parent: $!\n";
644 }
645 }
646 POSIX::_exit($exitcode);
647 kill(-9, $$); # not really needed, just to be sure
648 }
649
650 # parent
651
652 POSIX::close ($psync[1]);
653 POSIX::close ($ctrlfd[1]) if $sync;
654 POSIX::close ($csync[0]);
655
656 my $readbuf = '';
657 # sync with child (wait until child starts)
658 POSIX::read($psync[0], $readbuf, 4096);
659
660 if (!$sync) {
661 POSIX::close($psync[0]);
662 &$register_worker($cpid, $user, $upid);
663 } else {
664 chomp $readbuf;
665 }
666
667 eval {
668 die "got no worker upid - start worker failed\n" if !$readbuf;
669
670 if ($readbuf =~ m/^ERROR:\s*(.+)$/m) {
671 die "starting worker failed: $1\n";
672 }
673
674 if ($readbuf ne $upid) {
675 die "got strange worker upid ('$readbuf' != '$upid') - start worker failed\n";
676 }
677
678 if ($sync) {
679 $outfh = PVE::Tools::upid_open($upid);
680 }
681 };
682 my $err = $@;
683
684 if (!$err) {
685 my $msg = 'OK';
686 POSIX::write($csync[1], $msg, length ($msg));
687 POSIX::close($csync[1]);
688
689 } else {
690 POSIX::close($csync[1]);
691 kill(-9, $cpid); # make sure it gets killed
692 die $err;
693 }
694
695 $self->log_cluster_msg('info', $user, "starting task $upid");
696
697 my $tlist = $self->active_workers($upid, $sync);
698 eval { $self->broadcast_tasklist($tlist); };
699 syslog('err', $@) if $@;
700
701 my $res = 0;
702
703 if ($sync) {
704
705 $tee_worker->($psync[0], $ctrlfd[0], $outfh, $cpid);
706
707 &$kill_process_group($cpid, $pstart); # make sure it gets killed
708
709 close($outfh);
710
711 waitpid($cpid, 0);
712 $res = $?;
713 &$log_task_result($upid, $user, $res);
714 }
715
716 return wantarray ? ($upid, $res) : $upid;
717 }
718
719 sub log_warn {
720 my ($message) = @_;
721
722 if ($rest_env) {
723 $rest_env->warn($message);
724 } else {
725 chomp($message);
726 print STDERR "WARN: $message\n";
727 }
728 }
729
730 sub warn {
731 my ($self, $message) = @_;
732
733 chomp($message);
734
735 print STDERR "WARN: $message\n";
736
737 $self->{warning_count}++;
738 }
739
740 # Abstract function
741
742 sub log_cluster_msg {
743 my ($self, $pri, $user, $msg) = @_;
744
745 syslog($pri, "%s", $msg);
746
747 # PVE::Cluster::log_msg($pri, $user, $msg);
748 }
749
750 sub broadcast_tasklist {
751 my ($self, $tlist) = @_;
752
753 # PVE::Cluster::broadcast_tasklist($tlist);
754 }
755
756 sub check_api2_permissions {
757 my ($self, $perm, $username, $param) = @_;
758
759 return 1 if !$username && $perm->{user} eq 'world';
760
761 raise_perm_exc("user != null") if !$username;
762
763 return 1 if $username eq 'root@pam';
764
765 raise_perm_exc('user != root@pam') if !$perm;
766
767 return 1 if $perm->{user} && $perm->{user} eq 'all';
768
769 ##return $self->exec_api2_perm_check($perm->{check}, $username, $param)
770 ##if $perm->{check};
771
772 raise_perm_exc();
773 }
774
775 # init_request - should be called before each REST/CLI request
776 sub init_request {
777 my ($self, %params) = @_;
778
779 $self->{result_attributes} = {}
780
781 # if you nedd more, implement in subclass
782 }
783
784 1;