bin/pvestatd

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5 use PVE::SafeSyslog;
   6 use POSIX ":sys_wait_h";
   7 use Fcntl ':flock';
   8 use Getopt::Long;
   9 use Time::HiRes qw (gettimeofday);
  10 use PVE::Tools qw(dir_glob_foreach file_read_firstline);
  11 use PVE::ProcFSTools;
  12 use Filesys::Df;
  13 use PVE::INotify;
  14 use PVE::Cluster qw(cfs_read_file);
  15 use PVE::Storage;
  16 use PVE::QemuServer;
  17 use PVE::OpenVZ;
  18 use PVE::RPCEnvironment;
  19 use PVE::API2::Subscription;
  20 use PVE::AutoBalloon;
  21
  22 $SIG{'__WARN__'} = sub {
  23     my $err = $@;
  24     my $t = $_[0];
  25     chomp $t;
  26     syslog('warning', "WARNING: %s", $t);
  27     $@ = $err;
  28 };
  29
  30 initlog('pvestatd');
  31
  32 $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
  33
  34 die "please run as root\n" if $> != 0;
  35
  36 my $nodename = PVE::INotify::nodename();
  37
  38 my $opt_debug;
  39
  40 if (!GetOptions ('debug' => \$opt_debug)) {
  41     die "USAGE: $0 [--debug]\n";
  42 }
  43
  44 my $opt_pidfile = "/var/run/pvestatd.pid";
  45
  46 sub lockpidfile {
  47     my $pidfile = shift;
  48     my $lkfn = "$pidfile.lock";
  49
  50     if (!open (FLCK, ">>$lkfn")) {
  51         my $msg = "can't aquire lock on file '$lkfn' - $!";
  52         syslog ('err', $msg);
  53         die "ERROR: $msg\n";
  54     }
  55
  56     if (!flock (FLCK, LOCK_EX|LOCK_NB)) {
  57         close (FLCK);
  58         my $msg = "can't aquire lock '$lkfn' - $!";
  59         syslog ('err', $msg);
  60         die "ERROR: $msg\n";
  61     }
  62 }
  63
  64 sub writepidfile {
  65     my $pidfile = shift;
  66
  67     if (!open (PIDFH, ">$pidfile")) {
  68         my $msg = "can't open pid file '$pidfile' - $!";
  69         syslog ('err', $msg);
  70         die "ERROR: $msg\n";
  71     }
  72     print PIDFH "$$\n";
  73     close (PIDFH);
  74 }
  75
  76 # try to get the lock
  77 lockpidfile($opt_pidfile);
  78
  79 # run in background
  80 my $spid;
  81
  82 my $restart = $ENV{RESTART_PVESTATD};
  83
  84 if (!$opt_debug) {
  85     open STDIN,  '</dev/null' || die "can't read /dev/null";
  86     open STDOUT, '>/dev/null' || die "can't write /dev/null";
  87 }
  88
  89 if (!$restart && !$opt_debug) {
  90     $spid = fork();
  91     if (!defined ($spid)) {
  92         my $msg =  "can't put server into background - fork failed";
  93         syslog('err', $msg);
  94         die "ERROR: $msg\n";
  95     } elsif ($spid) { #parent
  96         exit (0);
  97     }
  98 }
  99
 100 writepidfile($opt_pidfile);
 101
 102 open STDERR, '>&STDOUT' || die "can't close STDERR\n";
 103
 104 sub cleanup {
 105     unlink "$opt_pidfile.lock";
 106     unlink "$opt_pidfile";
 107 }
 108
 109 $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = sub {
 110     syslog('info' , "server closing");
 111
 112     $SIG{INT} = 'DEFAULT';
 113
 114     # wait for children
 115     1 while (waitpid(-1, POSIX::WNOHANG()) > 0);
 116
 117     cleanup();
 118
 119     exit (0);
 120 };
 121
 122 PVE::INotify::inotify_init();
 123
 124 my $reload_config;
 125
 126 if ($restart) {
 127     syslog('info' , "restarting server");
 128 } else {
 129     syslog('info' , "starting server");
 130 }
 131
 132 $SIG{HUP} = sub {
 133     $reload_config = 1;
 134 };
 135
 136 sub update_node_status {
 137
 138     my ($avg1, $avg5, $avg15) = PVE::ProcFSTools::read_loadavg();
 139
 140     my $stat = PVE::ProcFSTools::read_proc_stat();
 141
 142     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
 143
 144     my ($uptime) = PVE::ProcFSTools::read_proc_uptime();
 145
 146     my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 147
 148     my $maxcpu = $cpuinfo->{cpus};
 149
 150     my $subinfo = PVE::INotify::read_file('subscription');
 151     my $sublevel = $subinfo->{level} || '';
 152
 153     # traffic from/to physical interface cards
 154     my $netin = 0;
 155     my $netout = 0;
 156     foreach my $dev (keys %$netdev) {
 157         next if $dev !~ m/^eth\d+$/;
 158         $netin += $netdev->{$dev}->{receive};
 159         $netout += $netdev->{$dev}->{transmit};
 160     }
 161
 162     my $meminfo = PVE::ProcFSTools::read_meminfo();
 163
 164     my $dinfo = df('/', 1);     # output is bytes
 165
 166     my $ctime = time();
 167
 168     # everything not free is considered to be used
 169     my $dused = $dinfo->{blocks} - $dinfo->{bfree};
 170
 171     my $data = "$uptime:$sublevel:$ctime:$avg1:$maxcpu:$stat->{cpu}:$stat->{wait}:" .
 172         "$meminfo->{memtotal}:$meminfo->{memused}:" .
 173         "$meminfo->{swaptotal}:$meminfo->{swapused}:" .
 174         "$dinfo->{blocks}:$dused:$netin:$netout";
 175
 176     PVE::Cluster::broadcast_rrd("pve2-node/$nodename", $data);
 177 }
 178
 179 sub auto_balloning {
 180     my ($vmstatus) =  @_;
 181
 182     my $log = sub {
 183        return if !$opt_debug;
 184        print @_;
 185     };
 186
 187     my $hostmeminfo = PVE::ProcFSTools::read_meminfo();
 188
 189     # to debug, run 'pvestatd -d' and set  memtotal here
 190     #$hostmeminfo->{memtotal} = int(2*1024*1024*1024/0.8); # you can set this to test
 191
 192     my $hostfreemem = $hostmeminfo->{memtotal} - $hostmeminfo->{memused};
 193
 194     # we try to use about 80% host memory
 195     # goal: we want to change memory usage by this amount (positive or negative)
 196     my $goal = int($hostmeminfo->{memtotal}*0.8 - $hostmeminfo->{memused});
 197
 198     my $maxchange = 100*1024*1024;
 199     my $res = PVE::AutoBalloon::compute_alg1($vmstatus, $goal, $maxchange);
 200
 201     &$log("host goal: $goal free: $hostfreemem total: $hostmeminfo->{memtotal}\n");
 202
 203     foreach my $vmid (keys %$vmstatus) {
 204         next if !$res->{$vmid};
 205         my $d = $vmstatus->{$vmid};
 206         my $diff = int($res->{$vmid} - $d->{balloon});
 207         my $absdiff = $diff < 0 ? -$diff : $diff;
 208         if ($absdiff > 0) {
 209             &$log("BALLOON $vmid to $res->{$vmid} ($diff)\n");
 210             eval {
 211                 PVE::QemuServer::vm_mon_cmd($vmid, "balloon",
 212                                             value => int($res->{$vmid}));
 213             };
 214             warn $@ if $@;
 215         }
 216     }
 217 }
 218
 219 sub update_qemu_status {
 220
 221     my $ctime = time();
 222
 223     my $vmstatus = PVE::QemuServer::vmstatus(undef, 1);
 224
 225     eval { auto_balloning($vmstatus); };
 226     syslog('err', "auto ballooning error: $@") if $@;
 227
 228     foreach my $vmid (keys %$vmstatus) {
 229         my $d = $vmstatus->{$vmid};
 230         my $data;
 231         my $status = $d->{qmpstatus} || $d->{status} || 'stopped';
 232         my $template = $d->{template} ? $d->{template} : "0";
 233         if ($d->{pid}) { # running
 234             $data = "$d->{uptime}:$d->{name}:$status:$template:" .
 235                 "$ctime:$d->{cpus}:$d->{cpu}:" .
 236                 "$d->{maxmem}:$d->{mem}:" .
 237                 "$d->{maxdisk}:$d->{disk}:" .
 238                 "$d->{netin}:$d->{netout}:" .
 239                 "$d->{diskread}:$d->{diskwrite}";
 240         } else {
 241             $data = "0:$d->{name}:$status:$template:$ctime:$d->{cpus}::" .
 242                 "$d->{maxmem}::" .
 243                 "$d->{maxdisk}:$d->{disk}:" .
 244                 ":::";
 245         }
 246         PVE::Cluster::broadcast_rrd("pve2.3-vm/$vmid", $data);
 247     }
 248 }
 249
 250 sub find_vzctl_console_pids {
 251
 252     my $res = {};
 253
 254     dir_glob_foreach('/proc', '\d+', sub {
 255         my ($pid) = @_;
 256
 257         my $cmdline = file_read_firstline("/proc/$pid/cmdline");
 258         return if !$cmdline;
 259
 260         my @args = split(/\0/, $cmdline);
 261
 262         # serach for vzctl console <vmid>
 263         return if scalar(@args) != 3;
 264         return if $args[1] ne 'console';
 265         return if $args[2] !~ m/^\d+$/;
 266         return if $args[0] !~ m|^(/usr/sbin/)?vzctl$|;
 267
 268         my $vmid = $args[2];
 269
 270         push @{$res->{$vmid}}, $pid;
 271     });
 272
 273     return $res;
 274 }
 275 sub remove_stale_openvz_consoles {
 276
 277     my $vmstatus = PVE::OpenVZ::vmstatus();
 278     my $pidhash = find_vzctl_console_pids();
 279
 280     foreach my $vmid (keys %$pidhash) {
 281         next if defined($vmstatus->{$vmid});
 282         syslog('info', "remove stale vzctl console for CT $vmid");
 283         foreach my $pid (@{$pidhash->{$vmid}}) {
 284             kill(9, $pid);
 285         }
 286     }
 287 }
 288
 289 sub update_openvz_status {
 290
 291     my $ctime = time();
 292
 293     my $vmstatus = PVE::OpenVZ::vmstatus();
 294
 295     foreach my $vmid (keys %$vmstatus) {
 296         my $d = $vmstatus->{$vmid};
 297         my $data;
 298         if ($d->{status} eq 'running') { # running
 299             $data = "$d->{uptime}:$d->{name}:$d->{status}:0:$ctime:$d->{cpus}:$d->{cpu}:" .
 300                 "$d->{maxmem}:$d->{mem}:" .
 301                 "$d->{maxdisk}:$d->{disk}:" .
 302                 "$d->{netin}:$d->{netout}:" .
 303                 "$d->{diskread}:$d->{diskwrite}";
 304         } else {
 305             $data = "0:$d->{name}:$d->{status}:0:$ctime:$d->{cpus}::" .
 306                 "$d->{maxmem}::" .
 307                 "$d->{maxdisk}:$d->{disk}:" .
 308                 ":::";
 309         }
 310         PVE::Cluster::broadcast_rrd("pve2.3-vm/$vmid", $data);
 311     }
 312 }
 313
 314 sub update_storage_status {
 315
 316     my $cfg = cfs_read_file("storage.cfg");
 317
 318     my $ctime = time();
 319
 320     my $info = PVE::Storage::storage_info($cfg);
 321
 322     foreach my $storeid (keys %$info) {
 323         my $d = $info->{$storeid};
 324         next if !$d->{active};
 325
 326         # everything not free is considered to be used
 327         my $realused = $d->{total} - $d->{avail};
 328
 329         my $data = "$ctime:$d->{total}:$realused";
 330
 331         my $key = "pve2-storage/${nodename}/$storeid";
 332         PVE::Cluster::broadcast_rrd($key, $data);
 333     }
 334 }
 335
 336 sub update_status {
 337
 338     # update worker list. This is not really required and
 339     # we just call this to make sure that we have a correct
 340     # list in case of an unexpected crash.
 341     eval {
 342         my $tlist = PVE::RPCEnvironment::active_workers();
 343         PVE::Cluster::broadcast_tasklist($tlist);
 344     };
 345     my $err = $@;
 346     syslog('err', $err) if $err;
 347
 348     eval {
 349         update_node_status();
 350     };
 351     $err = $@;
 352     syslog('err', "node status update error: $err") if $err;
 353
 354     eval {
 355         update_qemu_status();
 356     };
 357     $err = $@;
 358     syslog('err', "qemu status update error: $err") if $err;
 359
 360     eval {
 361         update_openvz_status();
 362     };
 363     $err = $@;
 364     syslog('err', "openvz status update error: $err") if $err;
 365
 366     eval {
 367         update_storage_status();
 368     };
 369     $err = $@;
 370     syslog('err', "storage status update error: $err") if $err;
 371
 372     eval {
 373         remove_stale_openvz_consoles();
 374     };
 375     $err = $@;
 376     syslog('err', "openvz console cleanup error: $err") if $err;
 377 }
 378
 379 my $next_update = 0;
 380
 381 # do not update directly after startup, because install scripts
 382 # have a problem with that
 383 my $cycle = 0;
 384 my $updatetime = 10;
 385
 386 my $commandline = [$0, @ARGV];
 387
 388 $0 = "pvestatd";
 389
 390 sub restart_server {
 391     my $waittime = shift;
 392
 393     syslog('info', "server shutdown (restart)");
 394
 395     $ENV{RESTART_PVESTATD} = 1;
 396
 397     sleep($waittime) if $waittime; # avoid high server load due to restarts
 398
 399     exec (@$commandline);
 400     exit (-1); # never reached?
 401 }
 402
 403 my $initial_memory_usage;
 404
 405 for (;;) { # forever
 406
 407     eval {
 408         $next_update = time() + $updatetime;
 409
 410         if ($cycle) {
 411             my ($ccsec, $cusec) = gettimeofday ();
 412             eval {
 413                 $reload_config = 0;
 414                 # syslog('info', "start status update");
 415                 PVE::Cluster::cfs_update();
 416                 update_status();
 417             };
 418             my $err = $@;
 419
 420             if ($err) {
 421                 syslog('err', "status update error: $err");
 422             }
 423
 424             my ($ccsec_end, $cusec_end) = gettimeofday ();
 425             my $cptime = ($ccsec_end-$ccsec) + ($cusec_end - $cusec)/1000000;
 426
 427             syslog('info', sprintf("status update time (%.3f seconds)", $cptime))
 428                 if ($cptime > 5);
 429         }
 430
 431         $cycle++;
 432
 433         my $mem = PVE::ProcFSTools::read_memory_usage();
 434
 435         if (!defined($initial_memory_usage) || ($cycle < 10)) {
 436             $initial_memory_usage = $mem->{resident};
 437         } else {
 438             my $diff = $mem->{resident} - $initial_memory_usage;
 439             if ($diff > 5*1024*1024) {
 440                 syslog ('info', "restarting server after $cycle cycles to " .
 441                         "reduce memory usage (free $mem->{resident} ($diff) bytes)");
 442                 restart_server ();
 443             }
 444         }
 445
 446         my $wcount = 0;
 447         while ((time() < $next_update) &&
 448                ($wcount < $updatetime) && # protect against time wrap
 449                !$reload_config) { $wcount++; sleep (1); };
 450     };
 451
 452     my $err = $@;
 453
 454     if ($err) {
 455         syslog ('err', "ERROR: $err");
 456         restart_server(5);
 457         exit (0);
 458     }
 459 }
 460
 461 exit (0);
 462
 463 __END__
 464
 465 =head1 NAME
 466
 467 pvestatd - PVE Status Daemon
 468
 469 =head1 SYNOPSIS
 470
 471 pvestatd
 472
 473 =head1 DESCRIPTION
 474
 475 Documentation is available at www.proxmox.com
 476
 477
 478
 479
 480