bin/pvestatd

   1 #!/usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5 use PVE::SafeSyslog;
   6 use PVE::Daemon;
   7
   8 use Time::HiRes qw (gettimeofday);
   9 use PVE::Tools qw(dir_glob_foreach file_read_firstline);
  10 use PVE::ProcFSTools;
  11 use Filesys::Df;
  12 use PVE::INotify;
  13 use PVE::Cluster qw(cfs_read_file);
  14 use PVE::Storage;
  15 use PVE::QemuServer;
  16 use PVE::OpenVZ;
  17 use PVE::RPCEnvironment;
  18 use PVE::API2::Subscription;
  19 use PVE::AutoBalloon;
  20
  21 use base qw(PVE::Daemon);
  22
  23 my $opt_debug;
  24
  25 my $cmdline = [$0, @ARGV];
  26
  27 my %daemon_options = (restart_on_error => 5, stop_wait_time => 5);
  28
  29 my $daemon = __PACKAGE__->new('pvestatd', $cmdline, %daemon_options);
  30
  31 my $rpcenv = PVE::RPCEnvironment->init('cli');
  32
  33 $rpcenv->init_request();
  34 $rpcenv->set_language($ENV{LANG});
  35 $rpcenv->set_user('root@pam');
  36
  37 my $nodename = PVE::INotify::nodename();
  38 my $restart_request = 0;
  39
  40 sub init {
  41     my ($self) = @_;
  42
  43     PVE::Cluster::cfs_update();
  44 }
  45
  46 sub shutdown {
  47     my ($self) = @_;
  48
  49     syslog('info' , "server closing");
  50
  51     # wait for children
  52     1 while (waitpid(-1, POSIX::WNOHANG()) > 0);
  53
  54     $self->exit_daemon(0);
  55 }
  56
  57 sub hup {
  58     my ($self) = @_;
  59
  60     syslog('info' , "received signal HUP");
  61
  62     $restart_request = 1;
  63 }
  64
  65 sub update_node_status {
  66
  67     my ($avg1, $avg5, $avg15) = PVE::ProcFSTools::read_loadavg();
  68
  69     my $stat = PVE::ProcFSTools::read_proc_stat();
  70
  71     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
  72
  73     my ($uptime) = PVE::ProcFSTools::read_proc_uptime();
  74
  75     my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  76
  77     my $maxcpu = $cpuinfo->{cpus};
  78
  79     my $subinfo = PVE::INotify::read_file('subscription');
  80     my $sublevel = $subinfo->{level} || '';
  81
  82     # traffic from/to physical interface cards
  83     my $netin = 0;
  84     my $netout = 0;
  85     foreach my $dev (keys %$netdev) {
  86         next if $dev !~ m/^eth\d+$/;
  87         $netin += $netdev->{$dev}->{receive};
  88         $netout += $netdev->{$dev}->{transmit};
  89     }
  90
  91     my $meminfo = PVE::ProcFSTools::read_meminfo();
  92
  93     my $dinfo = df('/', 1);     # output is bytes
  94
  95     my $ctime = time();
  96
  97     # everything not free is considered to be used
  98     my $dused = $dinfo->{blocks} - $dinfo->{bfree};
  99
 100     my $data = "$uptime:$sublevel:$ctime:$avg1:$maxcpu:$stat->{cpu}:$stat->{wait}:" .
 101         "$meminfo->{memtotal}:$meminfo->{memused}:" .
 102         "$meminfo->{swaptotal}:$meminfo->{swapused}:" .
 103         "$dinfo->{blocks}:$dused:$netin:$netout";
 104
 105     PVE::Cluster::broadcast_rrd("pve2-node/$nodename", $data);
 106 }
 107
 108 sub auto_balloning {
 109     my ($vmstatus) =  @_;
 110
 111     my $log = sub {
 112        return if !$opt_debug;
 113        print @_;
 114     };
 115
 116     my $hostmeminfo = PVE::ProcFSTools::read_meminfo();
 117
 118     # to debug, run 'pvestatd -d' and set  memtotal here
 119     #$hostmeminfo->{memtotal} = int(2*1024*1024*1024/0.8); # you can set this to test
 120
 121     my $hostfreemem = $hostmeminfo->{memtotal} - $hostmeminfo->{memused};
 122
 123     # we try to use about 80% host memory
 124     # goal: we want to change memory usage by this amount (positive or negative)
 125     my $goal = int($hostmeminfo->{memtotal}*0.8 - $hostmeminfo->{memused});
 126
 127     my $maxchange = 100*1024*1024;
 128     my $res = PVE::AutoBalloon::compute_alg1($vmstatus, $goal, $maxchange);
 129
 130     &$log("host goal: $goal free: $hostfreemem total: $hostmeminfo->{memtotal}\n");
 131
 132     foreach my $vmid (keys %$vmstatus) {
 133         next if !$res->{$vmid};
 134         my $d = $vmstatus->{$vmid};
 135         my $diff = int($res->{$vmid} - $d->{balloon});
 136         my $absdiff = $diff < 0 ? -$diff : $diff;
 137         if ($absdiff > 0) {
 138             &$log("BALLOON $vmid to $res->{$vmid} ($diff)\n");
 139             eval {
 140                 PVE::QemuServer::vm_mon_cmd($vmid, "balloon",
 141                                             value => int($res->{$vmid}));
 142             };
 143             warn $@ if $@;
 144         }
 145     }
 146 }
 147
 148 sub update_qemu_status {
 149
 150     my $ctime = time();
 151
 152     my $vmstatus = PVE::QemuServer::vmstatus(undef, 1);
 153
 154     eval { auto_balloning($vmstatus); };
 155     syslog('err', "auto ballooning error: $@") if $@;
 156
 157     foreach my $vmid (keys %$vmstatus) {
 158         my $d = $vmstatus->{$vmid};
 159         my $data;
 160         my $status = $d->{qmpstatus} || $d->{status} || 'stopped';
 161         my $template = $d->{template} ? $d->{template} : "0";
 162         if ($d->{pid}) { # running
 163             $data = "$d->{uptime}:$d->{name}:$status:$template:" .
 164                 "$ctime:$d->{cpus}:$d->{cpu}:" .
 165                 "$d->{maxmem}:$d->{mem}:" .
 166                 "$d->{maxdisk}:$d->{disk}:" .
 167                 "$d->{netin}:$d->{netout}:" .
 168                 "$d->{diskread}:$d->{diskwrite}";
 169         } else {
 170             $data = "0:$d->{name}:$status:$template:$ctime:$d->{cpus}::" .
 171                 "$d->{maxmem}::" .
 172                 "$d->{maxdisk}:$d->{disk}:" .
 173                 ":::";
 174         }
 175         PVE::Cluster::broadcast_rrd("pve2.3-vm/$vmid", $data);
 176     }
 177 }
 178
 179 sub find_vzctl_console_pids {
 180
 181     my $res = {};
 182
 183     dir_glob_foreach('/proc', '\d+', sub {
 184         my ($pid) = @_;
 185
 186         my $cmdline = file_read_firstline("/proc/$pid/cmdline");
 187         return if !$cmdline;
 188
 189         my @args = split(/\0/, $cmdline);
 190
 191         # serach for vzctl console <vmid>
 192         return if scalar(@args) != 3;
 193         return if $args[1] ne 'console';
 194         return if $args[2] !~ m/^\d+$/;
 195         return if $args[0] !~ m|^(/usr/sbin/)?vzctl$|;
 196
 197         my $vmid = $args[2];
 198
 199         push @{$res->{$vmid}}, $pid;
 200     });
 201
 202     return $res;
 203 }
 204
 205 sub remove_stale_openvz_consoles {
 206
 207     my $vmstatus = PVE::OpenVZ::vmstatus();
 208     my $pidhash = find_vzctl_console_pids();
 209
 210     foreach my $vmid (keys %$pidhash) {
 211         next if defined($vmstatus->{$vmid});
 212         syslog('info', "remove stale vzctl console for CT $vmid");
 213         foreach my $pid (@{$pidhash->{$vmid}}) {
 214             kill(9, $pid);
 215         }
 216     }
 217 }
 218
 219 sub update_openvz_status {
 220
 221     my $ctime = time();
 222
 223     my $vmstatus = PVE::OpenVZ::vmstatus();
 224
 225     foreach my $vmid (keys %$vmstatus) {
 226         my $d = $vmstatus->{$vmid};
 227         my $data;
 228         if ($d->{status} eq 'running') { # running
 229             $data = "$d->{uptime}:$d->{name}:$d->{status}:0:$ctime:$d->{cpus}:$d->{cpu}:" .
 230                 "$d->{maxmem}:$d->{mem}:" .
 231                 "$d->{maxdisk}:$d->{disk}:" .
 232                 "$d->{netin}:$d->{netout}:" .
 233                 "$d->{diskread}:$d->{diskwrite}";
 234         } else {
 235             $data = "0:$d->{name}:$d->{status}:0:$ctime:$d->{cpus}::" .
 236                 "$d->{maxmem}::" .
 237                 "$d->{maxdisk}:$d->{disk}:" .
 238                 ":::";
 239         }
 240         PVE::Cluster::broadcast_rrd("pve2.3-vm/$vmid", $data);
 241     }
 242 }
 243
 244 sub update_storage_status {
 245
 246     my $cfg = cfs_read_file("storage.cfg");
 247
 248     my $ctime = time();
 249
 250     my $info = PVE::Storage::storage_info($cfg);
 251
 252     foreach my $storeid (keys %$info) {
 253         my $d = $info->{$storeid};
 254         next if !$d->{active};
 255
 256         # everything not free is considered to be used
 257         my $realused = $d->{total} - $d->{avail};
 258
 259         my $data = "$ctime:$d->{total}:$realused";
 260
 261         my $key = "pve2-storage/${nodename}/$storeid";
 262         PVE::Cluster::broadcast_rrd($key, $data);
 263     }
 264 }
 265
 266 sub update_status {
 267
 268     # update worker list. This is not really required and
 269     # we just call this to make sure that we have a correct
 270     # list in case of an unexpected crash.
 271     eval {
 272         my $tlist = PVE::RPCEnvironment::active_workers();
 273         PVE::Cluster::broadcast_tasklist($tlist);
 274     };
 275     my $err = $@;
 276     syslog('err', $err) if $err;
 277
 278     eval {
 279         update_node_status();
 280     };
 281     $err = $@;
 282     syslog('err', "node status update error: $err") if $err;
 283
 284     eval {
 285         update_qemu_status();
 286     };
 287     $err = $@;
 288     syslog('err', "qemu status update error: $err") if $err;
 289
 290     eval {
 291         update_openvz_status();
 292     };
 293     $err = $@;
 294     syslog('err', "openvz status update error: $err") if $err;
 295
 296     eval {
 297         update_storage_status();
 298     };
 299     $err = $@;
 300     syslog('err', "storage status update error: $err") if $err;
 301
 302     eval {
 303         remove_stale_openvz_consoles();
 304     };
 305     $err = $@;
 306     syslog('err', "openvz console cleanup error: $err") if $err;
 307 }
 308
 309 my $next_update = 0;
 310
 311 # do not update directly after startup, because install scripts
 312 # have a problem with that
 313 my $cycle = 0;
 314 my $updatetime = 10;
 315
 316 my $initial_memory_usage;
 317
 318 sub run {
 319     my ($self) = @_;
 320
 321     for (;;) { # forever
 322
 323         $next_update = time() + $updatetime;
 324
 325         if ($cycle) {
 326             my ($ccsec, $cusec) = gettimeofday ();
 327             eval {
 328                 # syslog('info', "start status update");
 329                 PVE::Cluster::cfs_update();
 330                 update_status();
 331             };
 332             my $err = $@;
 333
 334             if ($err) {
 335                 syslog('err', "status update error: $err");
 336             }
 337
 338             my ($ccsec_end, $cusec_end) = gettimeofday ();
 339             my $cptime = ($ccsec_end-$ccsec) + ($cusec_end - $cusec)/1000000;
 340
 341             syslog('info', sprintf("status update time (%.3f seconds)", $cptime))
 342                 if ($cptime > 5);
 343         }
 344
 345         $cycle++;
 346
 347         my $mem = PVE::ProcFSTools::read_memory_usage();
 348
 349         if (!defined($initial_memory_usage) || ($cycle < 10)) {
 350             $initial_memory_usage = $mem->{resident};
 351         } else {
 352             my $diff = $mem->{resident} - $initial_memory_usage;
 353             if ($diff > 5*1024*1024) {
 354                 syslog ('info', "restarting server after $cycle cycles to " .
 355                         "reduce memory usage (free $mem->{resident} ($diff) bytes)");
 356                 $self->restart_daemon();
 357             }
 358         }
 359
 360         my $wcount = 0;
 361         while ((time() < $next_update) &&
 362                ($wcount < $updatetime) && # protect against time wrap
 363                !$restart_request) { $wcount++; sleep (1); };
 364
 365         $self->restart_daemon() if $restart_request;
 366     }
 367 }
 368
 369 $daemon->register_start_command(__PACKAGE__);
 370 $daemon->register_restart_command(__PACKAGE__);
 371 $daemon->register_stop_command(__PACKAGE__);
 372 $daemon->register_status_command(__PACKAGE__);
 373
 374 my $cmddef = {
 375     start => [ __PACKAGE__, 'start', []],
 376     restart => [ __PACKAGE__, 'restart', []],
 377     stop => [ __PACKAGE__, 'stop', []],
 378     status => [ __PACKAGE__, 'status', [], undef, sub { print shift . "\n";} ],
 379 };
 380
 381 my $cmd = shift;
 382
 383 PVE::CLIHandler::handle_cmd($cmddef, $0, $cmd, \@ARGV, undef, $0);
 384
 385 exit (0);
 386
 387 __END__
 388
 389 =head1 NAME
 390
 391 pvestatd - PVE Status Daemon
 392
 393 =head1 SYNOPSIS
 394
 395 =include synopsis
 396
 397 =head1 DESCRIPTION
 398
 399 This daemom queries the status of VMs, storages and containers at
 400 regular intervals. The result is sent to all nodes in the cluster.
 401
 402 =include pve_copyright
 403
 404
 405
 406
 407