PVE/Service/pvestatd.pm

   1 package PVE::Service::pvestatd;
   2
   3 use strict;
   4 use warnings;
   5
   6 use PVE::SafeSyslog;
   7 use PVE::Daemon;
   8
   9 use Time::HiRes qw (gettimeofday);
  10 use PVE::Tools qw(dir_glob_foreach file_read_firstline);
  11 use PVE::ProcFSTools;
  12 use Filesys::Df;
  13 use PVE::INotify;
  14 use PVE::Cluster qw(cfs_read_file);
  15 use PVE::Storage;
  16 use PVE::QemuServer;
  17 use PVE::LXC;
  18 use PVE::RPCEnvironment;
  19 use PVE::API2::Subscription;
  20 use PVE::AutoBalloon;
  21
  22 use PVE::Status::Plugin;
  23 use PVE::Status::Graphite;
  24 use PVE::Status::InfluxDB;
  25
  26 PVE::Status::Graphite->register();
  27 PVE::Status::InfluxDB->register();
  28 PVE::Status::Plugin->init();
  29
  30 use base qw(PVE::Daemon);
  31
  32 my $opt_debug;
  33 my $restart_request;
  34
  35 my $nodename = PVE::INotify::nodename();
  36
  37 my $cmdline = [$0, @ARGV];
  38
  39 my %daemon_options = (restart_on_error => 5, stop_wait_time => 5);
  40 my $daemon = __PACKAGE__->new('pvestatd', $cmdline, %daemon_options);
  41
  42 sub init {
  43     my ($self) = @_;
  44
  45     $opt_debug = $self->{debug};
  46
  47     PVE::Cluster::cfs_update();
  48 }
  49
  50 sub shutdown {
  51     my ($self) = @_;
  52
  53     syslog('info' , "server closing");
  54
  55     # wait for children
  56     1 while (waitpid(-1, POSIX::WNOHANG()) > 0);
  57
  58     $self->exit_daemon(0);
  59 }
  60
  61 sub hup {
  62     my ($self) = @_;
  63
  64     $restart_request = 1;
  65 }
  66
  67 sub update_node_status {
  68     my ($status_cfg) = @_;
  69
  70     my ($avg1, $avg5, $avg15) = PVE::ProcFSTools::read_loadavg();
  71
  72     my $stat = PVE::ProcFSTools::read_proc_stat();
  73
  74     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
  75
  76     my ($uptime) = PVE::ProcFSTools::read_proc_uptime();
  77
  78     my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  79
  80     my $maxcpu = $cpuinfo->{cpus};
  81
  82     my $subinfo = PVE::INotify::read_file('subscription');
  83     my $sublevel = $subinfo->{level} || '';
  84
  85     # traffic from/to physical interface cards
  86     my $netin = 0;
  87     my $netout = 0;
  88     foreach my $dev (keys %$netdev) {
  89         next if $dev !~ m/^eth\d+$/;
  90         $netin += $netdev->{$dev}->{receive};
  91         $netout += $netdev->{$dev}->{transmit};
  92     }
  93
  94     my $meminfo = PVE::ProcFSTools::read_meminfo();
  95
  96     my $dinfo = df('/', 1);     # output is bytes
  97
  98     my $ctime = time();
  99
 100     # everything not free is considered to be used
 101     my $dused = $dinfo->{blocks} - $dinfo->{bfree};
 102
 103     my $data = "$uptime:$sublevel:$ctime:$avg1:$maxcpu:$stat->{cpu}:$stat->{wait}:" .
 104         "$meminfo->{memtotal}:$meminfo->{memused}:" .
 105         "$meminfo->{swaptotal}:$meminfo->{swapused}:" .
 106         "$dinfo->{blocks}:$dused:$netin:$netout";
 107
 108     PVE::Cluster::broadcast_rrd("pve2-node/$nodename", $data);
 109
 110     foreach my $id (keys %{$status_cfg->{ids}}) {
 111         my $plugin_config = $status_cfg->{ids}->{$id};
 112         next if $plugin_config->{disable};
 113         my $plugin = PVE::Status::Plugin->lookup($plugin_config->{type});
 114
 115         my $d = {};
 116         $d->{uptime} = $uptime;
 117         $d->{cpustat} = $stat;
 118         $d->{cpustat}->{avg1} = $avg1;
 119         $d->{cpustat}->{avg5} = $avg5;
 120         $d->{cpustat}->{avg15} = $avg15;
 121         $d->{cpustat}->{cpus} = $maxcpu;
 122         $d->{memory} = $meminfo;
 123         $d->{blockstat} = $dinfo;
 124         $d->{nics} = $netdev;
 125
 126         $plugin->update_node_status($plugin_config, $nodename, $d, $ctime);
 127     }
 128 }
 129
 130 sub auto_balloning {
 131     my ($vmstatus) =  @_;
 132
 133     my $log = sub {
 134        return if !$opt_debug;
 135        print @_;
 136     };
 137
 138     my $hostmeminfo = PVE::ProcFSTools::read_meminfo();
 139
 140     # to debug, run 'pvestatd -d' and set  memtotal here
 141     #$hostmeminfo->{memtotal} = int(2*1024*1024*1024/0.8); # you can set this to test
 142
 143     my $hostfreemem = $hostmeminfo->{memtotal} - $hostmeminfo->{memused};
 144
 145     # we try to use about 80% host memory
 146     # goal: we want to change memory usage by this amount (positive or negative)
 147     my $goal = int($hostmeminfo->{memtotal}*0.8 - $hostmeminfo->{memused});
 148
 149     my $maxchange = 100*1024*1024;
 150     my $res = PVE::AutoBalloon::compute_alg1($vmstatus, $goal, $maxchange);
 151
 152     &$log("host goal: $goal free: $hostfreemem total: $hostmeminfo->{memtotal}\n");
 153
 154     foreach my $vmid (keys %$vmstatus) {
 155         next if !$res->{$vmid};
 156         my $d = $vmstatus->{$vmid};
 157         my $diff = int($res->{$vmid} - $d->{balloon});
 158         my $absdiff = $diff < 0 ? -$diff : $diff;
 159         if ($absdiff > 0) {
 160             &$log("BALLOON $vmid to $res->{$vmid} ($diff)\n");
 161             eval {
 162                 PVE::QemuServer::vm_mon_cmd($vmid, "balloon",
 163                                             value => int($res->{$vmid}));
 164             };
 165             warn $@ if $@;
 166         }
 167     }
 168 }
 169
 170 sub update_qemu_status {
 171     my ($status_cfg) = @_;
 172
 173     my $ctime = time();
 174
 175     my $vmstatus = PVE::QemuServer::vmstatus(undef, 1);
 176
 177     eval { auto_balloning($vmstatus); };
 178     syslog('err', "auto ballooning error: $@") if $@;
 179
 180     foreach my $vmid (keys %$vmstatus) {
 181         my $d = $vmstatus->{$vmid};
 182         my $data;
 183         my $status = $d->{qmpstatus} || $d->{status} || 'stopped';
 184         my $template = $d->{template} ? $d->{template} : "0";
 185         if ($d->{pid}) { # running
 186             $data = "$d->{uptime}:$d->{name}:$status:$template:" .
 187                 "$ctime:$d->{cpus}:$d->{cpu}:" .
 188                 "$d->{maxmem}:$d->{mem}:" .
 189                 "$d->{maxdisk}:$d->{disk}:" .
 190                 "$d->{netin}:$d->{netout}:" .
 191                 "$d->{diskread}:$d->{diskwrite}";
 192         } else {
 193             $data = "0:$d->{name}:$status:$template:$ctime:$d->{cpus}::" .
 194                 "$d->{maxmem}::" .
 195                 "$d->{maxdisk}:$d->{disk}:" .
 196                 ":::";
 197         }
 198         PVE::Cluster::broadcast_rrd("pve2.3-vm/$vmid", $data);
 199
 200         foreach my $id (keys %{$status_cfg->{ids}}) {
 201             my $plugin_config = $status_cfg->{ids}->{$id};
 202             next if $plugin_config->{disable};
 203             my $plugin = PVE::Status::Plugin->lookup($plugin_config->{type});
 204             $plugin->update_qemu_status($plugin_config, $vmid, $d, $ctime);
 205         }
 206     }
 207 }
 208
 209 sub remove_stale_lxc_consoles {
 210
 211     my $vmstatus = PVE::LXC::vmstatus();
 212     my $pidhash = PVE::LXC::find_lxc_console_pids();
 213
 214     foreach my $vmid (keys %$pidhash) {
 215         next if defined($vmstatus->{$vmid});
 216         syslog('info', "remove stale lxc-console for CT $vmid");
 217         foreach my $pid (@{$pidhash->{$vmid}}) {
 218             kill(9, $pid);
 219         }
 220     }
 221 }
 222
 223 sub update_lxc_status {
 224     my ($status_cfg) = @_;
 225
 226     my $ctime = time();
 227
 228     my $vmstatus = PVE::LXC::vmstatus();
 229
 230     foreach my $vmid (keys %$vmstatus) {
 231         my $d = $vmstatus->{$vmid};
 232         my $template = $d->{template} ? $d->{template} : "0";
 233         my $data;
 234         if ($d->{status} eq 'running') { # running
 235             $data = "$d->{uptime}:$d->{name}:$d->{status}:$template:" .
 236                 "$ctime:$d->{cpus}:$d->{cpu}:" .
 237                 "$d->{maxmem}:$d->{mem}:" .
 238                 "$d->{maxdisk}:$d->{disk}:" .
 239                 "$d->{netin}:$d->{netout}:" .
 240                 "$d->{diskread}:$d->{diskwrite}";
 241         } else {
 242             $data = "0:$d->{name}:$d->{status}:$template:$ctime:$d->{cpus}::" .
 243                 "$d->{maxmem}::" .
 244                 "$d->{maxdisk}:$d->{disk}:" .
 245                 ":::";
 246         }
 247         PVE::Cluster::broadcast_rrd("pve2.3-vm/$vmid", $data);
 248
 249         foreach my $id (keys %{$status_cfg->{ids}}) {
 250             my $plugin_config = $status_cfg->{ids}->{$id};
 251             next if $plugin_config->{disable};
 252             my $plugin = PVE::Status::Plugin->lookup($plugin_config->{type});
 253             $plugin->update_lxc_status($plugin_config, $vmid, $d, $ctime);
 254         }
 255     }
 256 }
 257
 258 sub update_storage_status {
 259     my ($status_cfg) = @_;
 260
 261     my $cfg = cfs_read_file("storage.cfg");
 262
 263     my $ctime = time();
 264
 265     my $info = PVE::Storage::storage_info($cfg);
 266
 267     foreach my $storeid (keys %$info) {
 268         my $d = $info->{$storeid};
 269         next if !$d->{active};
 270
 271         my $data = "$ctime:$d->{total}:$d->{used}";
 272
 273         my $key = "pve2-storage/${nodename}/$storeid";
 274         PVE::Cluster::broadcast_rrd($key, $data);
 275
 276         foreach my $id (keys %{$status_cfg->{ids}}) {
 277             my $plugin_config = $status_cfg->{ids}->{$id};
 278             next if $plugin_config->{disable};
 279             my $plugin = PVE::Status::Plugin->lookup($plugin_config->{type});
 280             $plugin->update_storage_status($plugin_config, $nodename, $storeid, $d, $ctime);
 281         }
 282     }
 283 }
 284
 285 sub update_status {
 286
 287     # update worker list. This is not really required and
 288     # we just call this to make sure that we have a correct
 289     # list in case of an unexpected crash.
 290     eval {
 291         my $tlist = PVE::RPCEnvironment::active_workers();
 292         PVE::Cluster::broadcast_tasklist($tlist);
 293     };
 294     my $err = $@;
 295     syslog('err', $err) if $err;
 296
 297     my $status_cfg = PVE::Cluster::cfs_read_file('status.cfg');
 298
 299     eval {
 300         update_node_status($status_cfg);
 301     };
 302     $err = $@;
 303     syslog('err', "node status update error: $err") if $err;
 304
 305     eval {
 306         update_qemu_status($status_cfg);
 307     };
 308     $err = $@;
 309     syslog('err', "qemu status update error: $err") if $err;
 310
 311     eval {
 312         update_lxc_status($status_cfg);
 313     };
 314     $err = $@;
 315     syslog('err', "lxc status update error: $err") if $err;
 316
 317     eval {
 318         update_storage_status($status_cfg);
 319     };
 320     $err = $@;
 321     syslog('err', "storage status update error: $err") if $err;
 322
 323     eval {
 324         remove_stale_lxc_consoles();
 325     };
 326     $err = $@;
 327     syslog('err', "lxc console cleanup error: $err") if $err;
 328 }
 329
 330 my $next_update = 0;
 331
 332 # do not update directly after startup, because install scripts
 333 # have a problem with that
 334 my $cycle = 0;
 335 my $updatetime = 10;
 336
 337 my $initial_memory_usage;
 338
 339 sub run {
 340     my ($self) = @_;
 341
 342     for (;;) { # forever
 343
 344         $next_update = time() + $updatetime;
 345
 346         if ($cycle) {
 347             my ($ccsec, $cusec) = gettimeofday ();
 348             eval {
 349                 # syslog('info', "start status update");
 350                 PVE::Cluster::cfs_update();
 351                 update_status();
 352             };
 353             my $err = $@;
 354
 355             if ($err) {
 356                 syslog('err', "status update error: $err");
 357             }
 358
 359             my ($ccsec_end, $cusec_end) = gettimeofday ();
 360             my $cptime = ($ccsec_end-$ccsec) + ($cusec_end - $cusec)/1000000;
 361
 362             syslog('info', sprintf("status update time (%.3f seconds)", $cptime))
 363                 if ($cptime > 5);
 364         }
 365
 366         $cycle++;
 367
 368         my $mem = PVE::ProcFSTools::read_memory_usage();
 369
 370         if (!defined($initial_memory_usage) || ($cycle < 10)) {
 371             $initial_memory_usage = $mem->{resident};
 372         } else {
 373             my $diff = $mem->{resident} - $initial_memory_usage;
 374             if ($diff > 5*1024*1024) {
 375                 syslog ('info', "restarting server after $cycle cycles to " .
 376                         "reduce memory usage (free $mem->{resident} ($diff) bytes)");
 377                 $self->restart_daemon();
 378             }
 379         }
 380
 381         my $wcount = 0;
 382         while ((time() < $next_update) &&
 383                ($wcount < $updatetime) && # protect against time wrap
 384                !$restart_request) { $wcount++; sleep (1); };
 385
 386         $self->restart_daemon() if $restart_request;
 387     }
 388 }
 389
 390 $daemon->register_start_command();
 391 $daemon->register_restart_command(1);
 392 $daemon->register_stop_command();
 393 $daemon->register_status_command();
 394
 395 our $cmddef = {
 396     start => [ __PACKAGE__, 'start', []],
 397     restart => [ __PACKAGE__, 'restart', []],
 398     stop => [ __PACKAGE__, 'stop', []],
 399     status => [ __PACKAGE__, 'status', [], undef, sub { print shift . "\n";} ],
 400 };
 401
 402 #my $cmd = shift;
 403 #PVE::CLIHandler::handle_cmd($cmddef, $0, $cmd, \@ARGV, undef, $0);
 404 #exit (0);
 405
 406 1;
 407
 408 __END__
 409
 410 =head1 NAME
 411
 412 pvestatd - PVE Status Daemon
 413
 414 =head1 SYNOPSIS
 415
 416 =include synopsis
 417
 418 =head1 DESCRIPTION
 419
 420 This daemom queries the status of VMs, storages and containers at
 421 regular intervals. The result is sent to all nodes in the cluster.
 422
 423 =include pve_copyright
 424
 425
 426
 427
 428