]>
Commit | Line | Data |
---|---|---|
776de3bc | 1 | #!/usr/bin/perl |
aff192e6 DM |
2 | |
3 | use strict; | |
776de3bc | 4 | use warnings; |
aff192e6 | 5 | use PVE::SafeSyslog; |
891d1242 DM |
6 | use PVE::Daemon; |
7 | ||
aff192e6 | 8 | use Time::HiRes qw (gettimeofday); |
f9d4fc64 | 9 | use PVE::Tools qw(dir_glob_foreach file_read_firstline); |
aff192e6 DM |
10 | use PVE::ProcFSTools; |
11 | use Filesys::Df; | |
12 | use PVE::INotify; | |
13 | use PVE::Cluster qw(cfs_read_file); | |
14 | use PVE::Storage; | |
15 | use PVE::QemuServer; | |
16 | use PVE::RPCEnvironment; | |
16b69b6c | 17 | use PVE::API2::Subscription; |
53f13052 | 18 | use PVE::AutoBalloon; |
aff192e6 | 19 | |
891d1242 | 20 | use base qw(PVE::Daemon); |
aff192e6 DM |
21 | |
22 | my $opt_debug; | |
23 | ||
891d1242 | 24 | my $cmdline = [$0, @ARGV]; |
aff192e6 | 25 | |
891d1242 | 26 | my %daemon_options = (restart_on_error => 5, stop_wait_time => 5); |
aff192e6 | 27 | |
891d1242 | 28 | my $daemon = __PACKAGE__->new('pvestatd', $cmdline, %daemon_options); |
aff192e6 | 29 | |
891d1242 | 30 | my $rpcenv = PVE::RPCEnvironment->init('cli'); |
aff192e6 | 31 | |
891d1242 DM |
32 | $rpcenv->init_request(); |
33 | $rpcenv->set_language($ENV{LANG}); | |
34 | $rpcenv->set_user('root@pam'); | |
aff192e6 | 35 | |
891d1242 DM |
36 | my $nodename = PVE::INotify::nodename(); |
37 | my $restart_request = 0; | |
aff192e6 | 38 | |
891d1242 DM |
39 | sub init { |
40 | my ($self) = @_; | |
aff192e6 | 41 | |
6e12f8db DM |
42 | $opt_debug = $self->{debug}; |
43 | ||
891d1242 | 44 | PVE::Cluster::cfs_update(); |
aff192e6 DM |
45 | } |
46 | ||
891d1242 DM |
47 | sub shutdown { |
48 | my ($self) = @_; | |
aff192e6 | 49 | |
aff192e6 DM |
50 | syslog('info' , "server closing"); |
51 | ||
aff192e6 DM |
52 | # wait for children |
53 | 1 while (waitpid(-1, POSIX::WNOHANG()) > 0); | |
54 | ||
891d1242 DM |
55 | $self->exit_daemon(0); |
56 | } | |
aff192e6 | 57 | |
891d1242 DM |
58 | sub hup { |
59 | my ($self) = @_; | |
aff192e6 | 60 | |
891d1242 | 61 | $restart_request = 1; |
aff192e6 DM |
62 | } |
63 | ||
aff192e6 DM |
64 | sub update_node_status { |
65 | ||
66 | my ($avg1, $avg5, $avg15) = PVE::ProcFSTools::read_loadavg(); | |
67 | ||
68 | my $stat = PVE::ProcFSTools::read_proc_stat(); | |
69 | ||
70 | my $netdev = PVE::ProcFSTools::read_proc_net_dev(); | |
71 | ||
72 | my ($uptime) = PVE::ProcFSTools::read_proc_uptime(); | |
73 | ||
74 | my $cpuinfo = PVE::ProcFSTools::read_cpuinfo(); | |
75 | ||
76 | my $maxcpu = $cpuinfo->{cpus}; | |
77 | ||
16b69b6c DM |
78 | my $subinfo = PVE::INotify::read_file('subscription'); |
79 | my $sublevel = $subinfo->{level} || ''; | |
80 | ||
aff192e6 DM |
81 | # traffic from/to physical interface cards |
82 | my $netin = 0; | |
83 | my $netout = 0; | |
84 | foreach my $dev (keys %$netdev) { | |
85 | next if $dev !~ m/^eth\d+$/; | |
86 | $netin += $netdev->{$dev}->{receive}; | |
87 | $netout += $netdev->{$dev}->{transmit}; | |
88 | } | |
89 | ||
90 | my $meminfo = PVE::ProcFSTools::read_meminfo(); | |
91 | ||
92 | my $dinfo = df('/', 1); # output is bytes | |
93 | ||
94 | my $ctime = time(); | |
95 | ||
96 | # everything not free is considered to be used | |
97 | my $dused = $dinfo->{blocks} - $dinfo->{bfree}; | |
98 | ||
16b69b6c | 99 | my $data = "$uptime:$sublevel:$ctime:$avg1:$maxcpu:$stat->{cpu}:$stat->{wait}:" . |
aff192e6 DM |
100 | "$meminfo->{memtotal}:$meminfo->{memused}:" . |
101 | "$meminfo->{swaptotal}:$meminfo->{swapused}:" . | |
102 | "$dinfo->{blocks}:$dused:$netin:$netout"; | |
103 | ||
104 | PVE::Cluster::broadcast_rrd("pve2-node/$nodename", $data); | |
105 | } | |
106 | ||
0883a378 DM |
107 | sub auto_balloning { |
108 | my ($vmstatus) = @_; | |
109 | ||
110 | my $log = sub { | |
53f13052 DM |
111 | return if !$opt_debug; |
112 | print @_; | |
0883a378 | 113 | }; |
53f13052 | 114 | |
0883a378 DM |
115 | my $hostmeminfo = PVE::ProcFSTools::read_meminfo(); |
116 | ||
117 | # to debug, run 'pvestatd -d' and set memtotal here | |
53f13052 | 118 | #$hostmeminfo->{memtotal} = int(2*1024*1024*1024/0.8); # you can set this to test |
0883a378 DM |
119 | |
120 | my $hostfreemem = $hostmeminfo->{memtotal} - $hostmeminfo->{memused}; | |
121 | ||
122 | # we try to use about 80% host memory | |
123 | # goal: we want to change memory usage by this amount (positive or negative) | |
124 | my $goal = int($hostmeminfo->{memtotal}*0.8 - $hostmeminfo->{memused}); | |
125 | ||
0883a378 | 126 | my $maxchange = 100*1024*1024; |
53f13052 | 127 | my $res = PVE::AutoBalloon::compute_alg1($vmstatus, $goal, $maxchange); |
0883a378 | 128 | |
53f13052 | 129 | &$log("host goal: $goal free: $hostfreemem total: $hostmeminfo->{memtotal}\n"); |
0883a378 | 130 | |
53f13052 | 131 | foreach my $vmid (keys %$vmstatus) { |
0883a378 DM |
132 | next if !$res->{$vmid}; |
133 | my $d = $vmstatus->{$vmid}; | |
134 | my $diff = int($res->{$vmid} - $d->{balloon}); | |
135 | my $absdiff = $diff < 0 ? -$diff : $diff; | |
136 | if ($absdiff > 0) { | |
137 | &$log("BALLOON $vmid to $res->{$vmid} ($diff)\n"); | |
138 | eval { | |
139 | PVE::QemuServer::vm_mon_cmd($vmid, "balloon", | |
140 | value => int($res->{$vmid})); | |
141 | }; | |
142 | warn $@ if $@; | |
143 | } | |
144 | } | |
145 | } | |
146 | ||
aff192e6 DM |
147 | sub update_qemu_status { |
148 | ||
149 | my $ctime = time(); | |
150 | ||
cbb20c6e | 151 | my $vmstatus = PVE::QemuServer::vmstatus(undef, 1); |
aff192e6 | 152 | |
0883a378 DM |
153 | eval { auto_balloning($vmstatus); }; |
154 | syslog('err', "auto ballooning error: $@") if $@; | |
155 | ||
aff192e6 DM |
156 | foreach my $vmid (keys %$vmstatus) { |
157 | my $d = $vmstatus->{$vmid}; | |
158 | my $data; | |
3c353ed2 | 159 | my $status = $d->{qmpstatus} || $d->{status} || 'stopped'; |
68cd38c2 | 160 | my $template = $d->{template} ? $d->{template} : "0"; |
aff192e6 | 161 | if ($d->{pid}) { # running |
3c353ed2 DM |
162 | $data = "$d->{uptime}:$d->{name}:$status:$template:" . |
163 | "$ctime:$d->{cpus}:$d->{cpu}:" . | |
aff192e6 DM |
164 | "$d->{maxmem}:$d->{mem}:" . |
165 | "$d->{maxdisk}:$d->{disk}:" . | |
166 | "$d->{netin}:$d->{netout}:" . | |
167 | "$d->{diskread}:$d->{diskwrite}"; | |
168 | } else { | |
3c353ed2 | 169 | $data = "0:$d->{name}:$status:$template:$ctime:$d->{cpus}::" . |
aff192e6 DM |
170 | "$d->{maxmem}::" . |
171 | "$d->{maxdisk}:$d->{disk}:" . | |
172 | ":::"; | |
173 | } | |
3c353ed2 | 174 | PVE::Cluster::broadcast_rrd("pve2.3-vm/$vmid", $data); |
aff192e6 DM |
175 | } |
176 | } | |
177 | ||
f9d4fc64 DM |
178 | sub find_vzctl_console_pids { |
179 | ||
180 | my $res = {}; | |
181 | ||
182 | dir_glob_foreach('/proc', '\d+', sub { | |
183 | my ($pid) = @_; | |
184 | ||
185 | my $cmdline = file_read_firstline("/proc/$pid/cmdline"); | |
186 | return if !$cmdline; | |
187 | ||
188 | my @args = split(/\0/, $cmdline); | |
189 | ||
190 | # serach for vzctl console <vmid> | |
191 | return if scalar(@args) != 3; | |
192 | return if $args[1] ne 'console'; | |
193 | return if $args[2] !~ m/^\d+$/; | |
194 | return if $args[0] !~ m|^(/usr/sbin/)?vzctl$|; | |
195 | ||
196 | my $vmid = $args[2]; | |
197 | ||
198 | push @{$res->{$vmid}}, $pid; | |
199 | }); | |
200 | ||
201 | return $res; | |
202 | } | |
891d1242 | 203 | |
aff192e6 DM |
204 | sub update_storage_status { |
205 | ||
206 | my $cfg = cfs_read_file("storage.cfg"); | |
207 | ||
208 | my $ctime = time(); | |
209 | ||
210 | my $info = PVE::Storage::storage_info($cfg); | |
211 | ||
212 | foreach my $storeid (keys %$info) { | |
213 | my $d = $info->{$storeid}; | |
214 | next if !$d->{active}; | |
215 | ||
216 | # everything not free is considered to be used | |
217 | my $realused = $d->{total} - $d->{avail}; | |
218 | ||
219 | my $data = "$ctime:$d->{total}:$realused"; | |
220 | ||
221 | my $key = "pve2-storage/${nodename}/$storeid"; | |
222 | PVE::Cluster::broadcast_rrd($key, $data); | |
223 | } | |
224 | } | |
225 | ||
226 | sub update_status { | |
227 | ||
228 | # update worker list. This is not really required and | |
229 | # we just call this to make sure that we have a correct | |
230 | # list in case of an unexpected crash. | |
231 | eval { | |
232 | my $tlist = PVE::RPCEnvironment::active_workers(); | |
233 | PVE::Cluster::broadcast_tasklist($tlist); | |
234 | }; | |
235 | my $err = $@; | |
236 | syslog('err', $err) if $err; | |
237 | ||
238 | eval { | |
239 | update_node_status(); | |
240 | }; | |
241 | $err = $@; | |
242 | syslog('err', "node status update error: $err") if $err; | |
243 | ||
244 | eval { | |
245 | update_qemu_status(); | |
246 | }; | |
247 | $err = $@; | |
248 | syslog('err', "qemu status update error: $err") if $err; | |
249 | ||
250 | eval { | |
251 | update_storage_status(); | |
252 | }; | |
253 | $err = $@; | |
254 | syslog('err', "storage status update error: $err") if $err; | |
255 | } | |
256 | ||
257 | my $next_update = 0; | |
258 | ||
259 | # do not update directly after startup, because install scripts | |
260 | # have a problem with that | |
261 | my $cycle = 0; | |
262 | my $updatetime = 10; | |
263 | ||
350b3b46 DM |
264 | my $initial_memory_usage; |
265 | ||
891d1242 DM |
266 | sub run { |
267 | my ($self) = @_; | |
aff192e6 | 268 | |
891d1242 DM |
269 | for (;;) { # forever |
270 | ||
271 | $next_update = time() + $updatetime; | |
aff192e6 DM |
272 | |
273 | if ($cycle) { | |
274 | my ($ccsec, $cusec) = gettimeofday (); | |
275 | eval { | |
9b0aba10 | 276 | # syslog('info', "start status update"); |
aff192e6 DM |
277 | PVE::Cluster::cfs_update(); |
278 | update_status(); | |
279 | }; | |
280 | my $err = $@; | |
281 | ||
282 | if ($err) { | |
283 | syslog('err', "status update error: $err"); | |
284 | } | |
285 | ||
286 | my ($ccsec_end, $cusec_end) = gettimeofday (); | |
287 | my $cptime = ($ccsec_end-$ccsec) + ($cusec_end - $cusec)/1000000; | |
288 | ||
9b0aba10 DM |
289 | syslog('info', sprintf("status update time (%.3f seconds)", $cptime)) |
290 | if ($cptime > 5); | |
aff192e6 DM |
291 | } |
292 | ||
293 | $cycle++; | |
294 | ||
295 | my $mem = PVE::ProcFSTools::read_memory_usage(); | |
296 | ||
9dbdda49 | 297 | if (!defined($initial_memory_usage) || ($cycle < 10)) { |
350b3b46 DM |
298 | $initial_memory_usage = $mem->{resident}; |
299 | } else { | |
300 | my $diff = $mem->{resident} - $initial_memory_usage; | |
301 | if ($diff > 5*1024*1024) { | |
302 | syslog ('info', "restarting server after $cycle cycles to " . | |
303 | "reduce memory usage (free $mem->{resident} ($diff) bytes)"); | |
891d1242 | 304 | $self->restart_daemon(); |
350b3b46 | 305 | } |
aff192e6 DM |
306 | } |
307 | ||
308 | my $wcount = 0; | |
309 | while ((time() < $next_update) && | |
310 | ($wcount < $updatetime) && # protect against time wrap | |
891d1242 | 311 | !$restart_request) { $wcount++; sleep (1); }; |
aff192e6 | 312 | |
891d1242 | 313 | $self->restart_daemon() if $restart_request; |
aff192e6 DM |
314 | } |
315 | } | |
316 | ||
4209128e DM |
317 | $daemon->register_start_command(); |
318 | $daemon->register_restart_command(1); | |
319 | $daemon->register_stop_command(); | |
320 | $daemon->register_status_command(); | |
891d1242 DM |
321 | |
322 | my $cmddef = { | |
323 | start => [ __PACKAGE__, 'start', []], | |
324 | restart => [ __PACKAGE__, 'restart', []], | |
325 | stop => [ __PACKAGE__, 'stop', []], | |
326 | status => [ __PACKAGE__, 'status', [], undef, sub { print shift . "\n";} ], | |
327 | }; | |
328 | ||
329 | my $cmd = shift; | |
330 | ||
331 | PVE::CLIHandler::handle_cmd($cmddef, $0, $cmd, \@ARGV, undef, $0); | |
332 | ||
aff192e6 DM |
333 | exit (0); |
334 | ||
335 | __END__ | |
336 | ||
337 | =head1 NAME | |
338 | ||
339 | pvestatd - PVE Status Daemon | |
340 | ||
341 | =head1 SYNOPSIS | |
342 | ||
891d1242 | 343 | =include synopsis |
aff192e6 DM |
344 | |
345 | =head1 DESCRIPTION | |
346 | ||
891d1242 DM |
347 | This daemom queries the status of VMs, storages and containers at |
348 | regular intervals. The result is sent to all nodes in the cluster. | |
349 | ||
350 | =include pve_copyright | |
aff192e6 DM |
351 | |
352 | ||
353 | ||
354 | ||
355 |