]>
Commit | Line | Data |
---|---|---|
efd04666 DM |
1 | package PVE::Service::pvestatd; |
2 | ||
3 | use strict; | |
4 | use warnings; | |
5 | ||
6 | use PVE::SafeSyslog; | |
7 | use PVE::Daemon; | |
8 | ||
fea39196 DC |
9 | use JSON; |
10 | ||
efd04666 DM |
11 | use Time::HiRes qw (gettimeofday); |
12 | use PVE::Tools qw(dir_glob_foreach file_read_firstline); | |
13 | use PVE::ProcFSTools; | |
41db757b | 14 | use PVE::CpuSet; |
efd04666 DM |
15 | use Filesys::Df; |
16 | use PVE::INotify; | |
0fcced16 | 17 | use PVE::Network; |
efd04666 DM |
18 | use PVE::Cluster qw(cfs_read_file); |
19 | use PVE::Storage; | |
20 | use PVE::QemuServer; | |
7a108020 | 21 | use PVE::QemuServer::Monitor; |
efd04666 | 22 | use PVE::LXC; |
ce251651 | 23 | use PVE::CGroup; |
41db757b | 24 | use PVE::LXC::Config; |
efd04666 DM |
25 | use PVE::RPCEnvironment; |
26 | use PVE::API2::Subscription; | |
27 | use PVE::AutoBalloon; | |
5ea29d13 | 28 | use PVE::AccessControl; |
fea39196 DC |
29 | use PVE::Ceph::Services; |
30 | use PVE::Ceph::Tools; | |
a6a681b9 | 31 | use PVE::pvecfg; |
efd04666 | 32 | |
f1f4bfef | 33 | use PVE::ExtMetric; |
efd04666 | 34 | use PVE::Status::Plugin; |
efd04666 DM |
35 | |
36 | use base qw(PVE::Daemon); | |
37 | ||
a36565ba AD |
38 | my $have_sdn; |
39 | eval { | |
74058057 | 40 | require PVE::Network::SDN; |
a36565ba AD |
41 | $have_sdn = 1; |
42 | }; | |
43 | ||
efd04666 DM |
44 | my $opt_debug; |
45 | my $restart_request; | |
46 | ||
47 | my $nodename = PVE::INotify::nodename(); | |
48 | ||
49 | my $cmdline = [$0, @ARGV]; | |
50 | ||
51 | my %daemon_options = (restart_on_error => 5, stop_wait_time => 5); | |
52 | my $daemon = __PACKAGE__->new('pvestatd', $cmdline, %daemon_options); | |
53 | ||
54 | sub init { | |
55 | my ($self) = @_; | |
56 | ||
57 | $opt_debug = $self->{debug}; | |
58 | ||
59 | PVE::Cluster::cfs_update(); | |
60 | } | |
61 | ||
62 | sub shutdown { | |
63 | my ($self) = @_; | |
64 | ||
65 | syslog('info' , "server closing"); | |
66 | ||
67 | # wait for children | |
68 | 1 while (waitpid(-1, POSIX::WNOHANG()) > 0); | |
69 | ||
70 | $self->exit_daemon(0); | |
71 | } | |
72 | ||
73 | sub hup { | |
74 | my ($self) = @_; | |
75 | ||
76 | $restart_request = 1; | |
77 | } | |
78 | ||
00b58c8c SR |
79 | my $cached_kvm_version = ''; |
80 | my $next_flag_update_time; | |
81 | my $failed_flag_update_delay_sec = 120; | |
82 | ||
83 | sub update_supported_cpuflags { | |
84 | my $kvm_version = PVE::QemuServer::kvm_user_version(); | |
85 | ||
86 | # only update when QEMU/KVM version has changed, as that is the only reason | |
87 | # why flags could change without restarting pvestatd | |
88 | return if $cached_kvm_version && $cached_kvm_version eq $kvm_version; | |
89 | ||
90 | if ($next_flag_update_time && $next_flag_update_time > time()) { | |
91 | return; | |
92 | } | |
93 | $next_flag_update_time = 0; | |
94 | ||
95 | my $supported_cpuflags = eval { PVE::QemuServer::query_supported_cpu_flags() }; | |
96 | warn $@ if $@; | |
97 | ||
98 | if (!$supported_cpuflags || | |
99 | (!$supported_cpuflags->{tcg} && !$supported_cpuflags->{kvm})) { | |
100 | # something went wrong, clear broadcast flags and set try-again delay | |
101 | warn "CPU flag detection failed, will try again after delay\n"; | |
102 | $next_flag_update_time = time() + $failed_flag_update_delay_sec; | |
103 | ||
104 | $supported_cpuflags = {}; | |
105 | } else { | |
106 | # only set cached version if there's actually something to braodcast | |
107 | $cached_kvm_version = $kvm_version; | |
108 | } | |
109 | ||
110 | for my $accel ("tcg", "kvm") { | |
111 | if ($supported_cpuflags->{$accel}) { | |
112 | PVE::Cluster::broadcast_node_kv("cpuflags-$accel", join(' ', @{$supported_cpuflags->{$accel}})); | |
113 | } else { | |
114 | # clear potentially invalid data | |
115 | PVE::Cluster::broadcast_node_kv("cpuflags-$accel", ''); | |
116 | } | |
117 | } | |
118 | } | |
119 | ||
50786956 DM |
120 | my $generate_rrd_string = sub { |
121 | my ($data) = @_; | |
122 | ||
123 | return join(':', map { $_ // 'U' } @$data); | |
124 | }; | |
125 | ||
955912c5 FE |
126 | my sub broadcast_static_node_info { |
127 | my ($cpus, $memory) = @_; | |
128 | ||
129 | my $old = PVE::Cluster::get_node_kv('static-info', $nodename); | |
130 | $old = eval { decode_json($old->{$nodename}) } if defined($old->{$nodename}); | |
131 | ||
132 | if ( | |
133 | !defined($old->{cpus}) || $old->{cpus} != $cpus | |
134 | || !defined($old->{memory}) || $old->{memory} != $memory | |
135 | ) { | |
136 | my $info = { | |
137 | cpus => $cpus, | |
138 | memory => $memory, | |
139 | }; | |
140 | PVE::Cluster::broadcast_node_kv('static-info', encode_json($info)); | |
141 | } | |
142 | } | |
143 | ||
efd04666 DM |
144 | sub update_node_status { |
145 | my ($status_cfg) = @_; | |
146 | ||
efd04666 DM |
147 | my ($uptime) = PVE::ProcFSTools::read_proc_uptime(); |
148 | ||
78873100 TL |
149 | my ($avg1, $avg5, $avg15) = PVE::ProcFSTools::read_loadavg(); |
150 | my $stat = PVE::ProcFSTools::read_proc_stat(); | |
efd04666 | 151 | my $cpuinfo = PVE::ProcFSTools::read_cpuinfo(); |
78873100 | 152 | my $maxcpu = $cpuinfo->{cpus}; |
efd04666 | 153 | |
00b58c8c SR |
154 | update_supported_cpuflags(); |
155 | ||
d017de1f | 156 | my $subinfo = PVE::API2::Subscription::read_etc_subscription(); |
efd04666 DM |
157 | my $sublevel = $subinfo->{level} || ''; |
158 | ||
78873100 | 159 | my $netdev = PVE::ProcFSTools::read_proc_net_dev(); |
efd04666 | 160 | # traffic from/to physical interface cards |
78873100 TL |
161 | my ($netin, $netout) = (0, 0); |
162 | for my $dev (grep { /^$PVE::Network::PHYSICAL_NIC_RE$/ } keys %$netdev) { | |
efd04666 DM |
163 | $netin += $netdev->{$dev}->{receive}; |
164 | $netout += $netdev->{$dev}->{transmit}; | |
165 | } | |
78873100 | 166 | |
efd04666 DM |
167 | my $meminfo = PVE::ProcFSTools::read_meminfo(); |
168 | ||
169 | my $dinfo = df('/', 1); # output is bytes | |
efd04666 DM |
170 | # everything not free is considered to be used |
171 | my $dused = $dinfo->{blocks} - $dinfo->{bfree}; | |
172 | ||
78873100 TL |
173 | my $ctime = time(); |
174 | ||
50786956 DM |
175 | my $data = $generate_rrd_string->( |
176 | [$uptime, $sublevel, $ctime, $avg1, $maxcpu, $stat->{cpu}, $stat->{wait}, | |
177 | $meminfo->{memtotal}, $meminfo->{memused}, | |
178 | $meminfo->{swaptotal}, $meminfo->{swapused}, | |
78873100 TL |
179 | $dinfo->{blocks}, $dused, $netin, $netout] |
180 | ); | |
efd04666 DM |
181 | PVE::Cluster::broadcast_rrd("pve2-node/$nodename", $data); |
182 | ||
5e82aaac TL |
183 | my $node_metric = { |
184 | uptime => $uptime, | |
185 | cpustat => $stat, | |
186 | memory => $meminfo, | |
187 | blockstat => $dinfo, | |
188 | nics => $netdev, | |
189 | }; | |
190 | $node_metric->{cpustat}->@{qw(avg1 avg5 avg15)} = ($avg1, $avg5, $avg15); | |
191 | $node_metric->{cpustat}->{cpus} = $maxcpu; | |
192 | ||
87be2c19 TL |
193 | my $transactions = PVE::ExtMetric::transactions_start($status_cfg); |
194 | PVE::ExtMetric::update_all($transactions, 'node', $nodename, $node_metric, $ctime); | |
195 | PVE::ExtMetric::transactions_finish($transactions); | |
955912c5 FE |
196 | |
197 | broadcast_static_node_info($maxcpu, $meminfo->{memtotal}); | |
efd04666 DM |
198 | } |
199 | ||
200 | sub auto_balloning { | |
201 | my ($vmstatus) = @_; | |
202 | ||
0dd73a7f | 203 | my $log = sub { $opt_debug and printf @_ }; |
efd04666 DM |
204 | |
205 | my $hostmeminfo = PVE::ProcFSTools::read_meminfo(); | |
0dd73a7f | 206 | # NOTE: to debug, run 'pvestatd -d' and set memtotal here |
efd04666 | 207 | #$hostmeminfo->{memtotal} = int(2*1024*1024*1024/0.8); # you can set this to test |
efd04666 DM |
208 | my $hostfreemem = $hostmeminfo->{memtotal} - $hostmeminfo->{memused}; |
209 | ||
0dd73a7f TL |
210 | # try to use ~80% host memory; goal is the change amount required to achieve that |
211 | my $goal = int($hostmeminfo->{memtotal} * 0.8 - $hostmeminfo->{memused}); | |
212 | $log->("host goal: $goal free: $hostfreemem total: $hostmeminfo->{memtotal}\n"); | |
efd04666 DM |
213 | |
214 | my $maxchange = 100*1024*1024; | |
215 | my $res = PVE::AutoBalloon::compute_alg1($vmstatus, $goal, $maxchange); | |
efd04666 | 216 | |
0dd73a7f TL |
217 | for my $vmid (sort keys %$res) { |
218 | my $target = int($res->{$vmid}); | |
219 | my $current = int($vmstatus->{$vmid}->{balloon}); | |
220 | next if $target == $current; # no need to change | |
221 | ||
222 | $log->("BALLOON $vmid to $target (%d)\n", $target - $current); | |
e2509f4e | 223 | eval { PVE::QemuServer::Monitor::mon_cmd($vmid, "balloon", value => int($target)) }; |
0dd73a7f | 224 | warn $@ if $@; |
efd04666 DM |
225 | } |
226 | } | |
227 | ||
228 | sub update_qemu_status { | |
229 | my ($status_cfg) = @_; | |
230 | ||
231 | my $ctime = time(); | |
efd04666 DM |
232 | my $vmstatus = PVE::QemuServer::vmstatus(undef, 1); |
233 | ||
234 | eval { auto_balloning($vmstatus); }; | |
235 | syslog('err', "auto ballooning error: $@") if $@; | |
236 | ||
87be2c19 | 237 | my $transactions = PVE::ExtMetric::transactions_start($status_cfg); |
efd04666 DM |
238 | foreach my $vmid (keys %$vmstatus) { |
239 | my $d = $vmstatus->{$vmid}; | |
240 | my $data; | |
241 | my $status = $d->{qmpstatus} || $d->{status} || 'stopped'; | |
242 | my $template = $d->{template} ? $d->{template} : "0"; | |
243 | if ($d->{pid}) { # running | |
50786956 DM |
244 | $data = $generate_rrd_string->( |
245 | [$d->{uptime}, $d->{name}, $status, $template, $ctime, $d->{cpus}, $d->{cpu}, | |
246 | $d->{maxmem}, $d->{mem}, $d->{maxdisk}, $d->{disk}, | |
247 | $d->{netin}, $d->{netout}, $d->{diskread}, $d->{diskwrite}]); | |
efd04666 | 248 | } else { |
50786956 DM |
249 | $data = $generate_rrd_string->( |
250 | [0, $d->{name}, $status, $template, $ctime, $d->{cpus}, undef, | |
251 | $d->{maxmem}, undef, $d->{maxdisk}, $d->{disk}, undef, undef, undef, undef]); | |
efd04666 DM |
252 | } |
253 | PVE::Cluster::broadcast_rrd("pve2.3-vm/$vmid", $data); | |
254 | ||
87be2c19 | 255 | PVE::ExtMetric::update_all($transactions, 'qemu', $vmid, $d, $ctime, $nodename); |
efd04666 | 256 | } |
87be2c19 TL |
257 | |
258 | PVE::ExtMetric::transactions_finish($transactions); | |
efd04666 DM |
259 | } |
260 | ||
261 | sub remove_stale_lxc_consoles { | |
262 | ||
263 | my $vmstatus = PVE::LXC::vmstatus(); | |
264 | my $pidhash = PVE::LXC::find_lxc_console_pids(); | |
265 | ||
266 | foreach my $vmid (keys %$pidhash) { | |
267 | next if defined($vmstatus->{$vmid}); | |
268 | syslog('info', "remove stale lxc-console for CT $vmid"); | |
269 | foreach my $pid (@{$pidhash->{$vmid}}) { | |
270 | kill(9, $pid); | |
271 | } | |
272 | } | |
273 | } | |
274 | ||
b3f1adb2 DM |
275 | my $rebalance_error_count = {}; |
276 | ||
eacb5482 | 277 | my $NO_REBALANCE; |
41db757b | 278 | sub rebalance_lxc_containers { |
eacb5482 WB |
279 | # Make sure we can find the cpuset controller path: |
280 | return if $NO_REBALANCE; | |
ce251651 DC |
281 | my $cpuset_base = eval { PVE::CGroup::cpuset_controller_path() }; |
282 | if (my $err = $@) { | |
283 | syslog('info', "could not get cpuset controller path: $err"); | |
284 | } | |
285 | ||
eacb5482 WB |
286 | if (!defined($cpuset_base)) { |
287 | $NO_REBALANCE = 1; | |
288 | return; | |
289 | } | |
41db757b | 290 | |
eacb5482 WB |
291 | # Figure out the cpu count & highest ID |
292 | my $all_cpus = PVE::CpuSet->new_from_path($cpuset_base, 1); | |
41db757b DM |
293 | my @allowed_cpus = $all_cpus->members(); |
294 | my $cpucount = scalar(@allowed_cpus); | |
127470f4 | 295 | my $max_cpuid = $allowed_cpus[-1]; |
41db757b | 296 | |
127470f4 | 297 | my @cpu_ctcount = (0) x ($max_cpuid+1); |
41db757b DM |
298 | my @balanced_cts; |
299 | ||
eacb5482 WB |
300 | # A mapping { vmid => cgroup_payload_path } for containers where namespace |
301 | # separation is active and recognized. | |
302 | my $ctinfo = {}; | |
303 | ||
0b959507 DM |
304 | my $modify_cpuset = sub { |
305 | my ($vmid, $cpuset, $newset) = @_; | |
306 | ||
b3f1adb2 | 307 | if (!$rebalance_error_count->{$vmid}) { |
b707257a | 308 | syslog('info', "modified cpu set for lxc/$vmid: " . $newset->short_string()); |
b3f1adb2 DM |
309 | } |
310 | ||
0b959507 | 311 | eval { |
eacb5482 | 312 | my $cgbase = $ctinfo->{$vmid}; |
cbce367d | 313 | |
eacb5482 | 314 | if (defined($cgbase)) { |
cbce367d | 315 | # allow all, so that we can set new cpuset in /ns |
eacb5482 | 316 | $all_cpus->write_to_path($cgbase); |
cbce367d | 317 | eval { |
eacb5482 | 318 | $newset->write_to_path("$cgbase/ns"); |
cbce367d DM |
319 | }; |
320 | if (my $err = $@) { | |
321 | warn $err if !$rebalance_error_count->{$vmid}++; | |
322 | # restore original | |
eacb5482 | 323 | $cpuset->write_to_path($cgbase); |
cbce367d DM |
324 | } else { |
325 | # also apply to container root cgroup | |
eacb5482 | 326 | $newset->write_to_path($cgbase); |
cbce367d DM |
327 | $rebalance_error_count->{$vmid} = 0; |
328 | } | |
0b959507 | 329 | } else { |
cbce367d | 330 | # old style container |
eacb5482 | 331 | $newset->write_to_path($cgbase); |
b3f1adb2 | 332 | $rebalance_error_count->{$vmid} = 0; |
0b959507 DM |
333 | } |
334 | }; | |
b3f1adb2 DM |
335 | if (my $err = $@) { |
336 | warn $err if !$rebalance_error_count->{$vmid}++; | |
337 | } | |
0b959507 DM |
338 | }; |
339 | ||
e0dc09ad DM |
340 | my $ctlist = PVE::LXC::config_list(); |
341 | ||
342 | foreach my $vmid (sort keys %$ctlist) { | |
eacb5482 | 343 | my $cgpath = "$cpuset_base/lxc/$vmid"; |
eacb5482 WB |
344 | if (-d "$cgpath/ns") { |
345 | $ctinfo->{$vmid} = $cgpath; | |
346 | } else { | |
b707257a | 347 | next; # old style container |
eacb5482 | 348 | } |
41db757b | 349 | |
b707257a TL |
350 | my ($conf, $cpuset) = eval {( |
351 | PVE::LXC::Config->load_config($vmid), | |
352 | PVE::CpuSet->new_from_path($cgpath), | |
353 | )}; | |
41db757b DM |
354 | if (my $err = $@) { |
355 | warn $err; | |
356 | next; | |
357 | } | |
358 | ||
359 | my @cpuset_members = $cpuset->members(); | |
360 | ||
911a8e4f OB |
361 | if (!PVE::LXC::Config->has_lxc_entry($conf, 'lxc.cgroup.cpuset.cpus') |
362 | && !PVE::LXC::Config->has_lxc_entry($conf, 'lxc.cgroup2.cpuset.cpus') | |
363 | ) { | |
8b750abc DM |
364 | my $cores = $conf->{cores} || $cpucount; |
365 | $cores = $cpucount if $cores > $cpucount; | |
41db757b | 366 | |
b707257a | 367 | # see if the number of cores was hot-reduced or hasn't been enacted at all yet |
2499255b DM |
368 | my $newset = PVE::CpuSet->new(); |
369 | if ($cores < scalar(@cpuset_members)) { | |
370 | for (my $i = 0; $i < $cores; $i++) { | |
371 | $newset->insert($cpuset_members[$i]); | |
372 | } | |
373 | } elsif ($cores > scalar(@cpuset_members)) { | |
374 | my $count = $newset->insert(@cpuset_members); | |
375 | foreach my $cpu (@allowed_cpus) { | |
376 | $count += $newset->insert($cpu); | |
377 | last if $count >= $cores; | |
378 | } | |
379 | } else { | |
380 | $newset->insert(@cpuset_members); | |
381 | } | |
07f9595f | 382 | |
2499255b DM |
383 | # Apply hot-plugged changes if any: |
384 | if (!$newset->is_equal($cpuset)) { | |
385 | @cpuset_members = $newset->members(); | |
0b959507 | 386 | $modify_cpuset->($vmid, $cpuset, $newset); |
2499255b | 387 | } |
07f9595f | 388 | |
2499255b DM |
389 | # Note: no need to rebalance if we already use all cores |
390 | push @balanced_cts, [$vmid, $cores, $newset] | |
8b750abc | 391 | if defined($conf->{cores}) && ($cores != $cpucount); |
2499255b | 392 | } |
07f9595f | 393 | |
2499255b | 394 | foreach my $cpu (@cpuset_members) { |
ccfff920 | 395 | $cpu_ctcount[$cpu]++ if $cpu <= $max_cpuid; |
07f9595f | 396 | } |
2499255b | 397 | } |
07f9595f | 398 | |
2499255b DM |
399 | my $find_best_cpu = sub { |
400 | my ($cpulist, $cpu) = @_; | |
07f9595f | 401 | |
2499255b DM |
402 | my $cur_cost = $cpu_ctcount[$cpu]; |
403 | my $cur_cpu = $cpu; | |
41db757b | 404 | |
2499255b DM |
405 | foreach my $candidate (@$cpulist) { |
406 | my $cost = $cpu_ctcount[$candidate]; | |
e8c41dc5 | 407 | if ($cost < ($cur_cost - 1)) { |
2499255b DM |
408 | $cur_cost = $cost; |
409 | $cur_cpu = $candidate; | |
410 | } | |
07f9595f DM |
411 | } |
412 | ||
2499255b DM |
413 | return $cur_cpu; |
414 | }; | |
415 | ||
416 | foreach my $bct (@balanced_cts) { | |
417 | my ($vmid, $cores, $cpuset) = @$bct; | |
41db757b | 418 | |
e8c41dc5 | 419 | my $rest = [ grep { !$cpuset->has($_) } @allowed_cpus ]; |
2499255b | 420 | |
e8c41dc5 TL |
421 | my $newset = PVE::CpuSet->new(); |
422 | for my $cpu ($cpuset->members()) { | |
423 | my $best = $find_best_cpu->($rest, $cpu); | |
2499255b DM |
424 | if ($best != $cpu) { |
425 | $cpu_ctcount[$best]++; | |
426 | $cpu_ctcount[$cpu]--; | |
427 | } | |
428 | $newset->insert($best); | |
41db757b DM |
429 | } |
430 | ||
431 | if (!$newset->is_equal($cpuset)) { | |
0b959507 | 432 | $modify_cpuset->($vmid, $cpuset, $newset); |
41db757b DM |
433 | } |
434 | } | |
435 | } | |
436 | ||
efd04666 DM |
437 | sub update_lxc_status { |
438 | my ($status_cfg) = @_; | |
439 | ||
440 | my $ctime = time(); | |
efd04666 DM |
441 | my $vmstatus = PVE::LXC::vmstatus(); |
442 | ||
87be2c19 TL |
443 | my $transactions = PVE::ExtMetric::transactions_start($status_cfg); |
444 | ||
efd04666 DM |
445 | foreach my $vmid (keys %$vmstatus) { |
446 | my $d = $vmstatus->{$vmid}; | |
447 | my $template = $d->{template} ? $d->{template} : "0"; | |
448 | my $data; | |
449 | if ($d->{status} eq 'running') { # running | |
50786956 DM |
450 | $data = $generate_rrd_string->( |
451 | [$d->{uptime}, $d->{name}, $d->{status}, $template, | |
452 | $ctime, $d->{cpus}, $d->{cpu}, | |
453 | $d->{maxmem}, $d->{mem}, | |
454 | $d->{maxdisk}, $d->{disk}, | |
455 | $d->{netin}, $d->{netout}, | |
456 | $d->{diskread}, $d->{diskwrite}]); | |
efd04666 | 457 | } else { |
50786956 DM |
458 | $data = $generate_rrd_string->( |
459 | [0, $d->{name}, $d->{status}, $template, $ctime, $d->{cpus}, undef, | |
460 | $d->{maxmem}, undef, $d->{maxdisk}, $d->{disk}, undef, undef, undef, undef]); | |
efd04666 DM |
461 | } |
462 | PVE::Cluster::broadcast_rrd("pve2.3-vm/$vmid", $data); | |
463 | ||
87be2c19 | 464 | PVE::ExtMetric::update_all($transactions, 'lxc', $vmid, $d, $ctime, $nodename); |
efd04666 | 465 | } |
87be2c19 | 466 | PVE::ExtMetric::transactions_finish($transactions); |
efd04666 DM |
467 | } |
468 | ||
469 | sub update_storage_status { | |
470 | my ($status_cfg) = @_; | |
471 | ||
bbcfdc08 | 472 | my $cfg = PVE::Storage::config(); |
efd04666 | 473 | my $ctime = time(); |
efd04666 DM |
474 | my $info = PVE::Storage::storage_info($cfg); |
475 | ||
87be2c19 TL |
476 | my $transactions = PVE::ExtMetric::transactions_start($status_cfg); |
477 | ||
efd04666 DM |
478 | foreach my $storeid (keys %$info) { |
479 | my $d = $info->{$storeid}; | |
480 | next if !$d->{active}; | |
481 | ||
50786956 | 482 | my $data = $generate_rrd_string->([$ctime, $d->{total}, $d->{used}]); |
efd04666 DM |
483 | |
484 | my $key = "pve2-storage/${nodename}/$storeid"; | |
485 | PVE::Cluster::broadcast_rrd($key, $data); | |
486 | ||
87be2c19 | 487 | PVE::ExtMetric::update_all($transactions, 'storage', $nodename, $storeid, $d, $ctime); |
efd04666 | 488 | } |
87be2c19 | 489 | PVE::ExtMetric::transactions_finish($transactions); |
efd04666 DM |
490 | } |
491 | ||
5ea29d13 FG |
492 | sub rotate_authkeys { |
493 | PVE::AccessControl::rotate_authkey() if !PVE::AccessControl::check_authkey(1); | |
494 | } | |
495 | ||
a6dff455 TL |
496 | sub update_ceph_metadata { |
497 | return if !PVE::Ceph::Tools::check_ceph_inited(1); # nothing to do | |
498 | ||
499 | PVE::Ceph::Services::broadcast_ceph_services(); | |
a78fd21f | 500 | |
0496138e | 501 | PVE::Ceph::Services::broadcast_ceph_versions(); |
fea39196 DC |
502 | } |
503 | ||
a36565ba AD |
504 | sub update_sdn_status { |
505 | ||
506 | if($have_sdn) { | |
507 | my ($transport_status, $vnet_status) = PVE::Network::SDN::status(); | |
508 | ||
509 | my $status = $transport_status ? encode_json($transport_status) : undef; | |
510 | PVE::Cluster::broadcast_node_kv("sdn", $status); | |
511 | } | |
512 | } | |
513 | ||
a6a681b9 WB |
514 | my $broadcast_version_info_done = 0; |
515 | my sub broadcast_version_info : prototype() { | |
516 | if (!$broadcast_version_info_done) { | |
517 | PVE::Cluster::broadcast_node_kv( | |
518 | 'version-info', | |
519 | encode_json(PVE::pvecfg::version_info()), | |
520 | ); | |
521 | $broadcast_version_info_done = 1; | |
522 | } | |
523 | } | |
524 | ||
efd04666 | 525 | sub update_status { |
7658f010 TL |
526 | # update worker list. This is not really required, but we want to make sure that we also have a |
527 | # correct list in case of an unexpected crash. | |
8a9bf777 DM |
528 | my $rpcenv = PVE::RPCEnvironment::get(); |
529 | ||
efd04666 | 530 | eval { |
8a9bf777 | 531 | my $tlist = $rpcenv->active_workers(); |
efd04666 DM |
532 | PVE::Cluster::broadcast_tasklist($tlist); |
533 | }; | |
534 | my $err = $@; | |
535 | syslog('err', $err) if $err; | |
536 | ||
537 | my $status_cfg = PVE::Cluster::cfs_read_file('status.cfg'); | |
538 | ||
539 | eval { | |
540 | update_node_status($status_cfg); | |
541 | }; | |
542 | $err = $@; | |
543 | syslog('err', "node status update error: $err") if $err; | |
544 | ||
545 | eval { | |
546 | update_qemu_status($status_cfg); | |
547 | }; | |
548 | $err = $@; | |
549 | syslog('err', "qemu status update error: $err") if $err; | |
550 | ||
551 | eval { | |
552 | update_lxc_status($status_cfg); | |
553 | }; | |
554 | $err = $@; | |
555 | syslog('err', "lxc status update error: $err") if $err; | |
556 | ||
e0dc09ad DM |
557 | eval { |
558 | rebalance_lxc_containers(); | |
559 | }; | |
560 | $err = $@; | |
561 | syslog('err', "lxc cpuset rebalance error: $err") if $err; | |
562 | ||
efd04666 DM |
563 | eval { |
564 | update_storage_status($status_cfg); | |
565 | }; | |
566 | $err = $@; | |
567 | syslog('err', "storage status update error: $err") if $err; | |
568 | ||
569 | eval { | |
570 | remove_stale_lxc_consoles(); | |
571 | }; | |
572 | $err = $@; | |
573 | syslog('err', "lxc console cleanup error: $err") if $err; | |
5ea29d13 FG |
574 | |
575 | eval { | |
576 | rotate_authkeys(); | |
577 | }; | |
578 | $err = $@; | |
579 | syslog('err', "authkey rotation error: $err") if $err; | |
580 | ||
fea39196 | 581 | eval { |
a6dff455 | 582 | update_ceph_metadata(); |
fea39196 DC |
583 | }; |
584 | $err = $@; | |
2a8e5149 | 585 | syslog('err', "ceph metadata update error: $err") if $err; |
fea39196 | 586 | |
a36565ba AD |
587 | eval { |
588 | update_sdn_status(); | |
589 | }; | |
590 | $err = $@; | |
591 | syslog('err', "sdn status update error: $err") if $err; | |
592 | ||
a6a681b9 WB |
593 | eval { |
594 | broadcast_version_info(); | |
595 | }; | |
596 | $err = $@; | |
597 | syslog('err', "version info update error: $err") if $err; | |
efd04666 DM |
598 | } |
599 | ||
600 | my $next_update = 0; | |
601 | ||
602 | # do not update directly after startup, because install scripts | |
603 | # have a problem with that | |
604 | my $cycle = 0; | |
605 | my $updatetime = 10; | |
606 | ||
607 | my $initial_memory_usage; | |
608 | ||
609 | sub run { | |
610 | my ($self) = @_; | |
611 | ||
612 | for (;;) { # forever | |
613 | ||
614 | $next_update = time() + $updatetime; | |
615 | ||
616 | if ($cycle) { | |
617 | my ($ccsec, $cusec) = gettimeofday (); | |
618 | eval { | |
619 | # syslog('info', "start status update"); | |
620 | PVE::Cluster::cfs_update(); | |
621 | update_status(); | |
622 | }; | |
623 | my $err = $@; | |
624 | ||
625 | if ($err) { | |
626 | syslog('err', "status update error: $err"); | |
627 | } | |
628 | ||
629 | my ($ccsec_end, $cusec_end) = gettimeofday (); | |
630 | my $cptime = ($ccsec_end-$ccsec) + ($cusec_end - $cusec)/1000000; | |
631 | ||
632 | syslog('info', sprintf("status update time (%.3f seconds)", $cptime)) | |
633 | if ($cptime > 5); | |
634 | } | |
635 | ||
636 | $cycle++; | |
637 | ||
638 | my $mem = PVE::ProcFSTools::read_memory_usage(); | |
cc3d280b | 639 | my $resident_kb = $mem->{resident} / 1024; |
efd04666 DM |
640 | |
641 | if (!defined($initial_memory_usage) || ($cycle < 10)) { | |
cc3d280b | 642 | $initial_memory_usage = $resident_kb; |
efd04666 | 643 | } else { |
cc3d280b | 644 | my $diff = $resident_kb - $initial_memory_usage; |
2112d310 | 645 | if ($diff > 15 * 1024) { |
efd04666 | 646 | syslog ('info', "restarting server after $cycle cycles to " . |
cc3d280b | 647 | "reduce memory usage (free $resident_kb ($diff) KB)"); |
efd04666 DM |
648 | $self->restart_daemon(); |
649 | } | |
650 | } | |
651 | ||
652 | my $wcount = 0; | |
653 | while ((time() < $next_update) && | |
654 | ($wcount < $updatetime) && # protect against time wrap | |
655 | !$restart_request) { $wcount++; sleep (1); }; | |
656 | ||
657 | $self->restart_daemon() if $restart_request; | |
658 | } | |
659 | } | |
660 | ||
661 | $daemon->register_start_command(); | |
662 | $daemon->register_restart_command(1); | |
663 | $daemon->register_stop_command(); | |
664 | $daemon->register_status_command(); | |
665 | ||
666 | our $cmddef = { | |
667 | start => [ __PACKAGE__, 'start', []], | |
668 | restart => [ __PACKAGE__, 'restart', []], | |
669 | stop => [ __PACKAGE__, 'stop', []], | |
670 | status => [ __PACKAGE__, 'status', [], undef, sub { print shift . "\n";} ], | |
671 | }; | |
672 | ||
efd04666 DM |
673 | 1; |
674 | ||
efd04666 DM |
675 | |
676 | ||
677 | ||
678 |