]>
Commit | Line | Data |
---|---|---|
efd04666 DM |
1 | package PVE::Service::pvestatd; |
2 | ||
3 | use strict; | |
4 | use warnings; | |
5 | ||
6 | use PVE::SafeSyslog; | |
7 | use PVE::Daemon; | |
8 | ||
fea39196 DC |
9 | use JSON; |
10 | ||
efd04666 DM |
11 | use Time::HiRes qw (gettimeofday); |
12 | use PVE::Tools qw(dir_glob_foreach file_read_firstline); | |
13 | use PVE::ProcFSTools; | |
41db757b | 14 | use PVE::CpuSet; |
efd04666 DM |
15 | use Filesys::Df; |
16 | use PVE::INotify; | |
0fcced16 | 17 | use PVE::Network; |
efd04666 DM |
18 | use PVE::Cluster qw(cfs_read_file); |
19 | use PVE::Storage; | |
20 | use PVE::QemuServer; | |
7a108020 | 21 | use PVE::QemuServer::Monitor; |
efd04666 | 22 | use PVE::LXC; |
41db757b | 23 | use PVE::LXC::Config; |
efd04666 DM |
24 | use PVE::RPCEnvironment; |
25 | use PVE::API2::Subscription; | |
26 | use PVE::AutoBalloon; | |
5ea29d13 | 27 | use PVE::AccessControl; |
fea39196 DC |
28 | use PVE::Ceph::Services; |
29 | use PVE::Ceph::Tools; | |
efd04666 | 30 | |
f1f4bfef | 31 | use PVE::ExtMetric; |
efd04666 | 32 | use PVE::Status::Plugin; |
efd04666 DM |
33 | |
34 | use base qw(PVE::Daemon); | |
35 | ||
a36565ba AD |
36 | my $have_sdn; |
37 | eval { | |
38 | require PVE::API2::Network::SDN; | |
39 | $have_sdn = 1; | |
40 | }; | |
41 | ||
efd04666 DM |
42 | my $opt_debug; |
43 | my $restart_request; | |
44 | ||
45 | my $nodename = PVE::INotify::nodename(); | |
46 | ||
47 | my $cmdline = [$0, @ARGV]; | |
48 | ||
49 | my %daemon_options = (restart_on_error => 5, stop_wait_time => 5); | |
50 | my $daemon = __PACKAGE__->new('pvestatd', $cmdline, %daemon_options); | |
51 | ||
52 | sub init { | |
53 | my ($self) = @_; | |
54 | ||
55 | $opt_debug = $self->{debug}; | |
56 | ||
57 | PVE::Cluster::cfs_update(); | |
58 | } | |
59 | ||
60 | sub shutdown { | |
61 | my ($self) = @_; | |
62 | ||
63 | syslog('info' , "server closing"); | |
64 | ||
65 | # wait for children | |
66 | 1 while (waitpid(-1, POSIX::WNOHANG()) > 0); | |
67 | ||
68 | $self->exit_daemon(0); | |
69 | } | |
70 | ||
71 | sub hup { | |
72 | my ($self) = @_; | |
73 | ||
74 | $restart_request = 1; | |
75 | } | |
76 | ||
50786956 DM |
77 | my $generate_rrd_string = sub { |
78 | my ($data) = @_; | |
79 | ||
80 | return join(':', map { $_ // 'U' } @$data); | |
81 | }; | |
82 | ||
efd04666 DM |
83 | sub update_node_status { |
84 | my ($status_cfg) = @_; | |
85 | ||
efd04666 DM |
86 | my ($uptime) = PVE::ProcFSTools::read_proc_uptime(); |
87 | ||
78873100 TL |
88 | my ($avg1, $avg5, $avg15) = PVE::ProcFSTools::read_loadavg(); |
89 | my $stat = PVE::ProcFSTools::read_proc_stat(); | |
efd04666 | 90 | my $cpuinfo = PVE::ProcFSTools::read_cpuinfo(); |
78873100 | 91 | my $maxcpu = $cpuinfo->{cpus}; |
efd04666 DM |
92 | |
93 | my $subinfo = PVE::INotify::read_file('subscription'); | |
94 | my $sublevel = $subinfo->{level} || ''; | |
95 | ||
78873100 | 96 | my $netdev = PVE::ProcFSTools::read_proc_net_dev(); |
efd04666 | 97 | # traffic from/to physical interface cards |
78873100 TL |
98 | my ($netin, $netout) = (0, 0); |
99 | for my $dev (grep { /^$PVE::Network::PHYSICAL_NIC_RE$/ } keys %$netdev) { | |
efd04666 DM |
100 | $netin += $netdev->{$dev}->{receive}; |
101 | $netout += $netdev->{$dev}->{transmit}; | |
102 | } | |
78873100 | 103 | |
efd04666 DM |
104 | my $meminfo = PVE::ProcFSTools::read_meminfo(); |
105 | ||
106 | my $dinfo = df('/', 1); # output is bytes | |
efd04666 DM |
107 | # everything not free is considered to be used |
108 | my $dused = $dinfo->{blocks} - $dinfo->{bfree}; | |
109 | ||
78873100 TL |
110 | my $ctime = time(); |
111 | ||
50786956 DM |
112 | my $data = $generate_rrd_string->( |
113 | [$uptime, $sublevel, $ctime, $avg1, $maxcpu, $stat->{cpu}, $stat->{wait}, | |
114 | $meminfo->{memtotal}, $meminfo->{memused}, | |
115 | $meminfo->{swaptotal}, $meminfo->{swapused}, | |
78873100 TL |
116 | $dinfo->{blocks}, $dused, $netin, $netout] |
117 | ); | |
efd04666 DM |
118 | PVE::Cluster::broadcast_rrd("pve2-node/$nodename", $data); |
119 | ||
5e82aaac TL |
120 | my $node_metric = { |
121 | uptime => $uptime, | |
122 | cpustat => $stat, | |
123 | memory => $meminfo, | |
124 | blockstat => $dinfo, | |
125 | nics => $netdev, | |
126 | }; | |
127 | $node_metric->{cpustat}->@{qw(avg1 avg5 avg15)} = ($avg1, $avg5, $avg15); | |
128 | $node_metric->{cpustat}->{cpus} = $maxcpu; | |
129 | ||
87be2c19 TL |
130 | my $transactions = PVE::ExtMetric::transactions_start($status_cfg); |
131 | PVE::ExtMetric::update_all($transactions, 'node', $nodename, $node_metric, $ctime); | |
132 | PVE::ExtMetric::transactions_finish($transactions); | |
efd04666 DM |
133 | } |
134 | ||
135 | sub auto_balloning { | |
136 | my ($vmstatus) = @_; | |
137 | ||
0dd73a7f | 138 | my $log = sub { $opt_debug and printf @_ }; |
efd04666 DM |
139 | |
140 | my $hostmeminfo = PVE::ProcFSTools::read_meminfo(); | |
0dd73a7f | 141 | # NOTE: to debug, run 'pvestatd -d' and set memtotal here |
efd04666 | 142 | #$hostmeminfo->{memtotal} = int(2*1024*1024*1024/0.8); # you can set this to test |
efd04666 DM |
143 | my $hostfreemem = $hostmeminfo->{memtotal} - $hostmeminfo->{memused}; |
144 | ||
0dd73a7f TL |
145 | # try to use ~80% host memory; goal is the change amount required to achieve that |
146 | my $goal = int($hostmeminfo->{memtotal} * 0.8 - $hostmeminfo->{memused}); | |
147 | $log->("host goal: $goal free: $hostfreemem total: $hostmeminfo->{memtotal}\n"); | |
efd04666 DM |
148 | |
149 | my $maxchange = 100*1024*1024; | |
150 | my $res = PVE::AutoBalloon::compute_alg1($vmstatus, $goal, $maxchange); | |
efd04666 | 151 | |
0dd73a7f TL |
152 | for my $vmid (sort keys %$res) { |
153 | my $target = int($res->{$vmid}); | |
154 | my $current = int($vmstatus->{$vmid}->{balloon}); | |
155 | next if $target == $current; # no need to change | |
156 | ||
157 | $log->("BALLOON $vmid to $target (%d)\n", $target - $current); | |
7a108020 | 158 | eval { PVE::QemuServer::Monitor::mon_cmd($vmid, "balloon", value => $target) }; |
0dd73a7f | 159 | warn $@ if $@; |
efd04666 DM |
160 | } |
161 | } | |
162 | ||
163 | sub update_qemu_status { | |
164 | my ($status_cfg) = @_; | |
165 | ||
166 | my $ctime = time(); | |
efd04666 DM |
167 | my $vmstatus = PVE::QemuServer::vmstatus(undef, 1); |
168 | ||
169 | eval { auto_balloning($vmstatus); }; | |
170 | syslog('err', "auto ballooning error: $@") if $@; | |
171 | ||
87be2c19 | 172 | my $transactions = PVE::ExtMetric::transactions_start($status_cfg); |
efd04666 DM |
173 | foreach my $vmid (keys %$vmstatus) { |
174 | my $d = $vmstatus->{$vmid}; | |
175 | my $data; | |
176 | my $status = $d->{qmpstatus} || $d->{status} || 'stopped'; | |
177 | my $template = $d->{template} ? $d->{template} : "0"; | |
178 | if ($d->{pid}) { # running | |
50786956 DM |
179 | $data = $generate_rrd_string->( |
180 | [$d->{uptime}, $d->{name}, $status, $template, $ctime, $d->{cpus}, $d->{cpu}, | |
181 | $d->{maxmem}, $d->{mem}, $d->{maxdisk}, $d->{disk}, | |
182 | $d->{netin}, $d->{netout}, $d->{diskread}, $d->{diskwrite}]); | |
efd04666 | 183 | } else { |
50786956 DM |
184 | $data = $generate_rrd_string->( |
185 | [0, $d->{name}, $status, $template, $ctime, $d->{cpus}, undef, | |
186 | $d->{maxmem}, undef, $d->{maxdisk}, $d->{disk}, undef, undef, undef, undef]); | |
efd04666 DM |
187 | } |
188 | PVE::Cluster::broadcast_rrd("pve2.3-vm/$vmid", $data); | |
189 | ||
87be2c19 | 190 | PVE::ExtMetric::update_all($transactions, 'qemu', $vmid, $d, $ctime, $nodename); |
efd04666 | 191 | } |
87be2c19 TL |
192 | |
193 | PVE::ExtMetric::transactions_finish($transactions); | |
efd04666 DM |
194 | } |
195 | ||
196 | sub remove_stale_lxc_consoles { | |
197 | ||
198 | my $vmstatus = PVE::LXC::vmstatus(); | |
199 | my $pidhash = PVE::LXC::find_lxc_console_pids(); | |
200 | ||
201 | foreach my $vmid (keys %$pidhash) { | |
202 | next if defined($vmstatus->{$vmid}); | |
203 | syslog('info', "remove stale lxc-console for CT $vmid"); | |
204 | foreach my $pid (@{$pidhash->{$vmid}}) { | |
205 | kill(9, $pid); | |
206 | } | |
207 | } | |
208 | } | |
209 | ||
b3f1adb2 DM |
210 | my $rebalance_error_count = {}; |
211 | ||
41db757b | 212 | sub rebalance_lxc_containers { |
41db757b DM |
213 | |
214 | return if !-d '/sys/fs/cgroup/cpuset/lxc'; # nothing to do... | |
215 | ||
216 | my $all_cpus = PVE::CpuSet->new_from_cgroup('lxc', 'effective_cpus'); | |
217 | my @allowed_cpus = $all_cpus->members(); | |
218 | my $cpucount = scalar(@allowed_cpus); | |
127470f4 | 219 | my $max_cpuid = $allowed_cpus[-1]; |
41db757b | 220 | |
127470f4 | 221 | my @cpu_ctcount = (0) x ($max_cpuid+1); |
41db757b DM |
222 | my @balanced_cts; |
223 | ||
0b959507 DM |
224 | my $modify_cpuset = sub { |
225 | my ($vmid, $cpuset, $newset) = @_; | |
226 | ||
b3f1adb2 DM |
227 | if (!$rebalance_error_count->{$vmid}) { |
228 | syslog('info', "modified cpu set for lxc/$vmid: " . | |
229 | $newset->short_string()); | |
230 | } | |
231 | ||
0b959507 | 232 | eval { |
cbce367d DM |
233 | |
234 | if (-d "/sys/fs/cgroup/cpuset/lxc/$vmid/ns") { | |
235 | # allow all, so that we can set new cpuset in /ns | |
236 | $all_cpus->write_to_cgroup("lxc/$vmid"); | |
237 | eval { | |
238 | $newset->write_to_cgroup("lxc/$vmid/ns"); | |
239 | }; | |
240 | if (my $err = $@) { | |
241 | warn $err if !$rebalance_error_count->{$vmid}++; | |
242 | # restore original | |
243 | $cpuset->write_to_cgroup("lxc/$vmid"); | |
244 | } else { | |
245 | # also apply to container root cgroup | |
246 | $newset->write_to_cgroup("lxc/$vmid"); | |
247 | $rebalance_error_count->{$vmid} = 0; | |
248 | } | |
0b959507 | 249 | } else { |
cbce367d | 250 | # old style container |
0b959507 | 251 | $newset->write_to_cgroup("lxc/$vmid"); |
b3f1adb2 | 252 | $rebalance_error_count->{$vmid} = 0; |
0b959507 DM |
253 | } |
254 | }; | |
b3f1adb2 DM |
255 | if (my $err = $@) { |
256 | warn $err if !$rebalance_error_count->{$vmid}++; | |
257 | } | |
0b959507 DM |
258 | }; |
259 | ||
e0dc09ad DM |
260 | my $ctlist = PVE::LXC::config_list(); |
261 | ||
262 | foreach my $vmid (sort keys %$ctlist) { | |
263 | next if ! -d "/sys/fs/cgroup/cpuset/lxc/$vmid"; | |
41db757b DM |
264 | |
265 | my ($conf, $cpuset); | |
266 | eval { | |
267 | ||
268 | $conf = PVE::LXC::Config->load_config($vmid); | |
269 | ||
270 | $cpuset = PVE::CpuSet->new_from_cgroup("lxc/$vmid"); | |
271 | }; | |
272 | if (my $err = $@) { | |
273 | warn $err; | |
274 | next; | |
275 | } | |
276 | ||
277 | my @cpuset_members = $cpuset->members(); | |
278 | ||
8b750abc | 279 | if (!PVE::LXC::Config->has_lxc_entry($conf, 'lxc.cgroup.cpuset.cpus')) { |
2499255b | 280 | |
8b750abc DM |
281 | my $cores = $conf->{cores} || $cpucount; |
282 | $cores = $cpucount if $cores > $cpucount; | |
41db757b | 283 | |
2499255b DM |
284 | # see if the number of cores was hot-reduced or |
285 | # hasn't been enacted at all yet | |
286 | my $newset = PVE::CpuSet->new(); | |
287 | if ($cores < scalar(@cpuset_members)) { | |
288 | for (my $i = 0; $i < $cores; $i++) { | |
289 | $newset->insert($cpuset_members[$i]); | |
290 | } | |
291 | } elsif ($cores > scalar(@cpuset_members)) { | |
292 | my $count = $newset->insert(@cpuset_members); | |
293 | foreach my $cpu (@allowed_cpus) { | |
294 | $count += $newset->insert($cpu); | |
295 | last if $count >= $cores; | |
296 | } | |
297 | } else { | |
298 | $newset->insert(@cpuset_members); | |
299 | } | |
07f9595f | 300 | |
2499255b DM |
301 | # Apply hot-plugged changes if any: |
302 | if (!$newset->is_equal($cpuset)) { | |
303 | @cpuset_members = $newset->members(); | |
0b959507 | 304 | $modify_cpuset->($vmid, $cpuset, $newset); |
2499255b | 305 | } |
07f9595f | 306 | |
2499255b DM |
307 | # Note: no need to rebalance if we already use all cores |
308 | push @balanced_cts, [$vmid, $cores, $newset] | |
8b750abc | 309 | if defined($conf->{cores}) && ($cores != $cpucount); |
2499255b | 310 | } |
07f9595f | 311 | |
2499255b | 312 | foreach my $cpu (@cpuset_members) { |
ccfff920 | 313 | $cpu_ctcount[$cpu]++ if $cpu <= $max_cpuid; |
07f9595f | 314 | } |
2499255b | 315 | } |
07f9595f | 316 | |
2499255b DM |
317 | my $find_best_cpu = sub { |
318 | my ($cpulist, $cpu) = @_; | |
07f9595f | 319 | |
2499255b DM |
320 | my $cur_cost = $cpu_ctcount[$cpu]; |
321 | my $cur_cpu = $cpu; | |
41db757b | 322 | |
2499255b DM |
323 | foreach my $candidate (@$cpulist) { |
324 | my $cost = $cpu_ctcount[$candidate]; | |
325 | if ($cost < ($cur_cost -1)) { | |
326 | $cur_cost = $cost; | |
327 | $cur_cpu = $candidate; | |
328 | } | |
07f9595f DM |
329 | } |
330 | ||
2499255b DM |
331 | return $cur_cpu; |
332 | }; | |
333 | ||
334 | foreach my $bct (@balanced_cts) { | |
335 | my ($vmid, $cores, $cpuset) = @$bct; | |
41db757b DM |
336 | |
337 | my $newset = PVE::CpuSet->new(); | |
338 | ||
2499255b DM |
339 | my $rest = []; |
340 | foreach my $cpu (@allowed_cpus) { | |
341 | next if $cpuset->has($cpu); | |
342 | push @$rest, $cpu; | |
343 | } | |
344 | ||
345 | my @members = $cpuset->members(); | |
346 | foreach my $cpu (@members) { | |
347 | my $best = &$find_best_cpu($rest, $cpu); | |
348 | if ($best != $cpu) { | |
349 | $cpu_ctcount[$best]++; | |
350 | $cpu_ctcount[$cpu]--; | |
351 | } | |
352 | $newset->insert($best); | |
41db757b DM |
353 | } |
354 | ||
355 | if (!$newset->is_equal($cpuset)) { | |
0b959507 | 356 | $modify_cpuset->($vmid, $cpuset, $newset); |
41db757b DM |
357 | } |
358 | } | |
359 | } | |
360 | ||
efd04666 DM |
361 | sub update_lxc_status { |
362 | my ($status_cfg) = @_; | |
363 | ||
364 | my $ctime = time(); | |
efd04666 DM |
365 | my $vmstatus = PVE::LXC::vmstatus(); |
366 | ||
87be2c19 TL |
367 | my $transactions = PVE::ExtMetric::transactions_start($status_cfg); |
368 | ||
efd04666 DM |
369 | foreach my $vmid (keys %$vmstatus) { |
370 | my $d = $vmstatus->{$vmid}; | |
371 | my $template = $d->{template} ? $d->{template} : "0"; | |
372 | my $data; | |
373 | if ($d->{status} eq 'running') { # running | |
50786956 DM |
374 | $data = $generate_rrd_string->( |
375 | [$d->{uptime}, $d->{name}, $d->{status}, $template, | |
376 | $ctime, $d->{cpus}, $d->{cpu}, | |
377 | $d->{maxmem}, $d->{mem}, | |
378 | $d->{maxdisk}, $d->{disk}, | |
379 | $d->{netin}, $d->{netout}, | |
380 | $d->{diskread}, $d->{diskwrite}]); | |
efd04666 | 381 | } else { |
50786956 DM |
382 | $data = $generate_rrd_string->( |
383 | [0, $d->{name}, $d->{status}, $template, $ctime, $d->{cpus}, undef, | |
384 | $d->{maxmem}, undef, $d->{maxdisk}, $d->{disk}, undef, undef, undef, undef]); | |
efd04666 DM |
385 | } |
386 | PVE::Cluster::broadcast_rrd("pve2.3-vm/$vmid", $data); | |
387 | ||
87be2c19 | 388 | PVE::ExtMetric::update_all($transactions, 'lxc', $vmid, $d, $ctime, $nodename); |
efd04666 | 389 | } |
87be2c19 | 390 | PVE::ExtMetric::transactions_finish($transactions); |
efd04666 DM |
391 | } |
392 | ||
393 | sub update_storage_status { | |
394 | my ($status_cfg) = @_; | |
395 | ||
bbcfdc08 | 396 | my $cfg = PVE::Storage::config(); |
efd04666 | 397 | my $ctime = time(); |
efd04666 DM |
398 | my $info = PVE::Storage::storage_info($cfg); |
399 | ||
87be2c19 TL |
400 | my $transactions = PVE::ExtMetric::transactions_start($status_cfg); |
401 | ||
efd04666 DM |
402 | foreach my $storeid (keys %$info) { |
403 | my $d = $info->{$storeid}; | |
404 | next if !$d->{active}; | |
405 | ||
50786956 | 406 | my $data = $generate_rrd_string->([$ctime, $d->{total}, $d->{used}]); |
efd04666 DM |
407 | |
408 | my $key = "pve2-storage/${nodename}/$storeid"; | |
409 | PVE::Cluster::broadcast_rrd($key, $data); | |
410 | ||
87be2c19 | 411 | PVE::ExtMetric::update_all($transactions, 'storage', $nodename, $storeid, $d, $ctime); |
efd04666 | 412 | } |
87be2c19 | 413 | PVE::ExtMetric::transactions_finish($transactions); |
efd04666 DM |
414 | } |
415 | ||
5ea29d13 FG |
416 | sub rotate_authkeys { |
417 | PVE::AccessControl::rotate_authkey() if !PVE::AccessControl::check_authkey(1); | |
418 | } | |
419 | ||
a6dff455 TL |
420 | sub update_ceph_metadata { |
421 | return if !PVE::Ceph::Tools::check_ceph_inited(1); # nothing to do | |
422 | ||
423 | PVE::Ceph::Services::broadcast_ceph_services(); | |
a78fd21f | 424 | |
1aaca6fd TL |
425 | my ($version, $buildcommit, $vers_parts) = PVE::Ceph::Tools::get_local_version(1); |
426 | ||
427 | ||
428 | my $local_last_version = PVE::Cluster::get_node_kv('ceph-versions'); | |
429 | ||
fea39196 | 430 | if ($version) { |
1aaca6fd | 431 | # FIXME: remove with 7.0 - for backward compat only |
fea39196 | 432 | PVE::Cluster::broadcast_node_kv("ceph-version", $version); |
1aaca6fd TL |
433 | |
434 | my $node_versions = { | |
435 | version => { | |
436 | str => $version, | |
437 | parts => $vers_parts, | |
438 | }, | |
439 | buildcommit => $buildcommit, | |
440 | }; | |
441 | PVE::Cluster::broadcast_node_kv("ceph-versions", encode_json($node_versions)); | |
fea39196 DC |
442 | } |
443 | } | |
444 | ||
a36565ba AD |
445 | sub update_sdn_status { |
446 | ||
447 | if($have_sdn) { | |
448 | my ($transport_status, $vnet_status) = PVE::Network::SDN::status(); | |
449 | ||
450 | my $status = $transport_status ? encode_json($transport_status) : undef; | |
451 | PVE::Cluster::broadcast_node_kv("sdn", $status); | |
452 | } | |
453 | } | |
454 | ||
efd04666 DM |
455 | sub update_status { |
456 | ||
457 | # update worker list. This is not really required and | |
458 | # we just call this to make sure that we have a correct | |
459 | # list in case of an unexpected crash. | |
8a9bf777 DM |
460 | my $rpcenv = PVE::RPCEnvironment::get(); |
461 | ||
efd04666 | 462 | eval { |
8a9bf777 | 463 | my $tlist = $rpcenv->active_workers(); |
efd04666 DM |
464 | PVE::Cluster::broadcast_tasklist($tlist); |
465 | }; | |
466 | my $err = $@; | |
467 | syslog('err', $err) if $err; | |
468 | ||
469 | my $status_cfg = PVE::Cluster::cfs_read_file('status.cfg'); | |
470 | ||
471 | eval { | |
472 | update_node_status($status_cfg); | |
473 | }; | |
474 | $err = $@; | |
475 | syslog('err', "node status update error: $err") if $err; | |
476 | ||
477 | eval { | |
478 | update_qemu_status($status_cfg); | |
479 | }; | |
480 | $err = $@; | |
481 | syslog('err', "qemu status update error: $err") if $err; | |
482 | ||
483 | eval { | |
484 | update_lxc_status($status_cfg); | |
485 | }; | |
486 | $err = $@; | |
487 | syslog('err', "lxc status update error: $err") if $err; | |
488 | ||
e0dc09ad DM |
489 | eval { |
490 | rebalance_lxc_containers(); | |
491 | }; | |
492 | $err = $@; | |
493 | syslog('err', "lxc cpuset rebalance error: $err") if $err; | |
494 | ||
efd04666 DM |
495 | eval { |
496 | update_storage_status($status_cfg); | |
497 | }; | |
498 | $err = $@; | |
499 | syslog('err', "storage status update error: $err") if $err; | |
500 | ||
501 | eval { | |
502 | remove_stale_lxc_consoles(); | |
503 | }; | |
504 | $err = $@; | |
505 | syslog('err', "lxc console cleanup error: $err") if $err; | |
5ea29d13 FG |
506 | |
507 | eval { | |
508 | rotate_authkeys(); | |
509 | }; | |
510 | $err = $@; | |
511 | syslog('err', "authkey rotation error: $err") if $err; | |
512 | ||
fea39196 | 513 | eval { |
a6dff455 | 514 | update_ceph_metadata(); |
fea39196 DC |
515 | }; |
516 | $err = $@; | |
2a8e5149 | 517 | syslog('err', "ceph metadata update error: $err") if $err; |
fea39196 | 518 | |
a36565ba AD |
519 | eval { |
520 | update_sdn_status(); | |
521 | }; | |
522 | $err = $@; | |
523 | syslog('err', "sdn status update error: $err") if $err; | |
524 | ||
efd04666 DM |
525 | } |
526 | ||
527 | my $next_update = 0; | |
528 | ||
529 | # do not update directly after startup, because install scripts | |
530 | # have a problem with that | |
531 | my $cycle = 0; | |
532 | my $updatetime = 10; | |
533 | ||
534 | my $initial_memory_usage; | |
535 | ||
536 | sub run { | |
537 | my ($self) = @_; | |
538 | ||
539 | for (;;) { # forever | |
540 | ||
541 | $next_update = time() + $updatetime; | |
542 | ||
543 | if ($cycle) { | |
544 | my ($ccsec, $cusec) = gettimeofday (); | |
545 | eval { | |
546 | # syslog('info', "start status update"); | |
547 | PVE::Cluster::cfs_update(); | |
548 | update_status(); | |
549 | }; | |
550 | my $err = $@; | |
551 | ||
552 | if ($err) { | |
553 | syslog('err', "status update error: $err"); | |
554 | } | |
555 | ||
556 | my ($ccsec_end, $cusec_end) = gettimeofday (); | |
557 | my $cptime = ($ccsec_end-$ccsec) + ($cusec_end - $cusec)/1000000; | |
558 | ||
559 | syslog('info', sprintf("status update time (%.3f seconds)", $cptime)) | |
560 | if ($cptime > 5); | |
561 | } | |
562 | ||
563 | $cycle++; | |
564 | ||
565 | my $mem = PVE::ProcFSTools::read_memory_usage(); | |
cc3d280b | 566 | my $resident_kb = $mem->{resident} / 1024; |
efd04666 DM |
567 | |
568 | if (!defined($initial_memory_usage) || ($cycle < 10)) { | |
cc3d280b | 569 | $initial_memory_usage = $resident_kb; |
efd04666 | 570 | } else { |
cc3d280b | 571 | my $diff = $resident_kb - $initial_memory_usage; |
2112d310 | 572 | if ($diff > 15 * 1024) { |
efd04666 | 573 | syslog ('info', "restarting server after $cycle cycles to " . |
cc3d280b | 574 | "reduce memory usage (free $resident_kb ($diff) KB)"); |
efd04666 DM |
575 | $self->restart_daemon(); |
576 | } | |
577 | } | |
578 | ||
579 | my $wcount = 0; | |
580 | while ((time() < $next_update) && | |
581 | ($wcount < $updatetime) && # protect against time wrap | |
582 | !$restart_request) { $wcount++; sleep (1); }; | |
583 | ||
584 | $self->restart_daemon() if $restart_request; | |
585 | } | |
586 | } | |
587 | ||
588 | $daemon->register_start_command(); | |
589 | $daemon->register_restart_command(1); | |
590 | $daemon->register_stop_command(); | |
591 | $daemon->register_status_command(); | |
592 | ||
593 | our $cmddef = { | |
594 | start => [ __PACKAGE__, 'start', []], | |
595 | restart => [ __PACKAGE__, 'restart', []], | |
596 | stop => [ __PACKAGE__, 'stop', []], | |
597 | status => [ __PACKAGE__, 'status', [], undef, sub { print shift . "\n";} ], | |
598 | }; | |
599 | ||
efd04666 DM |
600 | 1; |
601 | ||
efd04666 DM |
602 | |
603 | ||
604 | ||
605 |