]>
Commit | Line | Data |
---|---|---|
75584009 FG |
1 | package PVE::CLI::pve5to6; |
2 | ||
3 | use strict; | |
4 | use warnings; | |
5 | ||
6 | use PVE::API2::APT; | |
7 | use PVE::API2::Ceph; | |
8 | use PVE::API2::LXC; | |
9 | use PVE::API2::Qemu; | |
91bebedb | 10 | use PVE::API2::Certificates; |
75584009 FG |
11 | |
12 | use PVE::Ceph::Tools; | |
13 | use PVE::Cluster; | |
14 | use PVE::Corosync; | |
15 | use PVE::INotify; | |
16 | use PVE::JSONSchema; | |
17 | use PVE::RPCEnvironment; | |
18 | use PVE::Storage; | |
bd6a59c2 | 19 | use PVE::Tools qw(run_command); |
03f79391 | 20 | use PVE::QemuServer; |
75584009 | 21 | |
6acde780 FG |
22 | use Term::ANSIColor; |
23 | ||
75584009 FG |
24 | use PVE::CLIHandler; |
25 | ||
26 | use base qw(PVE::CLIHandler); | |
27 | ||
28 | my $nodename = PVE::INotify::nodename(); | |
29 | ||
30 | sub setup_environment { | |
31 | PVE::RPCEnvironment->setup_default_cli_env(); | |
32 | } | |
33 | ||
f31c3e4a TL |
34 | my $min_pve_major = 5; |
35 | my $min_pve_minor = 4; | |
75584009 FG |
36 | my $min_pve_pkgrel = 2; |
37 | ||
38 | my $counters = { | |
39 | pass => 0, | |
40 | skip => 0, | |
41 | warn => 0, | |
42 | fail => 0, | |
43 | }; | |
44 | ||
45 | my $log_line = sub { | |
46 | my ($level, $line) = @_; | |
47 | ||
48 | $counters->{$level}++ if defined($level) && defined($counters->{$level}); | |
49 | ||
50 | print uc($level), ': ' if defined($level); | |
51 | print "$line\n"; | |
52 | }; | |
53 | ||
54 | sub log_pass { | |
6acde780 | 55 | print color('green'); |
75584009 | 56 | $log_line->('pass', @_); |
6acde780 | 57 | print color('reset'); |
75584009 FG |
58 | } |
59 | ||
60 | sub log_info { | |
61 | $log_line->('info', @_); | |
62 | } | |
63 | sub log_skip { | |
64 | $log_line->('skip', @_); | |
65 | } | |
66 | sub log_warn { | |
6acde780 | 67 | print color('yellow'); |
75584009 | 68 | $log_line->('warn', @_); |
6acde780 | 69 | print color('reset'); |
75584009 FG |
70 | } |
71 | sub log_fail { | |
6acde780 | 72 | print color('red'); |
75584009 | 73 | $log_line->('fail', @_); |
6acde780 | 74 | print color('reset'); |
75584009 FG |
75 | } |
76 | ||
465b3ea2 TL |
77 | my $print_header_first = 1; |
78 | sub print_header { | |
79 | my ($h) = @_; | |
80 | print "\n" if !$print_header_first; | |
81 | print "= $h =\n\n"; | |
82 | $print_header_first = 0; | |
83 | } | |
84 | ||
14886691 TL |
85 | my $get_systemd_unit_state = sub { |
86 | my ($unit) = @_; | |
87 | ||
88 | my $state; | |
89 | my $filter_output = sub { | |
90 | $state = shift; | |
91 | chomp $state; | |
92 | }; | |
93 | eval { | |
94 | run_command(['systemctl', 'is-enabled', "$unit"], outfunc => $filter_output, noerr => 1); | |
95 | return if !defined($state); | |
96 | run_command(['systemctl', 'is-active', "$unit"], outfunc => $filter_output, noerr => 1); | |
97 | }; | |
98 | ||
99 | return $state // 'unknown'; | |
100 | }; | |
101 | my $log_systemd_unit_state = sub { | |
102 | my ($unit, $no_fail_on_inactive) = @_; | |
103 | ||
104 | my $log_method = \&log_warn; | |
105 | ||
106 | my $state = $get_systemd_unit_state->($unit); | |
107 | if ($state eq 'active') { | |
108 | $log_method = \&log_pass; | |
109 | } elsif ($state eq 'inactive') { | |
110 | $log_method = $no_fail_on_inactive ? \&log_warn : \&log_fail; | |
111 | } elsif ($state eq 'failed') { | |
112 | $log_method = \&log_fail; | |
113 | } | |
114 | ||
115 | $log_method->("systemd unit '$unit' is in state '$state'"); | |
116 | }; | |
117 | ||
7d1b353b | 118 | my $versions; |
75584009 FG |
119 | my $get_pkg = sub { |
120 | my ($pkg) = @_; | |
121 | ||
7d1b353b | 122 | $versions = eval { PVE::API2::APT->versions({ node => $nodename }) } if !defined($versions); |
75584009 FG |
123 | |
124 | if (!defined($versions)) { | |
125 | my $msg = "unable to retrieve package version information"; | |
126 | $msg .= "- $@" if $@; | |
127 | log_fail("$msg"); | |
128 | return undef; | |
129 | } | |
130 | ||
131 | my $pkgs = [ grep { $_->{Package} eq $pkg } @$versions ]; | |
132 | if (!defined $pkgs || $pkgs == 0) { | |
133 | log_fail("unable to determine installed $pkg version."); | |
134 | return undef; | |
135 | } else { | |
136 | return $pkgs->[0]; | |
137 | } | |
138 | }; | |
139 | ||
140 | sub check_pve_packages { | |
465b3ea2 | 141 | print_header("CHECKING VERSION INFORMATION FOR PVE PACKAGES"); |
75584009 | 142 | |
465b3ea2 | 143 | print "Checking for package updates..\n"; |
75584009 FG |
144 | my $updates = eval { PVE::API2::APT->list_updates({ node => $nodename }); }; |
145 | if (!defined($updates)) { | |
146 | log_warn("$@") if $@; | |
147 | log_fail("unable to retrieve list of package updates!"); | |
148 | } elsif (@$updates > 0) { | |
149 | my $pkgs = join(', ', map { $_->{Package} } @$updates); | |
079d1188 | 150 | log_warn("updates for the following packages are available:\n $pkgs"); |
75584009 FG |
151 | } else { |
152 | log_pass("all packages uptodate"); | |
153 | } | |
154 | ||
155 | print "\nChecking proxmox-ve package version..\n"; | |
156 | if (defined(my $proxmox_ve = $get_pkg->('proxmox-ve'))) { | |
f31c3e4a | 157 | my $min_pve_ver = "$min_pve_major.$min_pve_minor-$min_pve_pkgrel"; |
75584009 | 158 | |
f31c3e4a TL |
159 | my ($maj, $min, $pkgrel) = $proxmox_ve->{OldVersion} =~ m/^(\d+)\.(\d+)-(\d+)/; |
160 | ||
f034380d TL |
161 | my $upgraded = 0; |
162 | ||
f31c3e4a | 163 | if ($maj > $min_pve_major) { |
1d2413c0 TL |
164 | if ($maj > $min_pve_major + 1) { |
165 | log_warn("already upgraded at least two major versions of Proxmox VE, this script is probably not useful anymore"); | |
166 | } else { | |
167 | log_pass("already upgraded to Proxmox VE " . ($min_pve_major + 1)); | |
168 | } | |
169 | log_pass("There's a newer major release available, maybe you wanted to call 'pve6to7'?") if 0; # TODO: enable once PVE 7 is out | |
f034380d | 170 | $upgraded = 1; |
f31c3e4a | 171 | } elsif ($maj >= $min_pve_major && $min >= $min_pve_minor && $pkgrel >= $min_pve_pkgrel) { |
75584009 FG |
172 | log_pass("proxmox-ve package has version >= $min_pve_ver"); |
173 | } else { | |
174 | log_fail("proxmox-ve package is too old, please upgrade to >= $min_pve_ver!"); | |
175 | } | |
f034380d TL |
176 | |
177 | my ($krunning, $kinstalled) = (qr/5\./, 'pve-kernel-5.0'); | |
178 | if (!$upgraded) { | |
179 | ($krunning, $kinstalled) = (qr/4\.15/, 'pve-kernel-4.15'); | |
180 | } | |
181 | ||
f8da1299 FG |
182 | print "\nChecking running kernel version..\n"; |
183 | my $kernel_ver = $proxmox_ve->{RunningKernel}; | |
184 | if (!defined($kernel_ver)) { | |
185 | log_fail("unable to determine running kernel version."); | |
f034380d | 186 | } elsif ($kernel_ver =~ /^$krunning/) { |
f8da1299 | 187 | log_pass("expected running kernel '$kernel_ver'."); |
f034380d TL |
188 | } elsif ($get_pkg->($kinstalled)) { |
189 | log_warn("expected kernel '$kinstalled' intalled but not yet rebooted!"); | |
f8da1299 | 190 | } else { |
f034380d | 191 | log_warn("unexpected running and installed kernel '$kernel_ver'."); |
f8da1299 | 192 | } |
75584009 FG |
193 | } |
194 | } | |
195 | ||
03f79391 DC |
196 | sub get_vms_with_vmx { |
197 | my $res = { | |
198 | cpu => [], | |
199 | flag => [], | |
200 | }; | |
201 | my $vmlist = PVE::QemuServer::vzlist(); | |
000acaca | 202 | |
03f79391 DC |
203 | foreach my $vmid ( sort { $a <=> $b } keys %$vmlist ) { |
204 | my $pid = $vmlist->{$vmid}->{pid}; | |
205 | next if !$pid; # skip not running vms | |
000acaca | 206 | |
03f79391 DC |
207 | my $cmdline = eval { PVE::Tools::file_get_contents("/proc/$pid/cmdline") }; |
208 | if ($cmdline) { | |
209 | my @args = split(/\0/, $cmdline); | |
210 | for (my $i = 0; $i < scalar(@args); $i++) { | |
211 | next if !$args[$i] || $args[$i] !~ m/^-?-cpu$/; | |
000acaca | 212 | |
03f79391 DC |
213 | my $cpuarg = $args[$i+1]; |
214 | if ($cpuarg =~ m/^(host|max)/) { | |
215 | push @{$res->{cpu}}, $vmid; | |
216 | } elsif ($cpuarg =~ m/\+(vmx|svm)/) { | |
217 | push @{$res->{flag}}, $vmid; | |
218 | } | |
219 | } | |
220 | } | |
221 | } | |
000acaca TL |
222 | |
223 | $res = undef if (scalar(@{$res->{cpu}}) + scalar(@{$res->{flag}})) <= 0; | |
224 | ||
225 | return $res; | |
03f79391 DC |
226 | } |
227 | ||
eb8bf430 | 228 | sub check_kvm_nested { |
000acaca TL |
229 | log_info("Checking KVM nesting support, which breaks live migration for VMs using it.."); |
230 | ||
eb8bf430 DC |
231 | my $module_sysdir = "/sys/module"; |
232 | if (-e "$module_sysdir/kvm_amd") { | |
233 | $module_sysdir .= "/kvm_amd/parameters"; | |
234 | } elsif (-e "$module_sysdir/kvm_intel") { | |
235 | $module_sysdir .= "/kvm_intel/parameters"; | |
236 | } else { | |
237 | log_skip("no kvm module found"); | |
238 | return; | |
239 | } | |
240 | ||
241 | if (-f "$module_sysdir/nested") { | |
242 | my $val = eval { PVE::Tools::file_read_firstline("$module_sysdir/nested") }; | |
243 | if ($val && $val =~ m/Y|1/) { | |
000acaca TL |
244 | my $list = get_vms_with_vmx(); |
245 | if (!defined($list)) { | |
246 | log_pass("KVM nested parameter set, but currently no VM with a 'vmx' or 'svm' flag is running."); | |
03f79391 | 247 | } else { |
000acaca | 248 | my $warnmsg = "KVM nested enabled. It will not be possible to live migrate the following running VMs to PVE 6:\n"; |
03f79391 | 249 | if (@{$list->{cpu}}) { |
000acaca | 250 | $warnmsg .= " VMID(s) with cputype 'host' or 'max': " . join(',', @{$list->{cpu}}) . "\n"; |
03f79391 | 251 | } |
03f79391 | 252 | if (@{$list->{flag}}) { |
000acaca | 253 | $warnmsg .= " VMID(s) with enforced cpu flag 'vmx' or 'svm': " . join(',', @{$list->{flag}}) . "\n"; |
03f79391 | 254 | } |
03f79391 DC |
255 | log_warn($warnmsg); |
256 | } | |
eb8bf430 DC |
257 | } else { |
258 | log_pass("KVM nested parameter not set.") | |
259 | } | |
260 | } else { | |
261 | log_skip("KVM nested parameter not found."); | |
262 | } | |
263 | } | |
264 | ||
75584009 | 265 | sub check_storage_health { |
465b3ea2 | 266 | print_header("CHECKING CONFIGURED STORAGES"); |
75584009 FG |
267 | my $cfg = PVE::Storage::config(); |
268 | ||
269 | my $ctime = time(); | |
270 | ||
271 | my $info = PVE::Storage::storage_info($cfg); | |
272 | ||
273 | foreach my $storeid (keys %$info) { | |
274 | my $d = $info->{$storeid}; | |
275 | if ($d->{enabled}) { | |
68f0d161 | 276 | if ($d->{type} eq 'sheepdog') { |
b8bdb17e | 277 | log_fail("storage '$storeid' of type 'sheepdog' is enabled - experimental sheepdog support dropped in PVE 6") |
68f0d161 | 278 | } elsif ($d->{active}) { |
75584009 FG |
279 | log_pass("storage '$storeid' enabled and active."); |
280 | } else { | |
281 | log_warn("storage '$storeid' enabled but not active!"); | |
282 | } | |
283 | } else { | |
284 | log_skip("storage '$storeid' disabled."); | |
285 | } | |
286 | } | |
287 | } | |
288 | ||
289 | sub check_cluster_corosync { | |
465b3ea2 | 290 | print_header("CHECKING CLUSTER HEALTH/SETTINGS"); |
75584009 FG |
291 | |
292 | if (!PVE::Corosync::check_conf_exists(1)) { | |
293 | log_skip("standalone node."); | |
294 | return; | |
295 | } | |
296 | ||
14886691 TL |
297 | $log_systemd_unit_state->('pve-cluster.service'); |
298 | $log_systemd_unit_state->('corosync.service'); | |
299 | ||
75584009 | 300 | if (PVE::Cluster::check_cfs_quorum(1)) { |
5c420879 | 301 | log_pass("Cluster Filesystem is quorate."); |
75584009 | 302 | } else { |
5c420879 | 303 | log_fail("Cluster Filesystem readonly, lost quorum?!"); |
75584009 FG |
304 | } |
305 | ||
306 | my $conf = PVE::Cluster::cfs_read_file('corosync.conf'); | |
307 | my $conf_nodelist = PVE::Corosync::nodelist($conf); | |
82fd088c | 308 | my $node_votes = 0; |
75584009 | 309 | |
a2e5aa4e | 310 | print "\nAnalzying quorum settings and state..\n"; |
75584009 FG |
311 | if (!defined($conf_nodelist)) { |
312 | log_fail("unable to retrieve nodelist from corosync.conf"); | |
82fd088c FG |
313 | } else { |
314 | if (grep { $conf_nodelist->{$_}->{quorum_votes} != 1 } keys %$conf_nodelist) { | |
315 | log_warn("non-default quorum_votes distribution detected!"); | |
316 | } | |
317 | map { $node_votes += $conf_nodelist->{$_}->{quorum_votes} // 0 } keys %$conf_nodelist; | |
318 | } | |
319 | ||
320 | my ($expected_votes, $total_votes); | |
321 | my $filter_output = sub { | |
322 | my $line = shift; | |
323 | ($expected_votes) = $line =~ /^Expected votes:\s*(\d+)\s*$/ | |
324 | if !defined($expected_votes); | |
325 | ($total_votes) = $line =~ /^Total votes:\s*(\d+)\s*$/ | |
326 | if !defined($total_votes); | |
327 | }; | |
328 | eval { | |
bd6a59c2 | 329 | run_command(['corosync-quorumtool', '-s'], outfunc => $filter_output, noerr => 1); |
82fd088c FG |
330 | }; |
331 | ||
332 | if (!defined($expected_votes)) { | |
333 | log_fail("unable to get expected number of votes, setting to 0."); | |
334 | $expected_votes = 0; | |
335 | } | |
336 | if (!defined($total_votes)) { | |
337 | log_fail("unable to get expected number of votes, setting to 0."); | |
23a60185 | 338 | $total_votes = 0; |
75584009 FG |
339 | } |
340 | ||
341 | my $cfs_nodelist = PVE::Cluster::get_clinfo()->{nodelist}; | |
342 | my $offline_nodes = grep { $cfs_nodelist->{$_}->{online} != 1 } keys %$cfs_nodelist; | |
343 | if ($offline_nodes > 0) { | |
344 | log_fail("$offline_nodes nodes are offline!"); | |
345 | } | |
346 | ||
82fd088c FG |
347 | my $qdevice_votes = 0; |
348 | if (my $qdevice_setup = $conf->{main}->{quorum}->{device}) { | |
349 | $qdevice_votes = $qdevice_setup->{votes} // 1; | |
350 | } | |
351 | ||
352 | log_info("configured votes - nodes: $node_votes"); | |
353 | log_info("configured votes - qdevice: $qdevice_votes"); | |
354 | log_info("current expected votes: $expected_votes"); | |
355 | log_info("current total votes: $total_votes"); | |
356 | ||
357 | log_warn("expected votes set to non-standard value '$expected_votes'.") | |
358 | if $expected_votes != $node_votes + $qdevice_votes; | |
359 | log_warn("total votes < expected votes: $total_votes/$expected_votes!") | |
360 | if $total_votes < $expected_votes; | |
361 | ||
75584009 FG |
362 | my $conf_nodelist_count = scalar(keys %$conf_nodelist); |
363 | my $cfs_nodelist_count = scalar(keys %$cfs_nodelist); | |
364 | log_warn("cluster consists of less than three nodes!") | |
82fd088c | 365 | if $conf_nodelist_count < 3 && $conf_nodelist_count + $qdevice_votes < 3; |
75584009 FG |
366 | |
367 | log_fail("corosync.conf ($conf_nodelist_count) and pmxcfs ($cfs_nodelist_count) don't agree about size of nodelist.") | |
368 | if $conf_nodelist_count != $cfs_nodelist_count; | |
369 | ||
a2e5aa4e | 370 | print "\nChecking nodelist entries..\n"; |
75584009 FG |
371 | foreach my $cs_node (keys %$conf_nodelist) { |
372 | my $entry = $conf_nodelist->{$cs_node}; | |
b481d354 | 373 | log_fail("$cs_node: no name entry in corosync.conf.") |
75584009 | 374 | if !defined($entry->{name}); |
b481d354 | 375 | log_fail("$cs_node: no nodeid configured in corosync.conf.") |
75584009 | 376 | if !defined($entry->{nodeid}); |
d1dcb3e9 FG |
377 | log_fail("$cs_node: neither ring0_addr nor ring1_addr defined in corosync.conf.") |
378 | if !defined($entry->{ring0_addr}) && !defined($entry->{ring1_addr}); | |
75584009 FG |
379 | |
380 | my $verify_ring_ip = sub { | |
381 | my $key = shift; | |
382 | my $ring = $entry->{$key}; | |
0e78b515 FG |
383 | if (defined($ring)) { |
384 | my ($resolved_ip, undef) = PVE::Corosync::resolve_hostname_like_corosync($ring, $conf); | |
385 | if (defined($resolved_ip)) { | |
386 | if ($resolved_ip ne $ring) { | |
b481d354 | 387 | log_warn("$cs_node: $key '$ring' resolves to '$resolved_ip'.\n Consider replacing it with the currently resolved IP address."); |
0e78b515 | 388 | } else { |
b481d354 | 389 | log_pass("$cs_node: $key is configured to use IP address '$ring'"); |
0e78b515 FG |
390 | } |
391 | } else { | |
b481d354 | 392 | log_fail("$cs_node: unable to resolve $key '$ring' to an IP address according to Corosync's resolve strategy - cluster will potentially fail with Corosync 3.x/kronosnet!"); |
0e78b515 | 393 | } |
75584009 FG |
394 | } |
395 | }; | |
396 | $verify_ring_ip->('ring0_addr'); | |
397 | $verify_ring_ip->('ring1_addr'); | |
398 | } | |
399 | ||
a2e5aa4e | 400 | print "\nChecking totem settings..\n"; |
75584009 | 401 | my $totem = $conf->{main}->{totem}; |
75584009 FG |
402 | my $transport = $totem->{transport}; |
403 | if (defined($transport)) { | |
82662dc8 FG |
404 | if ($transport ne 'knet') { |
405 | log_fail("Corosync transport explicitly set to '$transport' instead of implicit default!"); | |
406 | } else { | |
407 | log_pass("Corosync transport set to '$transport'."); | |
408 | } | |
409 | } else { | |
410 | log_pass("Corosync transport set to implicit default."); | |
75584009 FG |
411 | } |
412 | ||
413 | if ((!defined($totem->{secauth}) || $totem->{secauth} ne 'on') && (!defined($totem->{crypto_cipher}) || $totem->{crypto_cipher} eq 'none')) { | |
414 | log_fail("Corosync authentication/encryption is not explicitly enabled (secauth / crypto_cipher / crypto_hash)!"); | |
82662dc8 FG |
415 | } else { |
416 | if (defined($totem->{crypto_cipher}) && $totem->{crypto_cipher} eq '3des') { | |
417 | log_fail("Corosync encryption cipher set to '3des', no longer supported in Corosync 3.x!"); | |
418 | } else { | |
419 | log_pass("Corosync encryption and authentication enabled."); | |
420 | } | |
75584009 FG |
421 | } |
422 | ||
6ab8927e | 423 | print "\n"; |
0e788b3e | 424 | log_info("run 'pvecm status' to get detailed cluster status.."); |
75584009 | 425 | |
465b3ea2 | 426 | print_header("CHECKING INSTALLED COROSYNC VERSION"); |
75584009 FG |
427 | if (defined(my $corosync = $get_pkg->('corosync'))) { |
428 | if ($corosync->{OldVersion} =~ m/^2\./) { | |
429 | log_fail("corosync 2.x installed, cluster-wide upgrade to 3.x needed!"); | |
430 | } elsif ($corosync->{OldVersion} =~ m/^3\./) { | |
431 | log_pass("corosync 3.x installed."); | |
432 | } else { | |
433 | log_fail("unexpected corosync version installed: $corosync->{OldVersion}!"); | |
434 | } | |
435 | } | |
436 | } | |
437 | ||
438 | sub check_ceph { | |
465b3ea2 | 439 | print_header("CHECKING HYPER-CONVERGED CEPH STATUS"); |
75584009 FG |
440 | |
441 | if (PVE::Ceph::Tools::check_ceph_inited(1)) { | |
442 | log_info("hyper-converged ceph setup detected!"); | |
443 | } else { | |
444 | log_skip("no hyper-converged ceph setup detected!"); | |
445 | return; | |
446 | } | |
447 | ||
448 | log_info("getting Ceph status/health information.."); | |
449 | my $ceph_status = eval { PVE::API2::Ceph->status({ node => $nodename }); }; | |
450 | my $osd_flags = eval { PVE::API2::Ceph->get_flags({ node => $nodename }); }; | |
a1687fd9 | 451 | my $noout_wanted = 1; |
75cac279 | 452 | my $noout = $osd_flags && $osd_flags =~ m/noout/; |
75584009 FG |
453 | |
454 | if (!$ceph_status || !$ceph_status->{health}) { | |
455 | log_fail("unable to determine Ceph status!"); | |
456 | } else { | |
457 | my $ceph_health = $ceph_status->{health}->{status}; | |
458 | if (!$ceph_health) { | |
459 | log_fail("unable to determine Ceph health!"); | |
460 | } elsif ($ceph_health eq 'HEALTH_OK') { | |
461 | log_pass("Ceph health reported as 'HEALTH_OK'."); | |
462 | } elsif ($ceph_health eq 'HEALTH_WARN' && $noout && (keys %{$ceph_status->{health}->{checks}} == 1)) { | |
463 | log_pass("Ceph health reported as 'HEALTH_WARN' with a single failing check and 'noout' flag set."); | |
464 | } else { | |
6deabaac TL |
465 | log_warn("Ceph health reported as '$ceph_health'.\n Use the PVE ". |
466 | "dashboard or 'ceph -s' to determine the specific issues and try to resolve them."); | |
75584009 FG |
467 | } |
468 | } | |
469 | ||
470 | log_info("getting Ceph OSD flags.."); | |
471 | eval { | |
472 | if (!$osd_flags) { | |
473 | log_fail("unable to get Ceph OSD flags!"); | |
474 | } else { | |
475 | if ($osd_flags =~ m/recovery_deletes/ && $osd_flags =~ m/purged_snapdirs/) { | |
476 | log_pass("all PGs have been scrubbed at least once while running Ceph Luminous."); | |
477 | } else { | |
478 | log_fail("missing 'recovery_deletes' and/or 'purged_snapdirs' flag, scrub of all PGs required before upgrading to Nautilus!"); | |
479 | } | |
75584009 FG |
480 | } |
481 | }; | |
482 | ||
483 | log_info("getting Ceph daemon versions.."); | |
484 | my $ceph_versions = eval { PVE::Ceph::Tools::get_cluster_versions(undef, 1); }; | |
485 | if (!$ceph_versions) { | |
486 | log_fail("unable to determine Ceph daemon versions!"); | |
487 | } else { | |
488 | my $services = [ | |
489 | { 'key' => 'mon', 'name' => 'monitor' }, | |
490 | { 'key' => 'mgr', 'name' => 'manager' }, | |
491 | { 'key' => 'mds', 'name' => 'MDS' }, | |
492 | { 'key' => 'osd', 'name' => 'OSD' }, | |
493 | ]; | |
494 | ||
495 | foreach my $service (@$services) { | |
496 | my $name = $service->{name}; | |
497 | if (my $service_versions = $ceph_versions->{$service->{key}}) { | |
498 | if (keys %$service_versions == 0) { | |
499 | log_skip("no running instances detected for daemon type $name."); | |
500 | } elsif (keys %$service_versions == 1) { | |
501 | log_pass("single running version detected for daemon type $name."); | |
502 | } else { | |
503 | log_warn("multiple running versions detected for daemon type $name!"); | |
504 | } | |
505 | } else { | |
506 | log_skip("unable to determine versions of running Ceph $name instances."); | |
507 | } | |
508 | } | |
509 | ||
510 | my $overall_versions = $ceph_versions->{overall}; | |
511 | if (!$overall_versions) { | |
512 | log_warn("unable to determine overall Ceph daemon versions!"); | |
513 | } elsif (keys %$overall_versions == 1) { | |
514 | log_pass("single running overall version detected for all Ceph daemon types."); | |
a1687fd9 FG |
515 | if ((keys %$overall_versions)[0] =~ /^ceph version 14\./) { |
516 | $noout_wanted = 0; | |
517 | } | |
75584009 FG |
518 | } else { |
519 | log_warn("overall version mismatch detected, check 'ceph versions' output for details!"); | |
520 | } | |
521 | } | |
32526d27 | 522 | |
a1687fd9 FG |
523 | if ($noout) { |
524 | if ($noout_wanted) { | |
5d54e2ee | 525 | log_pass("'noout' flag set to prevent rebalancing during cluster-wide upgrades."); |
a1687fd9 | 526 | } else { |
5d54e2ee | 527 | log_warn("'noout' flag set, Ceph cluster upgrade seems finished."); |
a1687fd9 FG |
528 | } |
529 | } elsif ($noout_wanted) { | |
5d54e2ee | 530 | log_warn("'noout' flag not set - recommended to prevent rebalancing during upgrades."); |
a1687fd9 FG |
531 | } |
532 | ||
c553da92 DC |
533 | log_info("checking Ceph config.."); |
534 | my $conf = PVE::Cluster::cfs_read_file('ceph.conf'); | |
f9f8cd7d | 535 | if (%$conf) { |
c553da92 | 536 | my $global = $conf->{global}; |
780a57b6 TL |
537 | |
538 | my $global_monhost = $global->{mon_host} // $global->{"mon host"} // $global->{"mon-host"}; | |
539 | if (!defined($global_monhost)) { | |
540 | log_warn("No 'mon_host' entry found in ceph config.\n It's recommended to add mon_host with all monitor addresses (without ports) to the global section."); | |
c553da92 | 541 | } else { |
5d54e2ee | 542 | log_pass("Found 'mon_host' entry."); |
c553da92 | 543 | } |
38bd1ffc | 544 | |
60b81f16 TL |
545 | my $ipv6 = $global->{ms_bind_ipv6} // $global->{"ms bind ipv6"} // $global->{"ms-bind-ipv6"}; |
546 | if ($ipv6) { | |
547 | my $ipv4 = $global->{ms_bind_ipv4} // $global->{"ms bind ipv4"} // $global->{"ms-bind-ipv4"}; | |
38bd1ffc | 548 | if ($ipv6 eq 'true' && (!defined($ipv4) || $ipv4 ne 'false')) { |
5d54e2ee | 549 | log_warn("'ms_bind_ipv6' is enabled but 'ms_bind_ipv4' is not disabled.\n Make sure to disable 'ms_bind_ipv4' for ipv6 only clusters, or add an ipv4 network to public/cluster network."); |
38bd1ffc | 550 | } else { |
5d54e2ee | 551 | log_pass("'ms_bind_ipv6' is enabled and 'ms_bind_ipv4' disabled"); |
38bd1ffc DC |
552 | } |
553 | } else { | |
5d54e2ee | 554 | log_pass("'ms_bind_ipv6' not enabled"); |
38bd1ffc | 555 | } |
88056323 FG |
556 | |
557 | if (defined($global->{keyring})) { | |
558 | log_warn("[global] config section contains 'keyring' option, which will prevent services from starting with Nautilus.\n Move 'keyring' option to [client] section instead."); | |
559 | } else { | |
560 | log_pass("no 'keyring' option in [global] section found."); | |
561 | } | |
562 | ||
c553da92 | 563 | } else { |
f9f8cd7d | 564 | log_warn("Empty ceph config found"); |
c553da92 DC |
565 | } |
566 | ||
32526d27 FG |
567 | my $local_ceph_ver = PVE::Ceph::Tools::get_local_version(1); |
568 | if (defined($local_ceph_ver)) { | |
569 | if ($local_ceph_ver == 14) { | |
8be1b3ba | 570 | my $ceph_volume_osds = PVE::Ceph::Tools::ceph_volume_list(); |
32526d27 | 571 | my $scanned_osds = PVE::Tools::dir_glob_regex('/etc/ceph/osd', '^.*\.json$'); |
8be1b3ba | 572 | if (-e '/var/lib/ceph/osd/' && !defined($scanned_osds) && !(keys %$ceph_volume_osds)) { |
32526d27 FG |
573 | log_warn("local Ceph version is Nautilus, local OSDs detected, but no conversion from ceph-disk to ceph-volume done (yet)."); |
574 | } | |
575 | } | |
576 | } else { | |
577 | log_fail("unable to determine local Ceph version."); | |
578 | } | |
75584009 FG |
579 | } |
580 | ||
581 | sub check_misc { | |
465b3ea2 | 582 | print_header("MISCELLANEOUS CHECKS"); |
75584009 | 583 | my $ssh_config = eval { PVE::Tools::file_get_contents('/root/.ssh/config') }; |
27228998 FG |
584 | if (defined($ssh_config)) { |
585 | log_fail("Unsupported SSH Cipher configured for root in /root/.ssh/config: $1") | |
586 | if $ssh_config =~ /^Ciphers .*(blowfish|arcfour|3des).*$/m; | |
587 | } else { | |
588 | log_skip("No SSH config file found."); | |
589 | } | |
75584009 | 590 | |
37247035 TL |
591 | log_info("Checking common daemon services.."); |
592 | $log_systemd_unit_state->('pveproxy.service'); | |
593 | $log_systemd_unit_state->('pvedaemon.service'); | |
594 | $log_systemd_unit_state->('pvestatd.service'); | |
595 | ||
75584009 FG |
596 | my $root_free = PVE::Tools::df('/', 10); |
597 | log_warn("Less than 2G free space on root file system.") | |
598 | if defined($root_free) && $root_free->{avail} < 2*1024*1024*1024; | |
599 | ||
b6c10c63 | 600 | log_info("Checking for running guests.."); |
75584009 | 601 | my $running_guests = 0; |
b6c10c63 | 602 | |
75584009 FG |
603 | my $vms = eval { PVE::API2::Qemu->vmlist({ node => $nodename }) }; |
604 | log_warn("Failed to retrieve information about this node's VMs - $@") if $@; | |
b6c10c63 TL |
605 | $running_guests += grep { $_->{status} eq 'running' } @$vms if defined($vms); |
606 | ||
75584009 FG |
607 | my $cts = eval { PVE::API2::LXC->vmlist({ node => $nodename }) }; |
608 | log_warn("Failed to retrieve information about this node's CTs - $@") if $@; | |
b6c10c63 TL |
609 | $running_guests += grep { $_->{status} eq 'running' } @$cts if defined($cts); |
610 | ||
611 | if ($running_guests > 0) { | |
612 | log_warn("$running_guests running guest(s) detected - consider migrating or stopping them.") | |
613 | } else { | |
614 | log_pass("no running guest detected.") | |
615 | } | |
0286a659 | 616 | |
33232071 | 617 | log_info("Checking if the local node's hostname '$nodename' is resolvable.."); |
6a494e2e | 618 | my $local_ip = eval { PVE::Network::get_ip_from_hostname($nodename) }; |
0286a659 | 619 | if ($@) { |
6a494e2e | 620 | log_warn("Failed to resolve hostname '$nodename' to IP - $@"); |
0286a659 | 621 | } else { |
6c9e9cc8 | 622 | log_info("Checking if resolved IP is configured on local node.."); |
0286a659 ML |
623 | my $cidr = Net::IP::ip_is_ipv6($local_ip) ? "$local_ip/128" : "$local_ip/32"; |
624 | my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr); | |
625 | my $ip_count = scalar(@$configured_ips); | |
bacd0b0b TL |
626 | |
627 | if ($ip_count <= 0) { | |
6a494e2e | 628 | log_fail("Resolved node IP '$local_ip' not configured or active for '$nodename'"); |
bacd0b0b TL |
629 | } elsif ($ip_count > 1) { |
630 | log_warn("Resolved node IP '$local_ip' active on multiple ($ip_count) interfaces!"); | |
b6c10c63 | 631 | } else { |
6c9e9cc8 | 632 | log_pass("Resolved node IP '$local_ip' configured and active on single interface."); |
bacd0b0b | 633 | } |
0286a659 | 634 | } |
e6191e3e | 635 | |
7b74b60a | 636 | log_info("Check node certificate's RSA key size"); |
91bebedb | 637 | my $certs = PVE::API2::Certificates->info({ node => $nodename }); |
7b74b60a TL |
638 | my $certs_check = { |
639 | 'rsaEncryption' => { | |
640 | minsize => 2048, | |
641 | name => 'RSA', | |
642 | }, | |
643 | 'id-ecPublicKey' => { | |
644 | minsize => 224, | |
645 | name => 'ECC', | |
646 | }, | |
647 | }; | |
648 | ||
649 | my $certs_check_failed = 0; | |
650 | foreach my $cert (@$certs) { | |
651 | my ($type, $size, $fn) = $cert->@{qw(public-key-type public-key-bits filename)}; | |
652 | ||
653 | if (!defined($type) || !defined($size)) { | |
654 | log_warn("'$fn': cannot check certificate, failed to get it's type or size!"); | |
655 | } | |
656 | ||
657 | my $check = $certs_check->{$type}; | |
658 | if (!defined($check)) { | |
659 | log_warn("'$fn': certificate's public key type '$type' unknown, check Debian Busters release notes"); | |
660 | next; | |
661 | } | |
662 | ||
663 | if ($size < $check->{minsize}) { | |
664 | log_fail("'$fn', certificate's $check->{name} public key size is less than 2048 bit"); | |
665 | $certs_check_failed = 1; | |
666 | } else { | |
667 | log_pass("Certificate '$fn' passed Debian Busters security level for TLS connections ($size >= 2048)"); | |
91bebedb AA |
668 | } |
669 | } | |
91bebedb | 670 | |
e6191e3e | 671 | check_kvm_nested(); |
75584009 FG |
672 | } |
673 | ||
674 | __PACKAGE__->register_method ({ | |
675 | name => 'checklist', | |
676 | path => 'checklist', | |
677 | method => 'GET', | |
678 | description => 'Check (pre-/post-)upgrade conditions.', | |
679 | parameters => { | |
680 | additionalProperties => 0, | |
681 | properties => { | |
682 | }, | |
683 | }, | |
684 | returns => { type => 'null' }, | |
685 | code => sub { | |
686 | my ($param) = @_; | |
687 | ||
688 | check_pve_packages(); | |
689 | check_cluster_corosync(); | |
690 | check_ceph(); | |
691 | check_storage_health(); | |
692 | check_misc(); | |
693 | ||
465b3ea2 | 694 | print_header("SUMMARY"); |
bc91ccac TL |
695 | |
696 | my $total = 0; | |
697 | $total += $_ for values %$counters; | |
698 | ||
699 | print "TOTAL: $total\n"; | |
1c6ac415 TL |
700 | print colored("PASSED: $counters->{pass}\n", 'green'); |
701 | print "SKIPPED: $counters->{skip}\n"; | |
6acde780 FG |
702 | print colored("WARNINGS: $counters->{warn}\n", 'yellow'); |
703 | print colored("FAILURES: $counters->{fail}\n", 'red'); | |
75584009 | 704 | |
c50590ef TL |
705 | if ($counters->{warn} > 0 || $counters->{fail} > 0) { |
706 | my $color = $counters->{fail} > 0 ? 'red' : 'yellow'; | |
707 | print colored("\nATTENTION: Please check the output for detailed information!\n", $color); | |
99181ca8 | 708 | print colored("Try to solve the problems one at a time and then run this checklist tool again.\n", $color) if $counters->{fail} > 0; |
c50590ef | 709 | } |
75584009 FG |
710 | |
711 | return undef; | |
712 | }}); | |
713 | ||
3f11a62c TL |
714 | our $cmddef = [ __PACKAGE__, 'checklist', [], {}]; |
715 | ||
716 | # for now drop all unknown params and just check | |
717 | @ARGV = (); | |
75584009 FG |
718 | |
719 | 1; |