PVE/CLI/pve5to6.pm

   1 package PVE::CLI::pve5to6;
   2
   3 use strict;
   4 use warnings;
   5
   6 use PVE::API2::APT;
   7 use PVE::API2::Ceph;
   8 use PVE::API2::LXC;
   9 use PVE::API2::Qemu;
  10
  11 use PVE::Ceph::Tools;
  12 use PVE::Cluster;
  13 use PVE::Corosync;
  14 use PVE::INotify;
  15 use PVE::JSONSchema;
  16 use PVE::RPCEnvironment;
  17 use PVE::Storage;
  18 use PVE::Tools;
  19
  20 use Term::ANSIColor;
  21
  22 use PVE::CLIHandler;
  23
  24 use base qw(PVE::CLIHandler);
  25
  26 my $nodename = PVE::INotify::nodename();
  27
  28 sub setup_environment {
  29     PVE::RPCEnvironment->setup_default_cli_env();
  30 }
  31
  32 my $min_pve_major = 5;
  33 my $min_pve_minor = 4;
  34 my $min_pve_pkgrel = 2;
  35
  36 my $counters = {
  37     pass => 0,
  38     skip => 0,
  39     warn => 0,
  40     fail => 0,
  41 };
  42
  43 my $log_line = sub {
  44     my ($level, $line) = @_;
  45
  46     $counters->{$level}++ if defined($level) && defined($counters->{$level});
  47
  48     print uc($level), ': ' if defined($level);
  49     print "$line\n";
  50 };
  51
  52 sub log_pass {
  53     print color('green');
  54     $log_line->('pass', @_);
  55     print color('reset');
  56 }
  57
  58 sub log_info {
  59     $log_line->('info', @_);
  60 }
  61 sub log_skip {
  62     $log_line->('skip', @_);
  63 }
  64 sub log_warn {
  65     print color('yellow');
  66     $log_line->('warn', @_);
  67     print color('reset');
  68 }
  69 sub log_fail {
  70     print color('red');
  71     $log_line->('fail', @_);
  72     print color('reset');
  73 }
  74
  75 my $print_header_first = 1;
  76 sub print_header {
  77     my ($h) = @_;
  78     print "\n" if !$print_header_first;
  79     print "= $h =\n\n";
  80     $print_header_first = 0;
  81 }
  82
  83 my $get_pkg = sub {
  84     my ($pkg) = @_;
  85
  86     my $versions = eval { PVE::API2::APT->versions({ node => $nodename }); };
  87
  88     if (!defined($versions)) {
  89         my $msg = "unable to retrieve package version information";
  90         $msg .= "- $@" if $@;
  91         log_fail("$msg");
  92         return undef;
  93     }
  94
  95     my $pkgs = [ grep { $_->{Package} eq $pkg } @$versions ];
  96     if (!defined $pkgs || $pkgs == 0) {
  97         log_fail("unable to determine installed $pkg version.");
  98         return undef;
  99     } else {
 100         return $pkgs->[0];
 101     }
 102 };
 103
 104 sub check_pve_packages {
 105     print_header("CHECKING VERSION INFORMATION FOR PVE PACKAGES");
 106
 107     print "Checking for package updates..\n";
 108     my $updates = eval { PVE::API2::APT->list_updates({ node => $nodename }); };
 109     if (!defined($updates)) {
 110         log_warn("$@") if $@;
 111         log_fail("unable to retrieve list of package updates!");
 112     } elsif (@$updates > 0) {
 113         my $pkgs = join(', ', map { $_->{Package} } @$updates);
 114         log_warn("updates for the following packages are available:\n  $pkgs");
 115     } else {
 116         log_pass("all packages uptodate");
 117     }
 118
 119     print "\nChecking proxmox-ve package version..\n";
 120     if (defined(my $proxmox_ve = $get_pkg->('proxmox-ve'))) {
 121         my $min_pve_ver = "$min_pve_major.$min_pve_minor-$min_pve_pkgrel";
 122
 123         my ($maj, $min, $pkgrel) = $proxmox_ve->{OldVersion} =~ m/^(\d+)\.(\d+)-(\d+)/;
 124
 125         if ($maj > $min_pve_major) {
 126             log_pass("already upgraded to Proxmox VE " . ($min_pve_major + 1));
 127         } elsif ($maj >= $min_pve_major && $min >= $min_pve_minor && $pkgrel >= $min_pve_pkgrel) {
 128             log_pass("proxmox-ve package has version >= $min_pve_ver");
 129         } else {
 130             log_fail("proxmox-ve package is too old, please upgrade to >= $min_pve_ver!");
 131         }
 132     }
 133 }
 134
 135 sub check_kvm_nested {
 136     my $module_sysdir = "/sys/module";
 137     if (-e "$module_sysdir/kvm_amd") {
 138         $module_sysdir .= "/kvm_amd/parameters";
 139     } elsif (-e "$module_sysdir/kvm_intel") {
 140         $module_sysdir .= "/kvm_intel/parameters";
 141     } else {
 142         log_skip("no kvm module found");
 143         return;
 144     }
 145
 146     if (-f "$module_sysdir/nested") {
 147         my $val = eval { PVE::Tools::file_read_firstline("$module_sysdir/nested") };
 148         if ($val && $val =~ m/Y|1/) {
 149             log_warn("KVM nested parameter set. VMs with vmx/svm flag will not be able to live migrate to PVE 6.");
 150         } else {
 151             log_pass("KVM nested parameter not set.")
 152         }
 153     } else {
 154         log_skip("KVM nested parameter not found.");
 155     }
 156 }
 157
 158 sub check_storage_health {
 159     print_header("CHECKING CONFIGURED STORAGES");
 160     my $cfg = PVE::Storage::config();
 161
 162     my $ctime = time();
 163
 164     my $info = PVE::Storage::storage_info($cfg);
 165
 166     foreach my $storeid (keys %$info) {
 167         my $d = $info->{$storeid};
 168         if ($d->{enabled}) {
 169             if ($d->{active}) {
 170                 log_pass("storage '$storeid' enabled and active.");
 171             } else {
 172                 log_warn("storage '$storeid' enabled but not active!");
 173             }
 174         } else {
 175             log_skip("storage '$storeid' disabled.");
 176         }
 177     }
 178 }
 179
 180 sub check_cluster_corosync {
 181     print_header("CHECKING CLUSTER HEALTH/SETTINGS");
 182
 183     if (!PVE::Corosync::check_conf_exists(1)) {
 184         log_skip("standalone node.");
 185         return;
 186     }
 187
 188     if (PVE::Cluster::check_cfs_quorum(1)) {
 189         log_pass("Cluster is quorate.");
 190     } else {
 191         log_fail("Cluster lost quorum!");
 192     }
 193
 194     my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
 195     my $conf_nodelist = PVE::Corosync::nodelist($conf);
 196
 197     if (!defined($conf_nodelist)) {
 198         log_fail("unable to retrieve nodelist from corosync.conf");
 199     } elsif (grep { $conf_nodelist->{$_}->{quorum_votes} != 1 } keys %$conf_nodelist) {
 200         log_warn("non-default quorum_votes distribution detected!");
 201     }
 202
 203     my $cfs_nodelist = PVE::Cluster::get_clinfo()->{nodelist};
 204     my $offline_nodes = grep { $cfs_nodelist->{$_}->{online} != 1 } keys %$cfs_nodelist;
 205     if ($offline_nodes > 0) {
 206         log_fail("$offline_nodes nodes are offline!");
 207     }
 208
 209     my $conf_nodelist_count = scalar(keys %$conf_nodelist);
 210     my $cfs_nodelist_count = scalar(keys %$cfs_nodelist);
 211     log_warn("cluster consists of less than three nodes!")
 212         if $conf_nodelist_count < 3;
 213
 214     log_fail("corosync.conf ($conf_nodelist_count) and pmxcfs ($cfs_nodelist_count) don't agree about size of nodelist.")
 215         if $conf_nodelist_count != $cfs_nodelist_count;
 216
 217     foreach my $cs_node (keys %$conf_nodelist) {
 218         my $entry = $conf_nodelist->{$cs_node};
 219         log_fail("No name entry for node '$cs_node' in corosync.conf.")
 220             if !defined($entry->{name});
 221         log_fail("No nodeid configured for node '$cs_node' in corosync.conf.")
 222             if !defined($entry->{nodeid});
 223
 224         my $verify_ring_ip = sub {
 225             my $key = shift;
 226             my $ring = $entry->{$key};
 227             if (defined($ring) && !PVE::JSONSchema::pve_verify_ip($ring, 1)) {
 228                 log_fail("$key '$ring' of node '$cs_node' is not an IP address, consider replacing it with the currently resolved IP address.");
 229             }
 230         };
 231         $verify_ring_ip->('ring0_addr');
 232         $verify_ring_ip->('ring1_addr');
 233     }
 234
 235     my $totem = $conf->{main}->{totem};
 236
 237     my $transport = $totem->{transport};
 238     if (defined($transport)) {
 239         log_fail("Corosync transport expliclitly set to '$transport' instead of implicit default!");
 240     }
 241
 242     if ((!defined($totem->{secauth}) || $totem->{secauth} ne 'on') && (!defined($totem->{crypto_cipher}) || $totem->{crypto_cipher} eq 'none')) {
 243         log_fail("Corosync authentication/encryption is not explicitly enabled (secauth / crypto_cipher / crypto_hash)!");
 244     }
 245
 246     if (defined($totem->{crypto_cipher}) && $totem->{crypto_cipher} eq '3des') {
 247         log_fail("Corosync encryption cipher set to '3des', no longer supported in Corosync 3.x!");
 248     }
 249
 250     my $prefix_info = sub { my $line = shift; log_info("$line"); };
 251     eval {
 252         print "\n";
 253         log_info("Printing detailed cluster status..");
 254         PVE::Tools::run_command(['corosync-quorumtool', '-siH'], outfunc => $prefix_info, errfunc => $prefix_info);
 255     };
 256
 257     print_header("CHECKING INSTALLED COROSYNC VERSION");
 258     if (defined(my $corosync = $get_pkg->('corosync'))) {
 259         if ($corosync->{OldVersion} =~ m/^2\./) {
 260             log_fail("corosync 2.x installed, cluster-wide upgrade to 3.x needed!");
 261         } elsif ($corosync->{OldVersion} =~ m/^3\./) {
 262             log_pass("corosync 3.x installed.");
 263         } else {
 264             log_fail("unexpected corosync version installed: $corosync->{OldVersion}!");
 265         }
 266     }
 267 }
 268
 269 sub check_ceph {
 270     print_header("CHECKING HYPER-CONVERGED CEPH STATUS");
 271
 272     if (PVE::Ceph::Tools::check_ceph_inited(1)) {
 273         log_info("hyper-converged ceph setup detected!");
 274     } else {
 275         log_skip("no hyper-converged ceph setup detected!");
 276         return;
 277     }
 278
 279     log_info("getting Ceph status/health information..");
 280     my $ceph_status = eval { PVE::API2::Ceph->status({ node => $nodename }); };
 281     my $osd_flags = eval { PVE::API2::Ceph->get_flags({ node => $nodename }); };
 282     my $noout;
 283     $noout = $osd_flags =~ m/noout/ if $osd_flags;
 284
 285     if (!$ceph_status || !$ceph_status->{health}) {
 286         log_fail("unable to determine Ceph status!");
 287     } else {
 288         my $ceph_health = $ceph_status->{health}->{status};
 289         if (!$ceph_health) {
 290             log_fail("unable to determine Ceph health!");
 291         } elsif ($ceph_health eq 'HEALTH_OK') {
 292             log_pass("Ceph health reported as 'HEALTH_OK'.");
 293         } elsif ($ceph_health eq 'HEALTH_WARN' && $noout && (keys %{$ceph_status->{health}->{checks}} == 1)) {
 294                 log_pass("Ceph health reported as 'HEALTH_WARN' with a single failing check and 'noout' flag set.");
 295         } else {
 296                 log_warn("Ceph health reported as '$ceph_health'");
 297         }
 298     }
 299
 300     log_info("getting Ceph OSD flags..");
 301     eval {
 302         if (!$osd_flags) {
 303             log_fail("unable to get Ceph OSD flags!");
 304         } else {
 305             if ($osd_flags =~ m/recovery_deletes/ && $osd_flags =~ m/purged_snapdirs/) {
 306                 log_pass("all PGs have been scrubbed at least once while running Ceph Luminous.");
 307             } else {
 308                 log_fail("missing 'recovery_deletes' and/or 'purged_snapdirs' flag, scrub of all PGs required before upgrading to Nautilus!");
 309             }
 310             if ($noout) {
 311                 log_pass("noout flag set to prevent rebalancing during cluster-wide upgrades.");
 312             }  else {
 313                 log_warn("noout flag not set - recommended to prevent rebalancing during upgrades.");
 314             }
 315         }
 316     };
 317
 318     log_info("getting Ceph daemon versions..");
 319     my $ceph_versions = eval { PVE::Ceph::Tools::get_cluster_versions(undef, 1); };
 320     if (!$ceph_versions) {
 321         log_fail("unable to determine Ceph daemon versions!");
 322     } else {
 323         my $services = [
 324             { 'key' => 'mon', 'name' => 'monitor' },
 325             { 'key' => 'mgr', 'name' => 'manager' },
 326             { 'key' => 'mds', 'name' => 'MDS' },
 327             { 'key' => 'osd', 'name' => 'OSD' },
 328         ];
 329
 330         foreach my $service (@$services) {
 331             my $name = $service->{name};
 332             if (my $service_versions = $ceph_versions->{$service->{key}}) {
 333                 if (keys %$service_versions == 0) {
 334                     log_skip("no running instances detected for daemon type $name.");
 335                 } elsif (keys %$service_versions == 1) {
 336                     log_pass("single running version detected for daemon type $name.");
 337                 } else {
 338                     log_warn("multiple running versions detected for daemon type $name!");
 339                 }
 340             } else {
 341                 log_skip("unable to determine versions of running Ceph $name instances.");
 342             }
 343         }
 344
 345         my $overall_versions = $ceph_versions->{overall};
 346         if (!$overall_versions) {
 347             log_warn("unable to determine overall Ceph daemon versions!");
 348         } elsif (keys %$overall_versions == 1) {
 349             log_pass("single running overall version detected for all Ceph daemon types.");
 350         } else {
 351             log_warn("overall version mismatch detected, check 'ceph versions' output for details!");
 352         }
 353     }
 354 }
 355
 356 sub check_misc {
 357     print_header("MISCELLANEOUS CHECKS");
 358     my $ssh_config = eval { PVE::Tools::file_get_contents('/root/.ssh/config') };
 359     if (defined($ssh_config)) {
 360         log_fail("Unsupported SSH Cipher configured for root in /root/.ssh/config: $1")
 361             if $ssh_config =~ /^Ciphers .*(blowfish|arcfour|3des).*$/m;
 362     } else {
 363         log_skip("No SSH config file found.");
 364     }
 365
 366     my $root_free = PVE::Tools::df('/', 10);
 367     log_warn("Less than 2G free space on root file system.")
 368         if defined($root_free) && $root_free->{avail} < 2*1024*1024*1024;
 369
 370     my $running_guests = 0;
 371     my $vms = eval { PVE::API2::Qemu->vmlist({ node => $nodename }) };
 372     log_warn("Failed to retrieve information about this node's VMs - $@") if $@;
 373     $running_guests += grep { $_->{status} eq 'running' } @$vms
 374         if defined($vms);
 375     my $cts = eval { PVE::API2::LXC->vmlist({ node => $nodename }) };
 376     log_warn("Failed to retrieve information about this node's CTs - $@") if $@;
 377     $running_guests += grep { $_->{status} eq 'running' } @$cts
 378         if defined($cts);
 379     log_warn("$running_guests running guests detected - consider migrating/stopping them.")
 380         if $running_guests > 0;
 381
 382     my $host = PVE::INotify::nodename();
 383     my $local_ip = eval { PVE::Network::get_ip_from_hostname($host) };
 384     if ($@) {
 385         log_warn("Failed to resolve hostname to IP - $@");
 386     } else {
 387         my $cidr = Net::IP::ip_is_ipv6($local_ip) ? "$local_ip/128" : "$local_ip/32";
 388         my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
 389         my $ip_count = scalar(@$configured_ips);
 390         log_warn("IP must be configured exactly once on local node - defined $ip_count times") if ($ip_count != 1);
 391     }
 392
 393     check_kvm_nested();
 394 }
 395
 396 __PACKAGE__->register_method ({
 397     name => 'checklist',
 398     path => 'checklist',
 399     method => 'GET',
 400     description => 'Check (pre-/post-)upgrade conditions.',
 401     parameters => {
 402         additionalProperties => 0,
 403         properties => {
 404         },
 405     },
 406     returns => { type => 'null' },
 407     code => sub {
 408         my ($param) = @_;
 409
 410         check_pve_packages();
 411         check_cluster_corosync();
 412         check_ceph();
 413         check_storage_health();
 414         check_misc();
 415
 416         print_header("SUMMARY");
 417
 418         my $total = 0;
 419         $total += $_ for values %$counters;
 420
 421         print "TOTAL:    $total\n";
 422         print colored("PASSED:   $counters->{pass}\n", 'green');
 423         print "SKIPPED:  $counters->{skip}\n";
 424         print colored("WARNINGS: $counters->{warn}\n", 'yellow');
 425         print colored("FAILURES: $counters->{fail}\n", 'red');
 426
 427         print colored("\nATTENTION: Please check the output for detailed information!\n", 'red')
 428             if ($counters->{warn} > 0 || $counters->{fail} > 0);
 429
 430         return undef;
 431     }});
 432
 433 our $cmddef = [ __PACKAGE__, 'checklist', [], {}];
 434
 435 # for now drop all unknown params and just check
 436 @ARGV = ();
 437
 438 1;