[pve-manager.git] / PVE / CLI / pve5to6.pm

package PVE::CLI::pve5to6;

use strict;
use warnings;

use PVE::API2::APT;
use PVE::API2::Ceph;
use PVE::API2::LXC;
use PVE::API2::Qemu;

use PVE::Ceph::Tools;
use PVE::Cluster;
use PVE::Corosync;
use PVE::INotify;
use PVE::JSONSchema;
use PVE::RPCEnvironment;
use PVE::Storage;
use PVE::Tools;
use PVE::QemuServer;

use Term::ANSIColor;

use PVE::CLIHandler;

use base qw(PVE::CLIHandler);

my $nodename = PVE::INotify::nodename();

sub setup_environment {
    PVE::RPCEnvironment->setup_default_cli_env();
}

my $min_pve_major = 5;
my $min_pve_minor = 4;
my $min_pve_pkgrel = 2;

my $counters = {
    pass => 0,
    skip => 0,
    warn => 0,
    fail => 0,
};

my $log_line = sub {
    my ($level, $line) = @_;

    $counters->{$level}++ if defined($level) && defined($counters->{$level});

    print uc($level), ': ' if defined($level);
    print "$line\n";
};

sub log_pass {
    print color('green');
    $log_line->('pass', @_);
    print color('reset');
}

sub log_info {
    $log_line->('info', @_);
}
sub log_skip {
    $log_line->('skip', @_);
}
sub log_warn {
    print color('yellow');
    $log_line->('warn', @_);
    print color('reset');
}
sub log_fail {
    print color('red');
    $log_line->('fail', @_);
    print color('reset');
}

my $print_header_first = 1;
sub print_header {
    my ($h) = @_;
    print "\n" if !$print_header_first;
    print "= $h =\n\n";
    $print_header_first = 0;
}

my $versions;
my $get_pkg = sub {
    my ($pkg) = @_;

    $versions = eval { PVE::API2::APT->versions({ node => $nodename }) } if !defined($versions);

    if (!defined($versions)) {
	my $msg = "unable to retrieve package version information";
	$msg .= "- $@" if $@;
	log_fail("$msg");
	return undef;
    }

    my $pkgs = [ grep { $_->{Package} eq $pkg } @$versions ];
    if (!defined $pkgs || $pkgs == 0) {
	log_fail("unable to determine installed $pkg version.");
	return undef;
    } else {
	return $pkgs->[0];
    }
};

sub check_pve_packages {
    print_header("CHECKING VERSION INFORMATION FOR PVE PACKAGES");

    print "Checking for package updates..\n";
    my $updates = eval { PVE::API2::APT->list_updates({ node => $nodename }); };
    if (!defined($updates)) {
	log_warn("$@") if $@;
	log_fail("unable to retrieve list of package updates!");
    } elsif (@$updates > 0) {
	my $pkgs = join(', ', map { $_->{Package} } @$updates);
	log_warn("updates for the following packages are available:\n  $pkgs");
    } else {
	log_pass("all packages uptodate");
    }

    print "\nChecking proxmox-ve package version..\n";
    if (defined(my $proxmox_ve = $get_pkg->('proxmox-ve'))) {
	my $min_pve_ver = "$min_pve_major.$min_pve_minor-$min_pve_pkgrel";

	my ($maj, $min, $pkgrel) = $proxmox_ve->{OldVersion} =~ m/^(\d+)\.(\d+)-(\d+)/;

	my $upgraded = 0;

	if ($maj > $min_pve_major) {
	    log_pass("already upgraded to Proxmox VE " . ($min_pve_major + 1));
	    $upgraded = 1;
	} elsif ($maj >= $min_pve_major && $min >= $min_pve_minor && $pkgrel >= $min_pve_pkgrel) {
	    log_pass("proxmox-ve package has version >= $min_pve_ver");
	} else {
	    log_fail("proxmox-ve package is too old, please upgrade to >= $min_pve_ver!");
	}

	my ($krunning, $kinstalled) = (qr/5\./, 'pve-kernel-5.0');
	if (!$upgraded) {
	    ($krunning, $kinstalled) = (qr/4\.15/, 'pve-kernel-4.15');
	}

	print "\nChecking running kernel version..\n";
	my $kernel_ver = $proxmox_ve->{RunningKernel};
	if (!defined($kernel_ver)) {
	    log_fail("unable to determine running kernel version.");
	} elsif ($kernel_ver =~ /^$krunning/) {
	    log_pass("expected running kernel '$kernel_ver'.");
	} elsif ($get_pkg->($kinstalled)) {
	    log_warn("expected kernel '$kinstalled' intalled but not yet rebooted!");
	} else {
	    log_warn("unexpected running and installed kernel '$kernel_ver'.");
	}
    }
}

sub get_vms_with_vmx {
    my $res = {
	cpu => [],
	flag => [],
    };
    my $vmlist = PVE::QemuServer::vzlist();

    foreach my $vmid ( sort { $a <=> $b } keys %$vmlist ) {
	my $pid = $vmlist->{$vmid}->{pid};
	next if !$pid; # skip not running vms

	my $cmdline = eval { PVE::Tools::file_get_contents("/proc/$pid/cmdline") };
	if ($cmdline) {
	    my @args = split(/\0/, $cmdline);
	    for (my $i = 0; $i < scalar(@args); $i++) {
		next if !$args[$i] || $args[$i] !~ m/^-?-cpu$/;

		my $cpuarg = $args[$i+1];
		if ($cpuarg =~ m/^(host|max)/) {
		    push @{$res->{cpu}}, $vmid;
		} elsif ($cpuarg =~ m/\+(vmx|svm)/) {
		    push @{$res->{flag}}, $vmid;
		}
	    }
	}
    }

    $res = undef if (scalar(@{$res->{cpu}}) + scalar(@{$res->{flag}})) <= 0;

    return $res;
}

sub check_kvm_nested {
    log_info("Checking KVM nesting support, which breaks live migration for VMs using it..");

    my $module_sysdir = "/sys/module";
    if (-e "$module_sysdir/kvm_amd") {
	$module_sysdir .= "/kvm_amd/parameters";
    } elsif (-e "$module_sysdir/kvm_intel") {
	$module_sysdir .= "/kvm_intel/parameters";
    } else {
	log_skip("no kvm module found");
	return;
    }

    if (-f "$module_sysdir/nested") {
	my $val = eval { PVE::Tools::file_read_firstline("$module_sysdir/nested") };
	if ($val && $val =~ m/Y|1/) {
	    my $list = get_vms_with_vmx();
	    if (!defined($list)) {
		log_pass("KVM nested parameter set, but currently no VM with a 'vmx' or 'svm' flag is running.");
	    } else {
		my $warnmsg = "KVM nested enabled. It will not be possible to live migrate the following running VMs to PVE 6:\n";
		if (@{$list->{cpu}}) {
		    $warnmsg .= "  VMID(s) with cputype 'host' or 'max': " . join(',', @{$list->{cpu}}) . "\n";
		}
		if (@{$list->{flag}}) {
		    $warnmsg .= "  VMID(s) with enforced cpu flag 'vmx' or 'svm': " . join(',', @{$list->{flag}}) . "\n";
		}
		log_warn($warnmsg);
	    }
	} else {
	    log_pass("KVM nested parameter not set.")
	}
    } else {
	log_skip("KVM nested parameter not found.");
    }
}

sub check_storage_health {
    print_header("CHECKING CONFIGURED STORAGES");
    my $cfg = PVE::Storage::config();

    my $ctime = time();

    my $info = PVE::Storage::storage_info($cfg);

    foreach my $storeid (keys %$info) {
	my $d = $info->{$storeid};
	if ($d->{enabled}) {
	    if ($d->{type} eq 'sheepdog') {
		log_fail("storage '$storeid' of type 'sheepdog' is enabled - experimental sheepdog support dropped in PVE 6")
	    } elsif ($d->{active}) {
		log_pass("storage '$storeid' enabled and active.");
	    } else {
		log_warn("storage '$storeid' enabled but not active!");
	    }
	} else {
	    log_skip("storage '$storeid' disabled.");
	}
    }
}

sub check_cluster_corosync {
    print_header("CHECKING CLUSTER HEALTH/SETTINGS");

    if (!PVE::Corosync::check_conf_exists(1)) {
	log_skip("standalone node.");
	return;
    }

    if (PVE::Cluster::check_cfs_quorum(1)) {
	log_pass("Cluster is quorate.");
    } else {
	log_fail("Cluster lost quorum!");
    }

    my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
    my $conf_nodelist = PVE::Corosync::nodelist($conf);

    if (!defined($conf_nodelist)) {
	log_fail("unable to retrieve nodelist from corosync.conf");
    } elsif (grep { $conf_nodelist->{$_}->{quorum_votes} != 1 } keys %$conf_nodelist) {
	log_warn("non-default quorum_votes distribution detected!");
    }

    my $cfs_nodelist = PVE::Cluster::get_clinfo()->{nodelist};
    my $offline_nodes = grep { $cfs_nodelist->{$_}->{online} != 1 } keys %$cfs_nodelist;
    if ($offline_nodes > 0) {
	log_fail("$offline_nodes nodes are offline!");
    }

    my $conf_nodelist_count = scalar(keys %$conf_nodelist);
    my $cfs_nodelist_count = scalar(keys %$cfs_nodelist);
    log_warn("cluster consists of less than three nodes!")
	if $conf_nodelist_count < 3;

    log_fail("corosync.conf ($conf_nodelist_count) and pmxcfs ($cfs_nodelist_count) don't agree about size of nodelist.")
	if $conf_nodelist_count != $cfs_nodelist_count;

    foreach my $cs_node (keys %$conf_nodelist) {
	my $entry = $conf_nodelist->{$cs_node};
	log_fail("No name entry for node '$cs_node' in corosync.conf.")
	    if !defined($entry->{name});
	log_fail("No nodeid configured for node '$cs_node' in corosync.conf.")
	    if !defined($entry->{nodeid});

	my $verify_ring_ip = sub {
	    my $key = shift;
	    my $ring = $entry->{$key};
	    if (defined($ring) && !PVE::JSONSchema::pve_verify_ip($ring, 1)) {
		log_fail("$key '$ring' of node '$cs_node' is not an IP address, consider replacing it with the currently resolved IP address.");
	    }
	};
	$verify_ring_ip->('ring0_addr');
	$verify_ring_ip->('ring1_addr');
    }

    my $totem = $conf->{main}->{totem};

    my $transport = $totem->{transport};
    if (defined($transport)) {
	log_fail("Corosync transport expliclitly set to '$transport' instead of implicit default!");
    }

    if ((!defined($totem->{secauth}) || $totem->{secauth} ne 'on') && (!defined($totem->{crypto_cipher}) || $totem->{crypto_cipher} eq 'none')) {
	log_fail("Corosync authentication/encryption is not explicitly enabled (secauth / crypto_cipher / crypto_hash)!");
    }

    if (defined($totem->{crypto_cipher}) && $totem->{crypto_cipher} eq '3des') {
	log_fail("Corosync encryption cipher set to '3des', no longer supported in Corosync 3.x!");
    }

    my $prefix_info = sub { my $line = shift; log_info("$line"); };
    eval {
	print "\n";
	log_info("Printing detailed cluster status..");
	PVE::Tools::run_command(['corosync-quorumtool', '-siH'], outfunc => $prefix_info, errfunc => $prefix_info);
    };

    print_header("CHECKING INSTALLED COROSYNC VERSION");
    if (defined(my $corosync = $get_pkg->('corosync'))) {
	if ($corosync->{OldVersion} =~ m/^2\./) {
	    log_fail("corosync 2.x installed, cluster-wide upgrade to 3.x needed!");
	} elsif ($corosync->{OldVersion} =~ m/^3\./) {
	    log_pass("corosync 3.x installed.");
	} else {
	    log_fail("unexpected corosync version installed: $corosync->{OldVersion}!");
	}
    }
}

sub check_ceph {
    print_header("CHECKING HYPER-CONVERGED CEPH STATUS");

    if (PVE::Ceph::Tools::check_ceph_inited(1)) {
	log_info("hyper-converged ceph setup detected!");
    } else {
	log_skip("no hyper-converged ceph setup detected!");
	return;
    }

    log_info("getting Ceph status/health information..");
    my $ceph_status = eval { PVE::API2::Ceph->status({ node => $nodename }); };
    my $osd_flags = eval { PVE::API2::Ceph->get_flags({ node => $nodename }); };
    my $noout_wanted = 1;
    my $noout = $osd_flags =~ m/noout/ if $osd_flags;

    if (!$ceph_status || !$ceph_status->{health}) {
	log_fail("unable to determine Ceph status!");
    } else {
	my $ceph_health = $ceph_status->{health}->{status};
	if (!$ceph_health) {
	    log_fail("unable to determine Ceph health!");
	} elsif ($ceph_health eq 'HEALTH_OK') {
	    log_pass("Ceph health reported as 'HEALTH_OK'.");
	} elsif ($ceph_health eq 'HEALTH_WARN' && $noout && (keys %{$ceph_status->{health}->{checks}} == 1)) {
		log_pass("Ceph health reported as 'HEALTH_WARN' with a single failing check and 'noout' flag set.");
	} else {
		log_warn("Ceph health reported as '$ceph_health'.\n      Use the PVE ".
		  "dashboard or 'ceph -s' to determine the specific issues and try to resolve them.");
	}
    }

    log_info("getting Ceph OSD flags..");
    eval {
	if (!$osd_flags) {
	    log_fail("unable to get Ceph OSD flags!");
	} else {
	    if ($osd_flags =~ m/recovery_deletes/ && $osd_flags =~ m/purged_snapdirs/) {
		log_pass("all PGs have been scrubbed at least once while running Ceph Luminous.");
	    } else {
		log_fail("missing 'recovery_deletes' and/or 'purged_snapdirs' flag, scrub of all PGs required before upgrading to Nautilus!");
	    }
	}
    };

    log_info("getting Ceph daemon versions..");
    my $ceph_versions = eval { PVE::Ceph::Tools::get_cluster_versions(undef, 1); };
    if (!$ceph_versions) {
	log_fail("unable to determine Ceph daemon versions!");
    } else {
	my $services = [
	    { 'key' => 'mon', 'name' => 'monitor' },
	    { 'key' => 'mgr', 'name' => 'manager' },
	    { 'key' => 'mds', 'name' => 'MDS' },
	    { 'key' => 'osd', 'name' => 'OSD' },
	];

	foreach my $service (@$services) {
	    my $name = $service->{name};
	    if (my $service_versions = $ceph_versions->{$service->{key}}) {
		if (keys %$service_versions == 0) {
		    log_skip("no running instances detected for daemon type $name.");
		} elsif (keys %$service_versions == 1) {
		    log_pass("single running version detected for daemon type $name.");
		} else {
		    log_warn("multiple running versions detected for daemon type $name!");
		}
	    } else {
		log_skip("unable to determine versions of running Ceph $name instances.");
	    }
	}

	my $overall_versions = $ceph_versions->{overall};
	if (!$overall_versions) {
	    log_warn("unable to determine overall Ceph daemon versions!");
	} elsif (keys %$overall_versions == 1) {
	    log_pass("single running overall version detected for all Ceph daemon types.");
	    if ((keys %$overall_versions)[0] =~ /^ceph version 14\./) {
		$noout_wanted = 0;
	    }
	} else {
	    log_warn("overall version mismatch detected, check 'ceph versions' output for details!");
	}
    }

    if ($noout) {
	if ($noout_wanted) {
	    log_pass("noout flag set to prevent rebalancing during cluster-wide upgrades.");
	} else {
	    log_warn("noout flag set, Ceph cluster upgrade seems finished.");
	}
    } elsif ($noout_wanted) {
	log_warn("noout flag not set - recommended to prevent rebalancing during upgrades.");
    }

    log_info("checking Ceph config..");
    my $conf = PVE::Cluster::cfs_read_file('ceph.conf');
    if (%$conf) {
	my $global = $conf->{global};

	my $global_monhost = $global->{mon_host} // $global->{"mon host"} // $global->{"mon-host"};
	if (!defined($global_monhost)) {
	    log_warn("No 'mon_host' entry found in ceph config.\n  It's recommended to add mon_host with all monitor addresses (without ports) to the global section.");
	} else {
	    log_pass("Found mon_host entry.");
	}

	my $ipv6 = $global->{ms_bind_ipv6} // $global->{"ms bind ipv6"} // $global->{"ms-bind-ipv6"};
	if ($ipv6) {
	    my $ipv4 = $global->{ms_bind_ipv4} // $global->{"ms bind ipv4"} // $global->{"ms-bind-ipv4"};
	    if ($ipv6 eq 'true' && (!defined($ipv4) || $ipv4 ne 'false')) {
		log_warn("ms_bind_ipv6 is enabled but ms_bind_ipv4 is not disabled.\n  Make sure to disable ms_bind_ipv4 for ipv6 only clusters, or add an ipv4 network to public/cluster network.");
	    } else {
		log_pass("ms_bind_ipv6 is enabled and ms_bind_ipv4 disabled");
	    }
	} else {
	    log_pass("ms_bind_ipv6 not enabled");
	}
    } else {
	log_warn("Empty ceph config found");
    }

    my $local_ceph_ver = PVE::Ceph::Tools::get_local_version(1);
    if (defined($local_ceph_ver)) {
	if ($local_ceph_ver == 14) {
	    my $scanned_osds = PVE::Tools::dir_glob_regex('/etc/ceph/osd', '^.*\.json$');
	    if (-e '/var/lib/ceph/osd/' && !defined($scanned_osds)) {
		log_warn("local Ceph version is Nautilus, local OSDs detected, but no conversion from ceph-disk to ceph-volume done (yet).");
	    }
	}
    } else {
	log_fail("unable to determine local Ceph version.");
    }
}

sub check_misc {
    print_header("MISCELLANEOUS CHECKS");
    my $ssh_config = eval { PVE::Tools::file_get_contents('/root/.ssh/config') };
    if (defined($ssh_config)) {
	log_fail("Unsupported SSH Cipher configured for root in /root/.ssh/config: $1")
	    if $ssh_config =~ /^Ciphers .*(blowfish|arcfour|3des).*$/m;
    } else {
	log_skip("No SSH config file found.");
    }

    my $root_free = PVE::Tools::df('/', 10);
    log_warn("Less than 2G free space on root file system.")
	if defined($root_free) && $root_free->{avail} < 2*1024*1024*1024;

    log_info("Checking for running guests..");
    my $running_guests = 0;

    my $vms = eval { PVE::API2::Qemu->vmlist({ node => $nodename }) };
    log_warn("Failed to retrieve information about this node's VMs - $@") if $@;
    $running_guests += grep { $_->{status} eq 'running' } @$vms if defined($vms);

    my $cts = eval { PVE::API2::LXC->vmlist({ node => $nodename }) };
    log_warn("Failed to retrieve information about this node's CTs - $@") if $@;
    $running_guests += grep { $_->{status} eq 'running' } @$cts if defined($cts);

    if ($running_guests > 0) {
	log_warn("$running_guests running guest(s) detected - consider migrating or stopping them.")
    } else {
	log_pass("no running guest detected.")
    }

    log_info("Checking if the local node's hostname '$nodename' is resolvable..");
    my $local_ip = eval { PVE::Network::get_ip_from_hostname($nodename) };
    if ($@) {
	log_warn("Failed to resolve hostname '$nodename' to IP - $@");
    } else {
	log_info("Checking if resolved IP is configured on local node..");
	my $cidr = Net::IP::ip_is_ipv6($local_ip) ? "$local_ip/128" : "$local_ip/32";
	my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
	my $ip_count = scalar(@$configured_ips);

	if ($ip_count <= 0) {
	    log_fail("Resolved node IP '$local_ip' not configured or active for '$nodename'");
	} elsif ($ip_count > 1) {
	    log_warn("Resolved node IP '$local_ip' active on multiple ($ip_count) interfaces!");
	} else {
	    log_pass("Resolved node IP '$local_ip' configured and active on single interface.");
	}
    }

    check_kvm_nested();
}

__PACKAGE__->register_method ({
    name => 'checklist',
    path => 'checklist',
    method => 'GET',
    description => 'Check (pre-/post-)upgrade conditions.',
    parameters => {
	additionalProperties => 0,
	properties => {
	},
    },
    returns => { type => 'null' },
    code => sub {
	my ($param) = @_;

	check_pve_packages();
	check_cluster_corosync();
	check_ceph();
	check_storage_health();
	check_misc();

	print_header("SUMMARY");

	my $total = 0;
	$total += $_ for values %$counters;

	print "TOTAL:    $total\n";
	print colored("PASSED:   $counters->{pass}\n", 'green');
	print "SKIPPED:  $counters->{skip}\n";
	print colored("WARNINGS: $counters->{warn}\n", 'yellow');
	print colored("FAILURES: $counters->{fail}\n", 'red');

	print colored("\nATTENTION: Please check the output for detailed information!\n", 'red')
	    if ($counters->{warn} > 0 || $counters->{fail} > 0);

	return undef;
    }});

our $cmddef = [ __PACKAGE__, 'checklist', [], {}];

# for now drop all unknown params and just check
@ARGV = ();

1;
Commit	Line	Data
75584009 FG	1	package PVE::CLI::pve5to6;
	2
	3	use strict;
	4	use warnings;
	5
	6	use PVE::API2::APT;
	7	use PVE::API2::Ceph;
	8	use PVE::API2::LXC;
	9	use PVE::API2::Qemu;
	10
	11	use PVE::Ceph::Tools;
	12	use PVE::Cluster;
	13	use PVE::Corosync;
	14	use PVE::INotify;
	15	use PVE::JSONSchema;
	16	use PVE::RPCEnvironment;
	17	use PVE::Storage;
	18	use PVE::Tools;
03f79391	19	use PVE::QemuServer;
75584009	20
6acde780 FG	21	use Term::ANSIColor;
6acde780 FG	22
75584009 FG	23	use PVE::CLIHandler;
	24
	25	use base qw(PVE::CLIHandler);
	26
	27	my $nodename = PVE::INotify::nodename();
	28
	29	sub setup_environment {
	30	PVE::RPCEnvironment->setup_default_cli_env();
	31	}
	32
f31c3e4a TL	33	my $min_pve_major = 5;
f31c3e4a TL	34	my $min_pve_minor = 4;
75584009 FG	35	my $min_pve_pkgrel = 2;
	36
	37	my $counters = {
	38	pass => 0,
	39	skip => 0,
	40	warn => 0,
	41	fail => 0,
	42	};
	43
	44	my $log_line = sub {
	45	my ($level, $line) = @_;
	46
	47	$counters->{$level}++ if defined($level) && defined($counters->{$level});
	48
	49	print uc($level), ': ' if defined($level);
	50	print "$line\n";
	51	};
	52
	53	sub log_pass {
6acde780	54	print color('green');
75584009	55	$log_line->('pass', @_);
6acde780	56	print color('reset');
75584009 FG	57	}
	58
	59	sub log_info {
	60	$log_line->('info', @_);
	61	}
	62	sub log_skip {
	63	$log_line->('skip', @_);
	64	}
	65	sub log_warn {
6acde780	66	print color('yellow');
75584009	67	$log_line->('warn', @_);
6acde780	68	print color('reset');
75584009 FG	69	}
75584009 FG	70	sub log_fail {
6acde780	71	print color('red');
75584009	72	$log_line->('fail', @_);
6acde780	73	print color('reset');
75584009 FG	74	}
75584009 FG	75
465b3ea2 TL	76	my $print_header_first = 1;
	77	sub print_header {
	78	my ($h) = @_;
	79	print "\n" if !$print_header_first;
	80	print "= $h =\n\n";
	81	$print_header_first = 0;
	82	}
	83
7d1b353b	84	my $versions;
75584009 FG	85	my $get_pkg = sub {
	86	my ($pkg) = @_;
	87
7d1b353b	88	$versions = eval { PVE::API2::APT->versions({ node => $nodename }) } if !defined($versions);
75584009 FG	89
	90	if (!defined($versions)) {
	91	my $msg = "unable to retrieve package version information";
	92	$msg .= "- $@" if $@;
	93	log_fail("$msg");
	94	return undef;
	95	}
	96
	97	my $pkgs = [ grep { $_->{Package} eq $pkg } @$versions ];
	98	if (!defined $pkgs \|\| $pkgs == 0) {
	99	log_fail("unable to determine installed $pkg version.");
	100	return undef;
	101	} else {
	102	return $pkgs->[0];
	103	}
	104	};
	105
	106	sub check_pve_packages {
465b3ea2	107	print_header("CHECKING VERSION INFORMATION FOR PVE PACKAGES");
75584009	108
465b3ea2	109	print "Checking for package updates..\n";
75584009 FG	110	my $updates = eval { PVE::API2::APT->list_updates({ node => $nodename }); };
	111	if (!defined($updates)) {
	112	log_warn("$@") if $@;
	113	log_fail("unable to retrieve list of package updates!");
	114	} elsif (@$updates > 0) {
	115	my $pkgs = join(', ', map { $_->{Package} } @$updates);
079d1188	116	log_warn("updates for the following packages are available:\n $pkgs");
75584009 FG	117	} else {
	118	log_pass("all packages uptodate");
	119	}
	120
	121	print "\nChecking proxmox-ve package version..\n";
	122	if (defined(my $proxmox_ve = $get_pkg->('proxmox-ve'))) {
f31c3e4a	123	my $min_pve_ver = "$min_pve_major.$min_pve_minor-$min_pve_pkgrel";
75584009	124
f31c3e4a TL	125	my ($maj, $min, $pkgrel) = $proxmox_ve->{OldVersion} =~ m/^(\d+)\.(\d+)-(\d+)/;
f31c3e4a TL	126
f034380d TL	127	my $upgraded = 0;
f034380d TL	128
f31c3e4a TL	129	if ($maj > $min_pve_major) {
f31c3e4a TL	130	log_pass("already upgraded to Proxmox VE " . ($min_pve_major + 1));
f034380d	131	$upgraded = 1;
f31c3e4a	132	} elsif ($maj >= $min_pve_major && $min >= $min_pve_minor && $pkgrel >= $min_pve_pkgrel) {
75584009 FG	133	log_pass("proxmox-ve package has version >= $min_pve_ver");
	134	} else {
	135	log_fail("proxmox-ve package is too old, please upgrade to >= $min_pve_ver!");
	136	}
f034380d TL	137
	138	my ($krunning, $kinstalled) = (qr/5\./, 'pve-kernel-5.0');
	139	if (!$upgraded) {
	140	($krunning, $kinstalled) = (qr/4\.15/, 'pve-kernel-4.15');
	141	}
	142
f8da1299 FG	143	print "\nChecking running kernel version..\n";
	144	my $kernel_ver = $proxmox_ve->{RunningKernel};
	145	if (!defined($kernel_ver)) {
	146	log_fail("unable to determine running kernel version.");
f034380d	147	} elsif ($kernel_ver =~ /^$krunning/) {
f8da1299	148	log_pass("expected running kernel '$kernel_ver'.");
f034380d TL	149	} elsif ($get_pkg->($kinstalled)) {
f034380d TL	150	log_warn("expected kernel '$kinstalled' intalled but not yet rebooted!");
f8da1299	151	} else {
f034380d	152	log_warn("unexpected running and installed kernel '$kernel_ver'.");
f8da1299	153	}
75584009 FG	154	}
	155	}
	156
03f79391 DC	157	sub get_vms_with_vmx {
	158	my $res = {
	159	cpu => [],
	160	flag => [],
	161	};
	162	my $vmlist = PVE::QemuServer::vzlist();
000acaca	163
03f79391 DC	164	foreach my $vmid ( sort { $a <=> $b } keys %$vmlist ) {
	165	my $pid = $vmlist->{$vmid}->{pid};
	166	next if !$pid; # skip not running vms
000acaca	167
03f79391 DC	168	my $cmdline = eval { PVE::Tools::file_get_contents("/proc/$pid/cmdline") };
	169	if ($cmdline) {
	170	my @args = split(/\0/, $cmdline);
	171	for (my $i = 0; $i < scalar(@args); $i++) {
	172	next if !$args[$i] \|\| $args[$i] !~ m/^-?-cpu$/;
000acaca	173
03f79391 DC	174	my $cpuarg = $args[$i+1];
	175	if ($cpuarg =~ m/^(host\|max)/) {
	176	push @{$res->{cpu}}, $vmid;
	177	} elsif ($cpuarg =~ m/\+(vmx\|svm)/) {
	178	push @{$res->{flag}}, $vmid;
	179	}
	180	}
	181	}
	182	}
000acaca TL	183
	184	$res = undef if (scalar(@{$res->{cpu}}) + scalar(@{$res->{flag}})) <= 0;
	185
	186	return $res;
03f79391 DC	187	}
03f79391 DC	188
eb8bf430	189	sub check_kvm_nested {
000acaca TL	190	log_info("Checking KVM nesting support, which breaks live migration for VMs using it..");
000acaca TL	191
eb8bf430 DC	192	my $module_sysdir = "/sys/module";
	193	if (-e "$module_sysdir/kvm_amd") {
	194	$module_sysdir .= "/kvm_amd/parameters";
	195	} elsif (-e "$module_sysdir/kvm_intel") {
	196	$module_sysdir .= "/kvm_intel/parameters";
	197	} else {
	198	log_skip("no kvm module found");
	199	return;
	200	}
	201
	202	if (-f "$module_sysdir/nested") {
	203	my $val = eval { PVE::Tools::file_read_firstline("$module_sysdir/nested") };
	204	if ($val && $val =~ m/Y\|1/) {
000acaca TL	205	my $list = get_vms_with_vmx();
	206	if (!defined($list)) {
	207	log_pass("KVM nested parameter set, but currently no VM with a 'vmx' or 'svm' flag is running.");
03f79391	208	} else {
000acaca	209	my $warnmsg = "KVM nested enabled. It will not be possible to live migrate the following running VMs to PVE 6:\n";
03f79391	210	if (@{$list->{cpu}}) {
000acaca	211	$warnmsg .= " VMID(s) with cputype 'host' or 'max': " . join(',', @{$list->{cpu}}) . "\n";
03f79391	212	}
03f79391	213	if (@{$list->{flag}}) {
000acaca	214	$warnmsg .= " VMID(s) with enforced cpu flag 'vmx' or 'svm': " . join(',', @{$list->{flag}}) . "\n";
03f79391	215	}
03f79391 DC	216	log_warn($warnmsg);
03f79391 DC	217	}
eb8bf430 DC	218	} else {
	219	log_pass("KVM nested parameter not set.")
	220	}
	221	} else {
	222	log_skip("KVM nested parameter not found.");
	223	}
	224	}
	225
75584009	226	sub check_storage_health {
465b3ea2	227	print_header("CHECKING CONFIGURED STORAGES");
75584009 FG	228	my $cfg = PVE::Storage::config();
	229
	230	my $ctime = time();
	231
	232	my $info = PVE::Storage::storage_info($cfg);
	233
	234	foreach my $storeid (keys %$info) {
	235	my $d = $info->{$storeid};
	236	if ($d->{enabled}) {
68f0d161	237	if ($d->{type} eq 'sheepdog') {
b8bdb17e	238	log_fail("storage '$storeid' of type 'sheepdog' is enabled - experimental sheepdog support dropped in PVE 6")
68f0d161	239	} elsif ($d->{active}) {
75584009 FG	240	log_pass("storage '$storeid' enabled and active.");
	241	} else {
	242	log_warn("storage '$storeid' enabled but not active!");
	243	}
	244	} else {
	245	log_skip("storage '$storeid' disabled.");
	246	}
	247	}
	248	}
	249
	250	sub check_cluster_corosync {
465b3ea2	251	print_header("CHECKING CLUSTER HEALTH/SETTINGS");
75584009 FG	252
	253	if (!PVE::Corosync::check_conf_exists(1)) {
	254	log_skip("standalone node.");
	255	return;
	256	}
	257
	258	if (PVE::Cluster::check_cfs_quorum(1)) {
	259	log_pass("Cluster is quorate.");
	260	} else {
	261	log_fail("Cluster lost quorum!");
	262	}
	263
	264	my $conf = PVE::Cluster::cfs_read_file('corosync.conf');
	265	my $conf_nodelist = PVE::Corosync::nodelist($conf);
	266
	267	if (!defined($conf_nodelist)) {
	268	log_fail("unable to retrieve nodelist from corosync.conf");
	269	} elsif (grep { $conf_nodelist->{$_}->{quorum_votes} != 1 } keys %$conf_nodelist) {
	270	log_warn("non-default quorum_votes distribution detected!");
	271	}
	272
	273	my $cfs_nodelist = PVE::Cluster::get_clinfo()->{nodelist};
	274	my $offline_nodes = grep { $cfs_nodelist->{$_}->{online} != 1 } keys %$cfs_nodelist;
	275	if ($offline_nodes > 0) {
	276	log_fail("$offline_nodes nodes are offline!");
	277	}
	278
	279	my $conf_nodelist_count = scalar(keys %$conf_nodelist);
	280	my $cfs_nodelist_count = scalar(keys %$cfs_nodelist);
	281	log_warn("cluster consists of less than three nodes!")
	282	if $conf_nodelist_count < 3;
	283
	284	log_fail("corosync.conf ($conf_nodelist_count) and pmxcfs ($cfs_nodelist_count) don't agree about size of nodelist.")
	285	if $conf_nodelist_count != $cfs_nodelist_count;
	286
	287	foreach my $cs_node (keys %$conf_nodelist) {
	288	my $entry = $conf_nodelist->{$cs_node};
	289	log_fail("No name entry for node '$cs_node' in corosync.conf.")
	290	if !defined($entry->{name});
	291	log_fail("No nodeid configured for node '$cs_node' in corosync.conf.")
	292	if !defined($entry->{nodeid});
	293
	294	my $verify_ring_ip = sub {
	295	my $key = shift;
	296	my $ring = $entry->{$key};
	297	if (defined($ring) && !PVE::JSONSchema::pve_verify_ip($ring, 1)) {
	298	log_fail("$key '$ring' of node '$cs_node' is not an IP address, consider replacing it with the currently resolved IP address.");
	299	}
	300	};
	301	$verify_ring_ip->('ring0_addr');
	302	$verify_ring_ip->('ring1_addr');
	303	}
	304
	305	my $totem = $conf->{main}->{totem};
	306
	307	my $transport = $totem->{transport};
	308	if (defined($transport)) {
	309	log_fail("Corosync transport expliclitly set to '$transport' instead of implicit default!");
	310	}
	311
	312	if ((!defined($totem->{secauth}) \|\| $totem->{secauth} ne 'on') && (!defined($totem->{crypto_cipher}) \|\| $totem->{crypto_cipher} eq 'none')) {
	313	log_fail("Corosync authentication/encryption is not explicitly enabled (secauth / crypto_cipher / crypto_hash)!");
	314	}
	315
316	if (defined($totem->{crypto_cipher}) && $totem->{crypto_cipher} eq '3des') {
317	log_fail("Corosync encryption cipher set to '3des', no longer supported in Corosync 3.x!");
318	}
319
320	my $prefix_info = sub { my $line = shift; log_info("$line"); };
321	eval {
322	print "\n";
323	log_info("Printing detailed cluster status..");
324	PVE::Tools::run_command(['corosync-quorumtool', '-siH'], outfunc => $prefix_info, errfunc => $prefix_info);
325	};
326
465b3ea2	327	print_header("CHECKING INSTALLED COROSYNC VERSION");
75584009 FG	328	if (defined(my $corosync = $get_pkg->('corosync'))) {
	329	if ($corosync->{OldVersion} =~ m/^2\./) {
	330	log_fail("corosync 2.x installed, cluster-wide upgrade to 3.x needed!");
	331	} elsif ($corosync->{OldVersion} =~ m/^3\./) {
	332	log_pass("corosync 3.x installed.");
	333	} else {
	334	log_fail("unexpected corosync version installed: $corosync->{OldVersion}!");
	335	}
	336	}
	337	}
	338
	339	sub check_ceph {
465b3ea2	340	print_header("CHECKING HYPER-CONVERGED CEPH STATUS");
75584009 FG	341
	342	if (PVE::Ceph::Tools::check_ceph_inited(1)) {
	343	log_info("hyper-converged ceph setup detected!");
	344	} else {
	345	log_skip("no hyper-converged ceph setup detected!");
	346	return;
	347	}
	348
	349	log_info("getting Ceph status/health information..");
	350	my $ceph_status = eval { PVE::API2::Ceph->status({ node => $nodename }); };
	351	my $osd_flags = eval { PVE::API2::Ceph->get_flags({ node => $nodename }); };
a1687fd9 FG	352	my $noout_wanted = 1;
a1687fd9 FG	353	my $noout = $osd_flags =~ m/noout/ if $osd_flags;
75584009 FG	354
	355	if (!$ceph_status \|\| !$ceph_status->{health}) {
	356	log_fail("unable to determine Ceph status!");
	357	} else {
	358	my $ceph_health = $ceph_status->{health}->{status};
	359	if (!$ceph_health) {
	360	log_fail("unable to determine Ceph health!");
	361	} elsif ($ceph_health eq 'HEALTH_OK') {
	362	log_pass("Ceph health reported as 'HEALTH_OK'.");
	363	} elsif ($ceph_health eq 'HEALTH_WARN' && $noout && (keys %{$ceph_status->{health}->{checks}} == 1)) {
	364	log_pass("Ceph health reported as 'HEALTH_WARN' with a single failing check and 'noout' flag set.");
	365	} else {
6deabaac TL	366	log_warn("Ceph health reported as '$ceph_health'.\n Use the PVE ".
6deabaac TL	367	"dashboard or 'ceph -s' to determine the specific issues and try to resolve them.");
75584009 FG	368	}
	369	}
	370
	371	log_info("getting Ceph OSD flags..");
	372	eval {
	373	if (!$osd_flags) {
	374	log_fail("unable to get Ceph OSD flags!");
	375	} else {
	376	if ($osd_flags =~ m/recovery_deletes/ && $osd_flags =~ m/purged_snapdirs/) {
	377	log_pass("all PGs have been scrubbed at least once while running Ceph Luminous.");
	378	} else {
	379	log_fail("missing 'recovery_deletes' and/or 'purged_snapdirs' flag, scrub of all PGs required before upgrading to Nautilus!");
	380	}
75584009 FG	381	}
	382	};
	383
	384	log_info("getting Ceph daemon versions..");
	385	my $ceph_versions = eval { PVE::Ceph::Tools::get_cluster_versions(undef, 1); };
	386	if (!$ceph_versions) {
	387	log_fail("unable to determine Ceph daemon versions!");
	388	} else {
	389	my $services = [
	390	{ 'key' => 'mon', 'name' => 'monitor' },
	391	{ 'key' => 'mgr', 'name' => 'manager' },
	392	{ 'key' => 'mds', 'name' => 'MDS' },
	393	{ 'key' => 'osd', 'name' => 'OSD' },
	394	];
	395
	396	foreach my $service (@$services) {
	397	my $name = $service->{name};
	398	if (my $service_versions = $ceph_versions->{$service->{key}}) {
	399	if (keys %$service_versions == 0) {
	400	log_skip("no running instances detected for daemon type $name.");
	401	} elsif (keys %$service_versions == 1) {
	402	log_pass("single running version detected for daemon type $name.");
	403	} else {
	404	log_warn("multiple running versions detected for daemon type $name!");
	405	}
	406	} else {
	407	log_skip("unable to determine versions of running Ceph $name instances.");
	408	}
	409	}
	410
	411	my $overall_versions = $ceph_versions->{overall};
	412	if (!$overall_versions) {
	413	log_warn("unable to determine overall Ceph daemon versions!");
	414	} elsif (keys %$overall_versions == 1) {
	415	log_pass("single running overall version detected for all Ceph daemon types.");
a1687fd9 FG	416	if ((keys %$overall_versions)[0] =~ /^ceph version 14\./) {
	417	$noout_wanted = 0;
	418	}
75584009 FG	419	} else {
	420	log_warn("overall version mismatch detected, check 'ceph versions' output for details!");
	421	}
	422	}
32526d27	423
a1687fd9 FG	424	if ($noout) {
	425	if ($noout_wanted) {
	426	log_pass("noout flag set to prevent rebalancing during cluster-wide upgrades.");
	427	} else {
	428	log_warn("noout flag set, Ceph cluster upgrade seems finished.");
	429	}
	430	} elsif ($noout_wanted) {
	431	log_warn("noout flag not set - recommended to prevent rebalancing during upgrades.");
	432	}
	433
c553da92 DC	434	log_info("checking Ceph config..");
c553da92 DC	435	my $conf = PVE::Cluster::cfs_read_file('ceph.conf');
f9f8cd7d	436	if (%$conf) {
c553da92	437	my $global = $conf->{global};
780a57b6 TL	438
	439	my $global_monhost = $global->{mon_host} // $global->{"mon host"} // $global->{"mon-host"};
	440	if (!defined($global_monhost)) {
	441	log_warn("No 'mon_host' entry found in ceph config.\n It's recommended to add mon_host with all monitor addresses (without ports) to the global section.");
c553da92 DC	442	} else {
	443	log_pass("Found mon_host entry.");
	444	}
38bd1ffc	445
60b81f16 TL	446	my $ipv6 = $global->{ms_bind_ipv6} // $global->{"ms bind ipv6"} // $global->{"ms-bind-ipv6"};
	447	if ($ipv6) {
	448	my $ipv4 = $global->{ms_bind_ipv4} // $global->{"ms bind ipv4"} // $global->{"ms-bind-ipv4"};
38bd1ffc DC	449	if ($ipv6 eq 'true' && (!defined($ipv4) \|\| $ipv4 ne 'false')) {
	450	log_warn("ms_bind_ipv6 is enabled but ms_bind_ipv4 is not disabled.\n Make sure to disable ms_bind_ipv4 for ipv6 only clusters, or add an ipv4 network to public/cluster network.");
	451	} else {
	452	log_pass("ms_bind_ipv6 is enabled and ms_bind_ipv4 disabled");
	453	}
	454	} else {
	455	log_pass("ms_bind_ipv6 not enabled");
	456	}
c553da92	457	} else {
f9f8cd7d	458	log_warn("Empty ceph config found");
c553da92 DC	459	}
c553da92 DC	460
32526d27 FG	461	my $local_ceph_ver = PVE::Ceph::Tools::get_local_version(1);
	462	if (defined($local_ceph_ver)) {
	463	if ($local_ceph_ver == 14) {
	464	my $scanned_osds = PVE::Tools::dir_glob_regex('/etc/ceph/osd', '^.*\.json$');
	465	if (-e '/var/lib/ceph/osd/' && !defined($scanned_osds)) {
	466	log_warn("local Ceph version is Nautilus, local OSDs detected, but no conversion from ceph-disk to ceph-volume done (yet).");
	467	}
	468	}
	469	} else {
	470	log_fail("unable to determine local Ceph version.");
	471	}
75584009 FG	472	}
	473
	474	sub check_misc {
465b3ea2	475	print_header("MISCELLANEOUS CHECKS");
75584009	476	my $ssh_config = eval { PVE::Tools::file_get_contents('/root/.ssh/config') };
27228998 FG	477	if (defined($ssh_config)) {
	478	log_fail("Unsupported SSH Cipher configured for root in /root/.ssh/config: $1")
	479	if $ssh_config =~ /^Ciphers .(blowfish\|arcfour\|3des).$/m;
	480	} else {
	481	log_skip("No SSH config file found.");
	482	}
75584009 FG	483
	484	my $root_free = PVE::Tools::df('/', 10);
	485	log_warn("Less than 2G free space on root file system.")
	486	if defined($root_free) && $root_free->{avail} < 210241024*1024;
	487
b6c10c63	488	log_info("Checking for running guests..");
75584009	489	my $running_guests = 0;
b6c10c63	490
75584009 FG	491	my $vms = eval { PVE::API2::Qemu->vmlist({ node => $nodename }) };
75584009 FG	492	log_warn("Failed to retrieve information about this node's VMs - $@") if $@;
b6c10c63 TL	493	$running_guests += grep { $_->{status} eq 'running' } @$vms if defined($vms);
b6c10c63 TL	494
75584009 FG	495	my $cts = eval { PVE::API2::LXC->vmlist({ node => $nodename }) };
75584009 FG	496	log_warn("Failed to retrieve information about this node's CTs - $@") if $@;
b6c10c63 TL	497	$running_guests += grep { $_->{status} eq 'running' } @$cts if defined($cts);
	498
	499	if ($running_guests > 0) {
	500	log_warn("$running_guests running guest(s) detected - consider migrating or stopping them.")
	501	} else {
	502	log_pass("no running guest detected.")
	503	}
0286a659	504
33232071	505	log_info("Checking if the local node's hostname '$nodename' is resolvable..");
6a494e2e	506	my $local_ip = eval { PVE::Network::get_ip_from_hostname($nodename) };
0286a659	507	if ($@) {
6a494e2e	508	log_warn("Failed to resolve hostname '$nodename' to IP - $@");
0286a659	509	} else {
6c9e9cc8	510	log_info("Checking if resolved IP is configured on local node..");
0286a659 ML	511	my $cidr = Net::IP::ip_is_ipv6($local_ip) ? "$local_ip/128" : "$local_ip/32";
	512	my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
	513	my $ip_count = scalar(@$configured_ips);
bacd0b0b TL	514
bacd0b0b TL	515	if ($ip_count <= 0) {
6a494e2e	516	log_fail("Resolved node IP '$local_ip' not configured or active for '$nodename'");
bacd0b0b TL	517	} elsif ($ip_count > 1) {
bacd0b0b TL	518	log_warn("Resolved node IP '$local_ip' active on multiple ($ip_count) interfaces!");
b6c10c63	519	} else {
6c9e9cc8	520	log_pass("Resolved node IP '$local_ip' configured and active on single interface.");
bacd0b0b	521	}
0286a659	522	}
e6191e3e ML	523
e6191e3e ML	524	check_kvm_nested();
75584009 FG	525	}
	526
	527	__PACKAGE__->register_method ({
	528	name => 'checklist',
	529	path => 'checklist',
	530	method => 'GET',
	531	description => 'Check (pre-/post-)upgrade conditions.',
	532	parameters => {
	533	additionalProperties => 0,
	534	properties => {
	535	},
	536	},
	537	returns => { type => 'null' },
	538	code => sub {
	539	my ($param) = @_;
	540
	541	check_pve_packages();
	542	check_cluster_corosync();
	543	check_ceph();
	544	check_storage_health();
	545	check_misc();
	546
465b3ea2	547	print_header("SUMMARY");
bc91ccac TL	548
	549	my $total = 0;
	550	$total += $_ for values %$counters;
	551
	552	print "TOTAL: $total\n";
1c6ac415 TL	553	print colored("PASSED: $counters->{pass}\n", 'green');
1c6ac415 TL	554	print "SKIPPED: $counters->{skip}\n";
6acde780 FG	555	print colored("WARNINGS: $counters->{warn}\n", 'yellow');
6acde780 FG	556	print colored("FAILURES: $counters->{fail}\n", 'red');
75584009	557
6acde780	558	print colored("\nATTENTION: Please check the output for detailed information!\n", 'red')
75584009 FG	559	if ($counters->{warn} > 0 \|\| $counters->{fail} > 0);
	560
	561	return undef;
	562	}});
	563
3f11a62c TL	564	our $cmddef = [ __PACKAGE__, 'checklist', [], {}];
	565
	566	# for now drop all unknown params and just check
	567	@ARGV = ();
75584009 FG	568
75584009 FG	569	1;