src/PMG/CLI/pmg7to8.pm

   1 package PMG::CLI::pmg7to8;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd ();
   7
   8 use PVE::INotify;
   9 use PVE::JSONSchema;
  10 use PVE::Tools qw(run_command split_list file_get_contents);
  11
  12 use PMG::API2::APT;
  13 use PMG::API2::Certificates;
  14 use PMG::API2::Cluster;
  15 use PMG::RESTEnvironment;
  16 use PMG::Utils;
  17
  18 use Term::ANSIColor;
  19
  20 use PVE::CLIHandler;
  21
  22 use base qw(PVE::CLIHandler);
  23
  24 my $nodename = PVE::INotify::nodename();
  25
  26 my $old_postgres_release = '13';
  27 my $new_postgres_release = '15';
  28
  29 my $old_suite = 'bullseye';
  30 my $new_suite = 'bookworm';
  31
  32 my $upgraded = 0; # set in check_pmg_packages
  33
  34 sub setup_environment {
  35     PMG::RESTEnvironment->setup_default_cli_env();
  36 }
  37
  38 my ($min_pmg_major, $min_pmg_minor, $min_pmg_pkgrel) = (7, 3, 2);
  39
  40 my $counters = {
  41     pass => 0,
  42     skip => 0,
  43     notice => 0,
  44     warn => 0,
  45     fail => 0,
  46 };
  47
  48 my $log_line = sub {
  49     my ($level, $line) = @_;
  50
  51     $counters->{$level}++ if defined($level) && defined($counters->{$level});
  52
  53     print uc($level), ': ' if defined($level);
  54     print "$line\n";
  55 };
  56
  57 sub log_pass {
  58     print color('green');
  59     $log_line->('pass', @_);
  60     print color('reset');
  61 }
  62
  63 sub log_info {
  64     $log_line->('info', @_);
  65 }
  66 sub log_skip {
  67     $log_line->('skip', @_);
  68 }
  69 sub log_notice {
  70     print color('bold');
  71     $log_line->('notice', @_);
  72     print color('reset');
  73 }
  74 sub log_warn {
  75     print color('yellow');
  76     $log_line->('warn', @_);
  77     print color('reset');
  78 }
  79 sub log_fail {
  80     print color('bold red');
  81     $log_line->('fail', @_);
  82     print color('reset');
  83 }
  84
  85 my $print_header_first = 1;
  86 sub print_header {
  87     my ($h) = @_;
  88     print "\n" if !$print_header_first;
  89     print "= $h =\n\n";
  90     $print_header_first = 0;
  91 }
  92
  93 my $get_systemd_unit_state = sub {
  94     my ($unit, $suppress_stderr) = @_;
  95
  96     my $state;
  97     my $filter_output = sub {
  98         $state = shift;
  99         chomp $state;
 100     };
 101
 102     my %extra = (outfunc => $filter_output, noerr => 1);
 103     $extra{errfunc} = sub {  } if $suppress_stderr;
 104
 105     eval {
 106         run_command(['systemctl', 'is-enabled', "$unit"], %extra);
 107         return if !defined($state);
 108         run_command(['systemctl', 'is-active', "$unit"], %extra);
 109     };
 110
 111     return $state // 'unknown';
 112 };
 113
 114 my $log_systemd_unit_state = sub {
 115     my ($unit, $no_fail_on_inactive) = @_;
 116
 117     my $log_method = \&log_warn;
 118
 119     my $state = $get_systemd_unit_state->($unit);
 120     if ($state eq 'active') {
 121         $log_method = \&log_pass;
 122     } elsif ($state eq 'inactive') {
 123         $log_method = $no_fail_on_inactive ? \&log_warn : \&log_fail;
 124     } elsif ($state eq 'failed') {
 125         $log_method = \&log_fail;
 126     }
 127
 128     $log_method->("systemd unit '$unit' is in state '$state'");
 129 };
 130
 131 my $versions;
 132 my $get_pkg = sub {
 133     my ($pkg) = @_;
 134
 135     $versions = eval { PMG::API2::APT->versions({ node => $nodename }) } if !defined($versions);
 136
 137     if (!defined($versions)) {
 138         my $msg = "unable to retrieve package version information";
 139         $msg .= "- $@" if $@;
 140         log_fail("$msg");
 141         return undef;
 142     }
 143
 144     my $pkgs = [ grep { $_->{Package} eq $pkg } @$versions ];
 145     if (!defined $pkgs || $pkgs == 0) {
 146         log_fail("unable to determine installed $pkg version.");
 147         return undef;
 148     } else {
 149         return $pkgs->[0];
 150     }
 151 };
 152
 153 sub check_pmg_packages {
 154     print_header("CHECKING VERSION INFORMATION FOR PMG PACKAGES");
 155
 156     print "Checking for package updates..\n";
 157     my $updates = eval { PMG::API2::APT->list_updates({ node => $nodename }); };
 158     if (!defined($updates)) {
 159         log_warn("$@") if $@;
 160         log_fail("unable to retrieve list of package updates!");
 161     } elsif (@$updates > 0) {
 162         my $pkgs = join(', ', map { $_->{Package} } @$updates);
 163         log_warn("updates for the following packages are available:\n  $pkgs");
 164     } else {
 165         log_pass("all packages up-to-date");
 166     }
 167
 168     print "\nChecking proxmox-mailgateway package version..\n";
 169     my $pkg = 'proxmox-mailgateway';
 170     my $pmg = $get_pkg->($pkg);
 171     if (!defined($pmg)) {
 172         print "\n$pkg not found, checking for proxmox-mailgateway-container..\n";
 173         $pkg = 'proxmox-mailgateway-container';
 174     }
 175     if (defined(my $pmg = $get_pkg->($pkg))) {
 176         # TODO: update to native version for pmg8to9
 177         my $min_pmg_ver = "$min_pmg_major.$min_pmg_minor-$min_pmg_pkgrel";
 178
 179         my ($maj, $min, $pkgrel) = $pmg->{OldVersion} =~ m/^(\d+)\.(\d+)[.-](\d+)/;
 180
 181         if ($maj > $min_pmg_major) {
 182             log_pass("already upgraded to Proxmox Mail Gateway " . ($min_pmg_major + 1));
 183             $upgraded = 1;
 184         } elsif ($maj >= $min_pmg_major && $min >= $min_pmg_minor && $pkgrel >= $min_pmg_pkgrel) {
 185             log_pass("$pkg package has version >= $min_pmg_ver");
 186         } else {
 187             log_fail("$pkg package is too old, please upgrade to >= $min_pmg_ver!");
 188         }
 189
 190         # FIXME: better differentiate between 6.2 from bullseye or bookworm
 191         my ($krunning, $kinstalled) = (qr/6\.(?:2\.(?:[2-9]\d+|1[6-8]|1\d\d+)|5)[^~]*$/, 'pve-kernel-6.2');
 192         if (!$upgraded) {
 193             # we got a few that avoided 5.15 in cluster with mixed CPUs, so allow older too
 194             ($krunning, $kinstalled) = (qr/(?:5\.(?:13|15)|6\.2)/, 'pve-kernel-5.15');
 195         }
 196
 197         print "\nChecking running kernel version..\n";
 198         my $kernel_ver = $pmg->{RunningKernel};
 199         if (!defined($kernel_ver)) {
 200             log_fail("unable to determine running kernel version.");
 201         } elsif ($kernel_ver =~ /^$krunning/) {
 202             if ($upgraded) {
 203                 log_pass("running new kernel '$kernel_ver' after upgrade.");
 204             } else {
 205                 log_pass("running kernel '$kernel_ver' is considered suitable for upgrade.");
 206             }
 207         } elsif ($get_pkg->($kinstalled)) {
 208             # with 6.2 kernel being available in both we might want to fine-tune the check?
 209             log_warn("a suitable kernel ($kinstalled) is intalled, but an unsuitable ($kernel_ver) is booted, missing reboot?!");
 210         } else {
 211             log_warn("unexpected running and installed kernel '$kernel_ver'.");
 212         }
 213
 214         if ($upgraded && $kernel_ver =~ /^$krunning/) {
 215             my $outdated_kernel_meta_pkgs = [];
 216             for my $kernel_meta_version ('5.4', '5.11', '5.13', '5.15') {
 217                 my $pkg = "pve-kernel-${kernel_meta_version}";
 218                 if ($get_pkg->($pkg)) {
 219                     push @$outdated_kernel_meta_pkgs, $pkg;
 220                 }
 221             }
 222             if (scalar(@$outdated_kernel_meta_pkgs) > 0) {
 223                 log_info(
 224                     "Found outdated kernel meta-packages, taking up extra space on boot partitions.\n"
 225                     ."      After a successful upgrade, you can remove them using this command:\n"
 226                     ."      apt remove " . join(' ', $outdated_kernel_meta_pkgs->@*)
 227                 );
 228             }
 229         }
 230     } else {
 231         log_fail("$pkg package not found!");
 232     }
 233 }
 234
 235 my sub check_max_length {
 236     my ($raw, $max_length, $warning) = @_;
 237     log_warn($warning) if defined($raw) && length($raw) > $max_length;
 238 }
 239
 240 my $is_cluster = 0;
 241 my $cluster_healthy = 0;
 242
 243 sub check_cluster_status {
 244     log_info("Checking if the cluster nodes are in sync");
 245
 246     my $rpcenv = PMG::RESTEnvironment->get();
 247     my $ticket = PMG::Ticket::assemble_ticket($rpcenv->get_user());
 248     $rpcenv->set_ticket($ticket);
 249
 250     my $nodes = PMG::API2::Cluster->status({});
 251     if (!scalar($nodes->@*)) {
 252         log_skip("no cluster, no sync status to check");
 253         $cluster_healthy = 1;
 254         return;
 255     }
 256
 257     $is_cluster = 1;
 258     my $syncing = 0;
 259     my $errors = 0;
 260
 261     for my $node ($nodes->@*) {
 262         if (!$node->{insync}) {
 263             $syncing = 1;
 264         }
 265         if ($node->{conn_error}) {
 266             $errors = 1;
 267         }
 268     }
 269
 270     if ($errors) {
 271         log_fail("Cluster not healthy, please fix the cluster before continuing");
 272     } elsif ($syncing) {
 273         log_warn("Cluster currently syncing.");
 274     } else {
 275         log_pass("Cluster healthy and in sync.");
 276         $cluster_healthy = 1;
 277     }
 278 }
 279
 280
 281 sub check_running_postgres {
 282     my $version = PMG::Utils::get_pg_server_version();
 283
 284     my $upgraded_db = 0;
 285
 286     if ($upgraded) {
 287         if ($version ne $new_postgres_release) {
 288             log_warn("Running postgres version is still $old_postgres_release. Please upgrade the database.");
 289         } else {
 290             log_pass("After upgrade and running postgres version is $new_postgres_release.");
 291             $upgraded_db = 1;
 292         }
 293     } else {
 294         if ($version ne $old_postgres_release) {
 295             log_fail("Running postgres version '$version' is not '$old_postgres_release', was a previous upgrade left unfinished?");
 296         } else {
 297             log_pass("Before upgrade and running postgres version is $old_postgres_release.");
 298         }
 299     }
 300
 301     return $upgraded_db;
 302 }
 303
 304 sub check_services_disabled {
 305     my ($upgraded_db) = @_;
 306     my $unit_inactive = sub { return $get_systemd_unit_state->($_[0], 1) eq 'inactive' ? $_[0] : undef };
 307
 308     my $services = [qw(postfix pmg-smtp-filter pmgpolicy pmgdaemon pmgproxy)];
 309
 310     if ($is_cluster) {
 311         push $services->@*, 'pmgmirror', 'pmgtunnel';
 312     }
 313
 314     my $active_list = [];
 315     my $inactive_list = [];
 316     for my $service ($services->@*) {
 317         if (!$unit_inactive->($service)) {
 318             push $active_list->@*, $service;
 319         } else {
 320             push $inactive_list->@*, $service;
 321         }
 322     }
 323
 324     if (!$upgraded) {
 325         if (scalar($active_list->@*) < 1) {
 326             log_pass("All services inactive.");
 327         } else {
 328             my $msg = "Not upgraded but core services still active. Consider stopping and masking them for the upgrade: \n    ";
 329             $msg .= join("\n    ", $active_list->@*);
 330             log_warn($msg);
 331         }
 332     } else {
 333         if (scalar($inactive_list->@*) < 1) {
 334             log_pass("All services active.");
 335         } elsif ($upgraded_db) {
 336             my $msg = "Already upgraded DB, but not all services active again. Consider unmasking and starting them: \n    ";
 337             $msg .= join("\n    ", $inactive_list->@*);
 338             log_warn($msg);
 339         } else {
 340             log_skip("Not all services active, but DB was not upgraded yet - please upgrade DB and then unmask and start services again.");
 341         }
 342     }
 343 }
 344
 345 sub check_apt_repos {
 346     log_info("Checking for package repository suite mismatches..");
 347
 348     my $dir = '/etc/apt/sources.list.d';
 349     my $in_dir = 0;
 350
 351     # TODO: check that (original) debian and Proxmox MG mirrors are present.
 352
 353     my ($found_suite, $found_suite_where);
 354     my ($mismatches, $strange_suite);
 355
 356     my $check_file = sub {
 357         my ($file) = @_;
 358
 359         $file = "${dir}/${file}" if $in_dir;
 360
 361         my $raw = eval { PVE::Tools::file_get_contents($file) };
 362         return if !defined($raw);
 363         my @lines = split(/\n/, $raw);
 364
 365         my $number = 0;
 366         for my $line (@lines) {
 367             $number++;
 368
 369             next if length($line) == 0; # split would result in undef then...
 370
 371             ($line) = split(/#/, $line);
 372
 373             next if $line !~ m/^deb[[:space:]]/; # is case sensitive
 374
 375             my $suite;
 376             if ($line =~ m|deb\s+\w+://\S+\s+(\S*)|i) {
 377                 $suite = $1;
 378             } else {
 379                 next;
 380             }
 381             my $where = "in ${file}:${number}";
 382
 383             $suite =~ s/-(?:updates|backports|security)$//;
 384             if ($suite ne $old_suite && $suite ne $new_suite) {
 385                 log_notice(
 386                     "found unusual suite '$suite', neither old '$old_suite' nor new '$new_suite'.."
 387                     ."\n    Affected file:line $where"
 388                     ."\n    Please assure this is shipping compatible packages for the upgrade!"
 389                 );
 390                 $strange_suite = 1;
 391                 next;
 392             }
 393
 394             if (!defined($found_suite)) {
 395                 $found_suite = $suite;
 396                 $found_suite_where = $where;
 397             } elsif ($suite ne $found_suite) {
 398                 if (!defined($mismatches)) {
 399                     $mismatches = [];
 400                     push $mismatches->@*,
 401                         { suite => $found_suite, where => $found_suite_where},
 402                         { suite => $suite, where => $where};
 403                 } else {
 404                     push $mismatches->@*, { suite => $suite, where => $where};
 405                 }
 406             }
 407         }
 408     };
 409
 410     $check_file->("/etc/apt/sources.list");
 411
 412     $in_dir = 1;
 413
 414     PVE::Tools::dir_glob_foreach($dir, '^.*\.list$', $check_file);
 415
 416     if (defined($mismatches)) {
 417         my @mismatch_list = map { "found suite $_->{suite} at $_->{where}" } $mismatches->@*;
 418
 419         log_fail(
 420             "Found mixed old and new package repository suites, fix before upgrading! Mismatches:"
 421             ."\n    " . join("\n    ", @mismatch_list)
 422         );
 423     } elsif ($strange_suite) {
 424         log_notice("found no suite mismatches, but found at least one strange suite");
 425     } else {
 426         log_pass("found no suite mismatch");
 427     }
 428 }
 429
 430 sub check_time_sync {
 431     my $unit_active = sub { return $get_systemd_unit_state->($_[0], 1) eq 'active' ? $_[0] : undef };
 432
 433     log_info("Checking for supported & active NTP service..");
 434     if ($unit_active->('systemd-timesyncd.service')) {
 435         log_warn(
 436             "systemd-timesyncd is not the best choice for time-keeping on servers, due to only applying"
 437             ." updates on boot.\n  While not necessary for the upgrade it's recommended to use one of:\n"
 438             ."    * chrony (Default in new Proxmox VE installations)\n    * ntpsec\n    * openntpd\n"
 439         );
 440     } elsif ($unit_active->('ntp.service')) {
 441         log_info("Debian deprecated and removed the ntp package for Bookworm, but the system"
 442             ." will automatically migrate to the 'ntpsec' replacement package on upgrade.");
 443     } elsif (my $active_ntp = ($unit_active->('chrony.service') || $unit_active->('openntpd.service') || $unit_active->('ntpsec.service'))) {
 444         log_pass("Detected active time synchronisation unit '$active_ntp'");
 445     } else {
 446         log_warn(
 447             "No (active) time synchronisation daemon (NTP) detected, but synchronized systems are important,"
 448             ." especially for cluster and/or ceph!"
 449         );
 450     }
 451 }
 452
 453 sub check_bootloader {
 454     log_info("Checking bootloader configuration...");
 455     if (!$upgraded) {
 456         log_skip("not yet upgraded, no need to check the presence of systemd-boot");
 457         return;
 458     }
 459
 460     if (! -f "/etc/kernel/proxmox-boot-uuids") {
 461         log_skip("proxmox-boot-tool not used for bootloader configuration");
 462         return;
 463     }
 464
 465     if (! -d "/sys/firmware/efi") {
 466         log_skip("System booted in legacy-mode - no need for systemd-boot");
 467         return;
 468     }
 469
 470     if ( -f "/usr/share/doc/systemd-boot/changelog.Debian.gz") {
 471         log_pass("systemd-boot is installed");
 472     } else {
 473         log_warn(
 474             "proxmox-boot-tool is used for bootloader configuration in uefi mode"
 475             . "but the separate systemd-boot package, existing in Debian Bookworm  is not installed"
 476             . "initializing new ESPs will not work until the package is installed"
 477         );
 478     }
 479 }
 480
 481 sub check_misc {
 482     print_header("MISCELLANEOUS CHECKS");
 483     my $ssh_config = eval { PVE::Tools::file_get_contents('/root/.ssh/config') };
 484     if (defined($ssh_config)) {
 485         log_fail("Unsupported SSH Cipher configured for root in /root/.ssh/config: $1")
 486             if $ssh_config =~ /^Ciphers .*(blowfish|arcfour|3des).*$/m;
 487     } else {
 488         log_skip("No SSH config file found.");
 489     }
 490
 491     check_time_sync();
 492
 493     my $root_free = PVE::Tools::df('/', 10);
 494     log_warn("Less than 5 GB free space on root file system.")
 495         if defined($root_free) && $root_free->{avail} < 5 * 1000*1000*1000;
 496
 497     log_info("Checking if the local node's hostname '$nodename' is resolvable..");
 498     my $local_ip = eval { PVE::Network::get_ip_from_hostname($nodename) };
 499     if ($@) {
 500         log_warn("Failed to resolve hostname '$nodename' to IP - $@");
 501     } else {
 502         log_info("Checking if resolved IP is configured on local node..");
 503         my $cidr = Net::IP::ip_is_ipv6($local_ip) ? "$local_ip/128" : "$local_ip/32";
 504         my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
 505         my $ip_count = scalar(@$configured_ips);
 506
 507         if ($ip_count <= 0) {
 508             log_fail("Resolved node IP '$local_ip' not configured or active for '$nodename'");
 509         } elsif ($ip_count > 1) {
 510             log_warn("Resolved node IP '$local_ip' active on multiple ($ip_count) interfaces!");
 511         } else {
 512             log_pass("Resolved node IP '$local_ip' configured and active on single interface.");
 513         }
 514     }
 515
 516     log_info("Check node certificate's RSA key size");
 517     my $certs = PMG::API2::Certificates->info({ node => $nodename });
 518     my $certs_check = {
 519         'rsaEncryption' => {
 520             minsize => 2048,
 521             name => 'RSA',
 522         },
 523         'id-ecPublicKey' => {
 524             minsize => 224,
 525             name => 'ECC',
 526         },
 527     };
 528
 529     my $certs_check_failed = 0;
 530     for my $cert (@$certs) {
 531         my ($type, $size, $fn) = $cert->@{qw(public-key-type public-key-bits filename)};
 532
 533         if (!defined($type) || !defined($size)) {
 534             log_warn("'$fn': cannot check certificate, failed to get it's type or size!");
 535         }
 536
 537         my $check = $certs_check->{$type};
 538         if (!defined($check)) {
 539             log_warn("'$fn': certificate's public key type '$type' unknown!");
 540             next;
 541         }
 542
 543         if ($size < $check->{minsize}) {
 544             log_fail("'$fn', certificate's $check->{name} public key size is less than 2048 bit");
 545             $certs_check_failed = 1;
 546         } else {
 547             log_pass("Certificate '$fn' passed Debian Busters (and newer) security level for TLS connections ($size >= 2048)");
 548         }
 549     }
 550
 551     check_apt_repos();
 552     check_bootloader();
 553 }
 554
 555 my sub colored_if {
 556     my ($str, $color, $condition) = @_;
 557     return "". ($condition ? colored($str, $color) : $str);
 558 }
 559
 560 __PACKAGE__->register_method ({
 561     name => 'checklist',
 562     path => 'checklist',
 563     method => 'GET',
 564     description => 'Check (pre-/post-)upgrade conditions.',
 565     parameters => {
 566         additionalProperties => 0,
 567         properties => {},
 568     },
 569     returns => { type => 'null' },
 570     code => sub {
 571         my ($param) = @_;
 572
 573         check_pmg_packages();
 574         check_cluster_status();
 575         my $upgraded_db = check_running_postgres();
 576         check_services_disabled($upgraded_db);
 577         check_misc();
 578
 579         print_header("SUMMARY");
 580
 581         my $total = 0;
 582         $total += $_ for values %$counters;
 583
 584         print "TOTAL:    $total\n";
 585         print colored("PASSED:   $counters->{pass}\n", 'green');
 586         print "SKIPPED:  $counters->{skip}\n";
 587         print colored_if("WARNINGS: $counters->{warn}\n", 'yellow', $counters->{warn} > 0);
 588         print colored_if("FAILURES: $counters->{fail}\n", 'bold red', $counters->{fail} > 0);
 589
 590         if ($counters->{warn} > 0 || $counters->{fail} > 0) {
 591             my $color = $counters->{fail} > 0 ? 'bold red' : 'yellow';
 592             print colored("\nATTENTION: Please check the output for detailed information!\n", $color);
 593             print colored("Try to solve the problems one at a time and then run this checklist tool again.\n", $color) if $counters->{fail} > 0;
 594         }
 595
 596         return undef;
 597     }});
 598
 599 our $cmddef = [ __PACKAGE__, 'checklist', [], {}];
 600
 601 1;