src/PMG/CLI/pmg7to8.pm

   1 package PMG::CLI::pmg7to8;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd ();
   7
   8 use PVE::INotify;
   9 use PVE::JSONSchema;
  10 use PVE::Tools qw(run_command split_list file_get_contents);
  11
  12 use PMG::API2::APT;
  13 use PMG::API2::Certificates;
  14 use PMG::API2::Cluster;
  15 use PMG::RESTEnvironment;
  16 use PMG::Utils;
  17
  18 use Term::ANSIColor;
  19
  20 use PVE::CLIHandler;
  21
  22 use base qw(PVE::CLIHandler);
  23
  24 my $nodename = PVE::INotify::nodename();
  25
  26 my $old_postgres_release = '13';
  27 my $new_postgres_release = '15';
  28
  29 my $old_suite = 'bullseye';
  30 my $new_suite = 'bookworm';
  31
  32 my $upgraded = 0; # set in check_pmg_packages
  33
  34 sub setup_environment {
  35     PMG::RESTEnvironment->setup_default_cli_env();
  36 }
  37
  38 my ($min_pmg_major, $min_pmg_minor, $min_pmg_pkgrel) = (7, 3, 2);
  39
  40 my $counters = {
  41     pass => 0,
  42     skip => 0,
  43     notice => 0,
  44     warn => 0,
  45     fail => 0,
  46 };
  47
  48 my $log_line = sub {
  49     my ($level, $line) = @_;
  50
  51     $counters->{$level}++ if defined($level) && defined($counters->{$level});
  52
  53     print uc($level), ': ' if defined($level);
  54     print "$line\n";
  55 };
  56
  57 sub log_pass {
  58     print color('green');
  59     $log_line->('pass', @_);
  60     print color('reset');
  61 }
  62
  63 sub log_info {
  64     $log_line->('info', @_);
  65 }
  66 sub log_skip {
  67     $log_line->('skip', @_);
  68 }
  69 sub log_notice {
  70     print color('bold');
  71     $log_line->('notice', @_);
  72     print color('reset');
  73 }
  74 sub log_warn {
  75     print color('yellow');
  76     $log_line->('warn', @_);
  77     print color('reset');
  78 }
  79 sub log_fail {
  80     print color('bold red');
  81     $log_line->('fail', @_);
  82     print color('reset');
  83 }
  84
  85 my $print_header_first = 1;
  86 sub print_header {
  87     my ($h) = @_;
  88     print "\n" if !$print_header_first;
  89     print "= $h =\n\n";
  90     $print_header_first = 0;
  91 }
  92
  93 my $get_systemd_unit_state = sub {
  94     my ($unit, $suppress_stderr) = @_;
  95
  96     my $state;
  97     my $filter_output = sub {
  98         $state = shift;
  99         chomp $state;
 100     };
 101
 102     my %extra = (outfunc => $filter_output, noerr => 1);
 103     $extra{errfunc} = sub {  } if $suppress_stderr;
 104
 105     eval {
 106         run_command(['systemctl', 'is-enabled', "$unit"], %extra);
 107         return if !defined($state);
 108         run_command(['systemctl', 'is-active', "$unit"], %extra);
 109     };
 110
 111     return $state // 'unknown';
 112 };
 113
 114 my $log_systemd_unit_state = sub {
 115     my ($unit, $no_fail_on_inactive) = @_;
 116
 117     my $log_method = \&log_warn;
 118
 119     my $state = $get_systemd_unit_state->($unit);
 120     if ($state eq 'active') {
 121         $log_method = \&log_pass;
 122     } elsif ($state eq 'inactive') {
 123         $log_method = $no_fail_on_inactive ? \&log_warn : \&log_fail;
 124     } elsif ($state eq 'failed') {
 125         $log_method = \&log_fail;
 126     }
 127
 128     $log_method->("systemd unit '$unit' is in state '$state'");
 129 };
 130
 131 my $versions;
 132 my $get_pkg = sub {
 133     my ($pkg) = @_;
 134
 135     $versions = eval { PMG::API2::APT->versions({ node => $nodename }) } if !defined($versions);
 136
 137     if (!defined($versions)) {
 138         my $msg = "unable to retrieve package version information";
 139         $msg .= "- $@" if $@;
 140         log_fail("$msg");
 141         return undef;
 142     }
 143
 144     my $pkgs = [ grep { $_->{Package} eq $pkg } @$versions ];
 145     if (!defined $pkgs || $pkgs == 0) {
 146         log_fail("unable to determine installed $pkg version.");
 147         return undef;
 148     } else {
 149         return $pkgs->[0];
 150     }
 151 };
 152
 153 sub check_pmg_packages {
 154     print_header("CHECKING VERSION INFORMATION FOR PMG PACKAGES");
 155
 156     print "Checking for package updates..\n";
 157     my $updates = eval { PMG::API2::APT->list_updates({ node => $nodename }); };
 158     if (!defined($updates)) {
 159         log_warn("$@") if $@;
 160         log_fail("unable to retrieve list of package updates!");
 161     } elsif (@$updates > 0) {
 162         my $pkgs = join(', ', map { $_->{Package} } @$updates);
 163         log_warn("updates for the following packages are available:\n  $pkgs");
 164     } else {
 165         log_pass("all packages up-to-date");
 166     }
 167
 168     print "\nChecking proxmox-mailgateway package version..\n";
 169     my $pkg = 'proxmox-mailgateway';
 170     my $pmg = $get_pkg->($pkg);
 171     if (!defined($pmg)) {
 172         print "\n$pkg not found, checking for proxmox-mailgateway-container..\n";
 173         $pkg = 'proxmox-mailgateway-container';
 174     }
 175     if (defined(my $pmg = $get_pkg->($pkg))) {
 176         # TODO: update to native version for pmg8to9
 177         my $min_pmg_ver = "$min_pmg_major.$min_pmg_minor-$min_pmg_pkgrel";
 178
 179         my ($maj, $min, $pkgrel) = $pmg->{OldVersion} =~ m/^(\d+)\.(\d+)[.-](\d+)/;
 180
 181         if ($maj > $min_pmg_major) {
 182             log_pass("already upgraded to Proxmox Mail Gateway " . ($min_pmg_major + 1));
 183             $upgraded = 1;
 184         } elsif ($maj >= $min_pmg_major && $min >= $min_pmg_minor && $pkgrel >= $min_pmg_pkgrel) {
 185             log_pass("$pkg package has version >= $min_pmg_ver");
 186         } else {
 187             log_fail("$pkg package is too old, please upgrade to >= $min_pmg_ver!");
 188         }
 189
 190         if ($pkg eq 'proxmox-mailgateway-container') {
 191             log_skip("Ignoring kernel version checks for $pkg meta-package");
 192             return;
 193         }
 194
 195         # FIXME: better differentiate between 6.2 from bullseye or bookworm
 196         my ($krunning, $kinstalled) = (qr/6\.(?:2\.(?:[2-9]\d+|1[6-8]|1\d\d+)|5)[^~]*$/, 'proxmox-kernel-6.2');
 197         if (!$upgraded) {
 198             # we got a few that avoided 5.15 in cluster with mixed CPUs, so allow older too
 199             ($krunning, $kinstalled) = (qr/(?:5\.(?:13|15)|6\.2)/, 'pve-kernel-5.15');
 200         }
 201
 202         print "\nChecking running kernel version..\n";
 203         my $kernel_ver = $pmg->{RunningKernel};
 204         if (!defined($kernel_ver)) {
 205             log_fail("unable to determine running kernel version.");
 206         } elsif ($kernel_ver =~ /^$krunning/) {
 207             if ($upgraded) {
 208                 log_pass("running new kernel '$kernel_ver' after upgrade.");
 209             } else {
 210                 log_pass("running kernel '$kernel_ver' is considered suitable for upgrade.");
 211             }
 212         } elsif ($get_pkg->($kinstalled)) {
 213             # with 6.2 kernel being available in both we might want to fine-tune the check?
 214             log_warn("a suitable kernel ($kinstalled) is intalled, but an unsuitable ($kernel_ver) is booted, missing reboot?!");
 215         } else {
 216             log_warn("unexpected running and installed kernel '$kernel_ver'.");
 217         }
 218
 219         if ($upgraded && $kernel_ver =~ /^$krunning/) {
 220             my $outdated_kernel_meta_pkgs = [];
 221             for my $kernel_meta_version ('5.4', '5.11', '5.13', '5.15') {
 222                 my $pkg = "pve-kernel-${kernel_meta_version}";
 223                 if ($get_pkg->($pkg)) {
 224                     push @$outdated_kernel_meta_pkgs, $pkg;
 225                 }
 226             }
 227             if (scalar(@$outdated_kernel_meta_pkgs) > 0) {
 228                 log_info(
 229                     "Found outdated kernel meta-packages, taking up extra space on boot partitions.\n"
 230                     ."      After a successful upgrade, you can remove them using this command:\n"
 231                     ."      apt remove " . join(' ', $outdated_kernel_meta_pkgs->@*)
 232                 );
 233             }
 234         }
 235     } else {
 236         log_fail("$pkg package not found!");
 237     }
 238 }
 239
 240 my sub check_max_length {
 241     my ($raw, $max_length, $warning) = @_;
 242     log_warn($warning) if defined($raw) && length($raw) > $max_length;
 243 }
 244
 245 my $is_cluster = 0;
 246 my $cluster_healthy = 0;
 247
 248 sub check_cluster_status {
 249     log_info("Checking if the cluster nodes are in sync");
 250
 251     my $rpcenv = PMG::RESTEnvironment->get();
 252     my $ticket = PMG::Ticket::assemble_ticket($rpcenv->get_user());
 253     $rpcenv->set_ticket($ticket);
 254
 255     my $nodes = PMG::API2::Cluster->status({});
 256     if (!scalar($nodes->@*)) {
 257         log_skip("no cluster, no sync status to check");
 258         $cluster_healthy = 1;
 259         return;
 260     }
 261
 262     $is_cluster = 1;
 263     my $syncing = 0;
 264     my $errors = 0;
 265
 266     for my $node ($nodes->@*) {
 267         if (!$node->{insync}) {
 268             $syncing = 1;
 269         }
 270         if ($node->{conn_error}) {
 271             $errors = 1;
 272         }
 273     }
 274
 275     if ($errors) {
 276         log_fail("Cluster not healthy, please fix the cluster before continuing");
 277     } elsif ($syncing) {
 278         log_warn("Cluster currently syncing.");
 279     } else {
 280         log_pass("Cluster healthy and in sync.");
 281         $cluster_healthy = 1;
 282     }
 283 }
 284
 285
 286 sub check_running_postgres {
 287     my $version = PMG::Utils::get_pg_server_version();
 288
 289     my $upgraded_db = 0;
 290
 291     if ($upgraded) {
 292         if ($version ne $new_postgres_release) {
 293             log_warn("Running postgres version is still $old_postgres_release. Please upgrade the database.");
 294         } else {
 295             log_pass("After upgrade and running postgres version is $new_postgres_release.");
 296             $upgraded_db = 1;
 297         }
 298     } else {
 299         if ($version ne $old_postgres_release) {
 300             log_fail("Running postgres version '$version' is not '$old_postgres_release', was a previous upgrade left unfinished?");
 301         } else {
 302             log_pass("Before upgrade and running postgres version is $old_postgres_release.");
 303         }
 304     }
 305
 306     return $upgraded_db;
 307 }
 308
 309 sub check_services_disabled {
 310     my ($upgraded_db) = @_;
 311     my $unit_inactive = sub { return $get_systemd_unit_state->($_[0], 1) eq 'inactive' ? $_[0] : undef };
 312
 313     my $services = [qw(postfix pmg-smtp-filter pmgpolicy pmgdaemon pmgproxy)];
 314
 315     if ($is_cluster) {
 316         push $services->@*, 'pmgmirror', 'pmgtunnel';
 317     }
 318
 319     my $active_list = [];
 320     my $inactive_list = [];
 321     for my $service ($services->@*) {
 322         if (!$unit_inactive->($service)) {
 323             push $active_list->@*, $service;
 324         } else {
 325             push $inactive_list->@*, $service;
 326         }
 327     }
 328
 329     if (!$upgraded) {
 330         if (scalar($active_list->@*) < 1) {
 331             log_pass("All services inactive.");
 332         } else {
 333             my $msg = "Not upgraded but core services still active. Consider stopping and masking them for the upgrade: \n    ";
 334             $msg .= join("\n    ", $active_list->@*);
 335             log_warn($msg);
 336         }
 337     } else {
 338         if (scalar($inactive_list->@*) < 1) {
 339             log_pass("All services active.");
 340         } elsif ($upgraded_db) {
 341             my $msg = "Already upgraded DB, but not all services active again. Consider unmasking and starting them: \n    ";
 342             $msg .= join("\n    ", $inactive_list->@*);
 343             log_warn($msg);
 344         } else {
 345             log_skip("Not all services active, but DB was not upgraded yet - please upgrade DB and then unmask and start services again.");
 346         }
 347     }
 348 }
 349
 350 sub check_apt_repos {
 351     log_info("Checking for package repository suite mismatches..");
 352
 353     my $dir = '/etc/apt/sources.list.d';
 354     my $in_dir = 0;
 355
 356     # TODO: check that (original) debian and Proxmox MG mirrors are present.
 357
 358     my ($found_suite, $found_suite_where);
 359     my ($mismatches, $strange_suite);
 360
 361     my $check_file = sub {
 362         my ($file) = @_;
 363
 364         $file = "${dir}/${file}" if $in_dir;
 365
 366         my $raw = eval { PVE::Tools::file_get_contents($file) };
 367         return if !defined($raw);
 368         my @lines = split(/\n/, $raw);
 369
 370         my $number = 0;
 371         for my $line (@lines) {
 372             $number++;
 373
 374             next if length($line) == 0; # split would result in undef then...
 375
 376             ($line) = split(/#/, $line);
 377
 378             next if $line !~ m/^deb[[:space:]]/; # is case sensitive
 379
 380             my $suite;
 381             if ($line =~ m|deb\s+\w+://\S+\s+(\S*)|i) {
 382                 $suite = $1;
 383             } else {
 384                 next;
 385             }
 386             my $where = "in ${file}:${number}";
 387
 388             $suite =~ s/-(?:(?:proposed-)?updates|backports|security)$//;
 389             if ($suite ne $old_suite && $suite ne $new_suite) {
 390                 log_notice(
 391                     "found unusual suite '$suite', neither old '$old_suite' nor new '$new_suite'.."
 392                     ."\n    Affected file:line $where"
 393                     ."\n    Please assure this is shipping compatible packages for the upgrade!"
 394                 );
 395                 $strange_suite = 1;
 396                 next;
 397             }
 398
 399             if (!defined($found_suite)) {
 400                 $found_suite = $suite;
 401                 $found_suite_where = $where;
 402             } elsif ($suite ne $found_suite) {
 403                 if (!defined($mismatches)) {
 404                     $mismatches = [];
 405                     push $mismatches->@*,
 406                         { suite => $found_suite, where => $found_suite_where},
 407                         { suite => $suite, where => $where};
 408                 } else {
 409                     push $mismatches->@*, { suite => $suite, where => $where};
 410                 }
 411             }
 412         }
 413     };
 414
 415     $check_file->("/etc/apt/sources.list");
 416
 417     $in_dir = 1;
 418
 419     PVE::Tools::dir_glob_foreach($dir, '^.*\.list$', $check_file);
 420
 421     if (defined($mismatches)) {
 422         my @mismatch_list = map { "found suite $_->{suite} at $_->{where}" } $mismatches->@*;
 423
 424         log_fail(
 425             "Found mixed old and new package repository suites, fix before upgrading! Mismatches:"
 426             ."\n    " . join("\n    ", @mismatch_list)
 427         );
 428     } elsif ($strange_suite) {
 429         log_notice("found no suite mismatches, but found at least one strange suite");
 430     } else {
 431         log_pass("found no suite mismatch");
 432     }
 433 }
 434
 435 sub check_time_sync {
 436     my $unit_active = sub { return $get_systemd_unit_state->($_[0], 1) eq 'active' ? $_[0] : undef };
 437
 438     log_info("Checking for supported & active NTP service..");
 439     if ($unit_active->('systemd-timesyncd.service')) {
 440         log_warn(
 441             "systemd-timesyncd is not the best choice for time-keeping on servers, due to only applying"
 442             ." updates on boot.\n  It's recommended to use one of:\n"
 443             ."    * chrony\n    * ntpsec\n    * openntpd\n"
 444         );
 445     } elsif ($unit_active->('ntp.service')) {
 446         log_info("Debian deprecated and removed the ntp package for Bookworm, but the system"
 447             ." will automatically migrate to the 'ntpsec' replacement package on upgrade.");
 448     } elsif (my $active_ntp = ($unit_active->('chrony.service') || $unit_active->('openntpd.service') || $unit_active->('ntpsec.service'))) {
 449         log_pass("Detected active time synchronisation unit '$active_ntp'");
 450     } else {
 451         log_notice("No (active) time synchronisation daemon (NTP) detected");
 452     }
 453 }
 454
 455 sub check_bootloader {
 456     log_info("Checking bootloader configuration...");
 457     if (!$upgraded) {
 458         log_skip("not yet upgraded, no need to check the presence of systemd-boot");
 459         return;
 460     }
 461
 462     if (! -f "/etc/kernel/proxmox-boot-uuids") {
 463         log_skip("proxmox-boot-tool not used for bootloader configuration");
 464         return;
 465     }
 466
 467     if (! -d "/sys/firmware/efi") {
 468         log_skip("System booted in legacy-mode - no need for systemd-boot");
 469         return;
 470     }
 471
 472     if ( -f "/usr/share/doc/systemd-boot/changelog.Debian.gz") {
 473         log_pass("systemd-boot is installed");
 474     } else {
 475         log_warn(
 476             "proxmox-boot-tool is used for bootloader configuration in uefi mode"
 477             . "but the separate systemd-boot package, existing in Debian Bookworm  is not installed"
 478             . "initializing new ESPs will not work until the package is installed"
 479         );
 480     }
 481 }
 482
 483 sub check_misc {
 484     print_header("MISCELLANEOUS CHECKS");
 485     my $ssh_config = eval { PVE::Tools::file_get_contents('/root/.ssh/config') };
 486     if (defined($ssh_config)) {
 487         log_fail("Unsupported SSH Cipher configured for root in /root/.ssh/config: $1")
 488             if $ssh_config =~ /^Ciphers .*(blowfish|arcfour|3des).*$/m;
 489     } else {
 490         log_skip("No SSH config file found.");
 491     }
 492
 493     check_time_sync();
 494
 495     my $root_free = PVE::Tools::df('/', 10);
 496     log_warn("Less than 5 GB free space on root file system.")
 497         if defined($root_free) && $root_free->{avail} < 5 * 1000*1000*1000;
 498
 499     log_info("Checking if the local node's hostname '$nodename' is resolvable..");
 500     my $local_ip = eval { PVE::Network::get_ip_from_hostname($nodename) };
 501     if ($@) {
 502         log_warn("Failed to resolve hostname '$nodename' to IP - $@");
 503     } else {
 504         log_info("Checking if resolved IP is configured on local node..");
 505         my $cidr = Net::IP::ip_is_ipv6($local_ip) ? "$local_ip/128" : "$local_ip/32";
 506         my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
 507         my $ip_count = scalar(@$configured_ips);
 508
 509         if ($ip_count <= 0) {
 510             log_fail("Resolved node IP '$local_ip' not configured or active for '$nodename'");
 511         } elsif ($ip_count > 1) {
 512             log_warn("Resolved node IP '$local_ip' active on multiple ($ip_count) interfaces!");
 513         } else {
 514             log_pass("Resolved node IP '$local_ip' configured and active on single interface.");
 515         }
 516     }
 517
 518     log_info("Check node certificate's RSA key size");
 519     my $certs = PMG::API2::Certificates->info({ node => $nodename });
 520     my $certs_check = {
 521         'rsaEncryption' => {
 522             minsize => 2048,
 523             name => 'RSA',
 524         },
 525         'id-ecPublicKey' => {
 526             minsize => 224,
 527             name => 'ECC',
 528         },
 529     };
 530
 531     my $certs_check_failed = 0;
 532     for my $cert (@$certs) {
 533         my ($type, $size, $fn) = $cert->@{qw(public-key-type public-key-bits filename)};
 534
 535         if (!defined($type) || !defined($size)) {
 536             log_warn("'$fn': cannot check certificate, failed to get it's type or size!");
 537         }
 538
 539         my $check = $certs_check->{$type};
 540         if (!defined($check)) {
 541             log_warn("'$fn': certificate's public key type '$type' unknown!");
 542             next;
 543         }
 544
 545         if ($size < $check->{minsize}) {
 546             log_fail("'$fn', certificate's $check->{name} public key size is less than 2048 bit");
 547             $certs_check_failed = 1;
 548         } else {
 549             log_pass("Certificate '$fn' passed Debian Busters (and newer) security level for TLS connections ($size >= 2048)");
 550         }
 551     }
 552
 553     check_apt_repos();
 554     check_bootloader();
 555
 556     my ($template_dir, $base_dir) = ('/etc/pmg/templates/', '/var/lib/pmg/templates');
 557     my @override_but_unmodified = ();
 558     PVE::Tools::dir_glob_foreach($base_dir, '.*\.(?:tt|in).*', sub {
 559         my ($filename) = @_;
 560         return if !-e "$template_dir/$filename";
 561
 562         my $shipped = PVE::Tools::file_get_contents("$base_dir/$filename", 1024*1024);
 563         my $override = PVE::Tools::file_get_contents("$template_dir/$filename", 1024*1024);
 564
 565         push @override_but_unmodified, $filename if $shipped eq $override;
 566     });
 567     if (scalar(@override_but_unmodified)) {
 568         my $msg = "Found overrides in '/etc/pmg/templates/' for template, but without modification."
 569             ." Consider simply removing them: \n    "
 570             . join("\n    ", @override_but_unmodified);
 571         log_notice($msg);
 572     }
 573 }
 574
 575 my sub colored_if {
 576     my ($str, $color, $condition) = @_;
 577     return "". ($condition ? colored($str, $color) : $str);
 578 }
 579
 580 __PACKAGE__->register_method ({
 581     name => 'checklist',
 582     path => 'checklist',
 583     method => 'GET',
 584     description => 'Check (pre-/post-)upgrade conditions.',
 585     parameters => {
 586         additionalProperties => 0,
 587         properties => {},
 588     },
 589     returns => { type => 'null' },
 590     code => sub {
 591         my ($param) = @_;
 592
 593         check_pmg_packages();
 594         check_cluster_status();
 595         my $upgraded_db = check_running_postgres();
 596         check_services_disabled($upgraded_db);
 597         check_misc();
 598
 599         print_header("SUMMARY");
 600
 601         my $total = 0;
 602         $total += $_ for values %$counters;
 603
 604         print "TOTAL:    $total\n";
 605         print colored("PASSED:   $counters->{pass}\n", 'green');
 606         print "SKIPPED:  $counters->{skip}\n";
 607         print colored_if("WARNINGS: $counters->{warn}\n", 'yellow', $counters->{warn} > 0);
 608         print colored_if("FAILURES: $counters->{fail}\n", 'bold red', $counters->{fail} > 0);
 609
 610         if ($counters->{warn} > 0 || $counters->{fail} > 0) {
 611             my $color = $counters->{fail} > 0 ? 'bold red' : 'yellow';
 612             print colored("\nATTENTION: Please check the output for detailed information!\n", $color);
 613             print colored("Try to solve the problems one at a time and then run this checklist tool again.\n", $color) if $counters->{fail} > 0;
 614         }
 615
 616         return undef;
 617     }});
 618
 619 our $cmddef = [ __PACKAGE__, 'checklist', [], {}];
 620
 621 1;