PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Clone;
   7 use Fcntl ':flock';
   8 use File::Basename;
   9 use File::Path;
  10 use IO::File;
  11 use IO::Select;
  12 use IPC::Open3;
  13 use POSIX qw(strftime);
  14 use Time::Local;
  15
  16 use PVE::Cluster qw(cfs_read_file);
  17 use PVE::DataCenterConfig;
  18 use PVE::Exception qw(raise_param_exc);
  19 use PVE::HA::Config;
  20 use PVE::HA::Env::PVE2;
  21 use PVE::JSONSchema qw(get_standard_option);
  22 use PVE::Notify;
  23 use PVE::RPCEnvironment;
  24 use PVE::Storage;
  25 use PVE::VZDump::Common;
  26 use PVE::VZDump::Plugin;
  27 use PVE::Tools qw(extract_param split_list);
  28 use PVE::API2Tools;
  29
  30 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  31
  32 my $lockfile = '/var/run/vzdump.lock';
  33 my $pidfile = '/var/run/vzdump.pid';
  34 my $logdir = '/var/log/vzdump';
  35
  36 my @plugins = qw();
  37
  38 my $confdesc = PVE::VZDump::Common::get_confdesc();
  39
  40 my $confdesc_for_defaults = Clone::clone($confdesc);
  41 delete $confdesc_for_defaults->{$_}->{requires} for qw(notes-template protected);
  42
  43 # Load available plugins
  44 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
  45 foreach my $plug (@pve_vzdump_classes) {
  46     my $filename = "/usr/share/perl5/$plug.pm";
  47     $filename =~ s!::!/!g;
  48     if (-f $filename) {
  49         eval { require $filename; };
  50         if (!$@) {
  51             $plug->import ();
  52             push @plugins, $plug;
  53         } else {
  54             die $@;
  55         }
  56     }
  57 }
  58
  59 sub get_storage_param {
  60     my ($param) = @_;
  61
  62     return if $param->{dumpdir};
  63     return $param->{storage} || 'local';
  64 }
  65
  66 # helper functions
  67
  68 sub debugmsg {
  69     my ($mtype, $msg, $logfd, $syslog) = @_;
  70
  71     PVE::VZDump::Plugin::debugmsg(@_);
  72 }
  73
  74 sub run_command {
  75     my ($logfd, $cmdstr, %param) = @_;
  76
  77     my $logfunc = sub {
  78         my $line = shift;
  79         debugmsg ('info', $line, $logfd);
  80     };
  81
  82     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
  83 }
  84
  85 my $verify_notes_template = sub {
  86     my ($template) = @_;
  87
  88     die "contains a line feed\n" if $template =~ /\n/;
  89
  90     my @problematic = ();
  91     while ($template =~ /\\(.)/g) {
  92         my $char = $1;
  93         push @problematic, "escape sequence '\\$char' at char " . (pos($template) - 2)
  94             if $char !~ /^[n\\]$/;
  95     }
  96
  97     while ($template =~ /\{\{([^\s{}]+)\}\}/g) {
  98         my $var = $1;
  99         push @problematic, "variable '$var' at char " . (pos($template) - length($var))
 100             if $var !~ /^(cluster|guestname|node|vmid)$/;
 101     }
 102
 103     die "found unknown: " . join(', ', @problematic) . "\n" if scalar(@problematic);
 104 };
 105
 106 my $generate_notes = sub {
 107     my ($notes_template, $task) = @_;
 108
 109     $verify_notes_template->($notes_template);
 110
 111     my $info = {
 112         cluster => PVE::Cluster::get_clinfo()->{cluster}->{name} // 'standalone node',
 113         guestname => $task->{hostname} // "VM $task->{vmid}", # is always set for CTs
 114         node => PVE::INotify::nodename(),
 115         vmid => $task->{vmid},
 116     };
 117
 118     my $unescape = sub {
 119         my ($char) = @_;
 120         return '\\' if $char eq '\\';
 121         return "\n" if $char eq 'n';
 122         die "unexpected escape character '$char'\n";
 123     };
 124
 125     $notes_template =~ s/\\(.)/$unescape->($1)/eg;
 126
 127     my $vars = join('|', keys $info->%*);
 128     $notes_template =~ s/\{\{($vars)\}\}/$info->{$1}/g;
 129
 130     return $notes_template;
 131 };
 132
 133 my sub parse_performance {
 134     my ($param) = @_;
 135
 136     if (defined(my $perf = $param->{performance})) {
 137         return if ref($perf) eq 'HASH'; # already parsed
 138         $param->{performance} = PVE::JSONSchema::parse_property_string('backup-performance', $perf);
 139     }
 140 }
 141
 142 my $parse_prune_backups_maxfiles = sub {
 143     my ($param, $kind) = @_;
 144
 145     my $maxfiles = delete $param->{maxfiles};
 146     my $prune_backups = $param->{'prune-backups'};
 147
 148     debugmsg('warn', "both 'maxfiles' and 'prune-backups' defined as ${kind} - ignoring 'maxfiles'")
 149         if defined($maxfiles) && defined($prune_backups);
 150
 151     if (defined($prune_backups)) {
 152         return if ref($prune_backups) eq 'HASH'; # already parsed
 153         $param->{'prune-backups'} = PVE::JSONSchema::parse_property_string(
 154             'prune-backups',
 155             $prune_backups
 156         );
 157     } elsif (defined($maxfiles)) {
 158         if ($maxfiles) {
 159             $param->{'prune-backups'} = { 'keep-last' => $maxfiles };
 160         } else {
 161             $param->{'prune-backups'} = { 'keep-all' => 1 };
 162         }
 163     }
 164 };
 165
 166 sub storage_info {
 167     my $storage = shift;
 168
 169     my $cfg = PVE::Storage::config();
 170     my $scfg = PVE::Storage::storage_config($cfg, $storage);
 171     my $type = $scfg->{type};
 172
 173     die "can't use storage '$storage' for backups - wrong content type\n"
 174         if (!$scfg->{content}->{backup});
 175
 176     my $info = {
 177         scfg => $scfg,
 178     };
 179
 180     $info->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'})
 181         if defined($scfg->{'prune-backups'});
 182
 183     if ($type eq 'pbs') {
 184         $info->{pbs} = 1;
 185     } else {
 186         $info->{dumpdir} = PVE::Storage::get_backup_dir($cfg, $storage);
 187     }
 188
 189     return $info;
 190 }
 191
 192 sub format_size {
 193     my $size = shift;
 194
 195     my $kb = $size / 1024;
 196
 197     if ($kb < 1024) {
 198         return int ($kb) . "KB";
 199     }
 200
 201     my $mb = $size / (1024*1024);
 202     if ($mb < 1024) {
 203         return int ($mb) . "MB";
 204     }
 205     my $gb = $mb / 1024;
 206     if ($gb < 1024) {
 207         return sprintf ("%.2fGB", $gb);
 208     }
 209     my $tb = $gb / 1024;
 210     return sprintf ("%.2fTB", $tb);
 211 }
 212
 213 sub format_time {
 214     my $seconds = shift;
 215
 216     my $hours = int ($seconds/3600);
 217     $seconds = $seconds - $hours*3600;
 218     my $min = int ($seconds/60);
 219     $seconds = $seconds - $min*60;
 220
 221     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 222 }
 223
 224 sub encode8bit {
 225     my ($str) = @_;
 226
 227     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 228
 229     return $str;
 230 }
 231
 232 sub escape_html {
 233     my ($str) = @_;
 234
 235     $str =~ s/&/&amp;/g;
 236     $str =~ s/</&lt;/g;
 237     $str =~ s/>/&gt;/g;
 238
 239     return $str;
 240 }
 241
 242 sub check_bin {
 243     my ($bin)  = @_;
 244
 245     foreach my $p (split (/:/, $ENV{PATH})) {
 246         my $fn = "$p/$bin";
 247         if (-x $fn) {
 248             return $fn;
 249         }
 250     }
 251
 252     die "unable to find command '$bin'\n";
 253 }
 254
 255 sub check_vmids {
 256     my (@vmids) = @_;
 257
 258     my $res = [];
 259     for my $vmid (sort {$a <=> $b} @vmids) {
 260         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 261         $vmid = int ($vmid); # remove leading zeros
 262         next if !$vmid;
 263         push @$res, $vmid;
 264     }
 265
 266     return $res;
 267 }
 268
 269
 270 sub read_vzdump_defaults {
 271
 272     my $fn = "/etc/vzdump.conf";
 273
 274     my $defaults = {
 275         map {
 276             my $default = $confdesc->{$_}->{default};
 277              defined($default) ? ($_ => $default) : ()
 278         } keys %$confdesc_for_defaults
 279     };
 280     $parse_prune_backups_maxfiles->($defaults, "defaults in VZDump schema");
 281     parse_performance($defaults);
 282
 283     my $raw;
 284     eval { $raw = PVE::Tools::file_get_contents($fn); };
 285     return $defaults if $@;
 286
 287     my $conf_schema = { type => 'object', properties => $confdesc_for_defaults };
 288     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 289     if (my $excludes = $res->{'exclude-path'}) {
 290         if (ref($excludes) eq 'ARRAY') {
 291             my $list = [];
 292             for my $path ($excludes->@*) {
 293                 # We still use `split_args` here to be compatible with old configs where one line
 294                 # still has multiple space separated entries.
 295                 push $list->@*, PVE::Tools::split_args($path)->@*;
 296             }
 297             $res->{'exclude-path'} = $list;
 298         } else {
 299             $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 300         }
 301     }
 302     if (defined($res->{mailto})) {
 303         my @mailto = split_list($res->{mailto});
 304         $res->{mailto} = [ @mailto ];
 305     }
 306     $parse_prune_backups_maxfiles->($res, "options in '$fn'");
 307     parse_performance($res);
 308
 309     foreach my $key (keys %$defaults) {
 310         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 311     }
 312
 313     if (defined($res->{storage}) && defined($res->{dumpdir})) {
 314         debugmsg('warn', "both 'storage' and 'dumpdir' defined in '$fn' - ignoring 'dumpdir'");
 315         delete $res->{dumpdir};
 316     }
 317
 318     return $res;
 319 }
 320
 321 sub read_backup_task_logs {
 322     my ($task_list) = @_;
 323
 324     my $task_logs = "";
 325
 326     for my $task (@$task_list) {
 327         my $vmid = $task->{vmid};
 328         my $log_file = $task->{tmplog};
 329         if (!$task->{tmplog}) {
 330             $task_logs .= "$vmid: no log available\n\n";
 331             next;
 332         }
 333         if (open (my $TMP, '<', "$log_file")) {
 334             while (my $line = <$TMP>) {
 335                 next if $line =~ /^status: \d+/; # not useful in mails
 336                 $task_logs .= encode8bit ("$vmid: $line");
 337             }
 338             close ($TMP);
 339         } else {
 340             $task_logs .= "$vmid: Could not open log file\n\n";
 341         }
 342         $task_logs .= "\n";
 343     }
 344
 345     return $task_logs;
 346 }
 347
 348 sub build_guest_table {
 349     my ($task_list) = @_;
 350
 351     my $table = {
 352         schema => {
 353             columns => [
 354                 {
 355                     label => "VMID",
 356                     id  => "vmid"
 357                 },
 358                 {
 359                     label => "Name",
 360                     id  => "name"
 361                 },
 362                 {
 363                     label => "Status",
 364                     id  => "status"
 365                 },
 366                 {
 367                     label => "Time",
 368                     id  => "time",
 369                     renderer => "duration"
 370                 },
 371                 {
 372                     label => "Size",
 373                     id  => "size",
 374                     renderer => "human-bytes"
 375                 },
 376                 {
 377                     label => "Filename",
 378                     id  => "filename"
 379                 },
 380             ]
 381         },
 382         data => []
 383     };
 384
 385     for my $task (@$task_list) {
 386         my $successful = $task->{state} eq 'ok';
 387         my $size = $successful ? $task->{size} : 0;
 388         my $filename = $successful ? $task->{target} : undef;
 389         push @{$table->{data}}, {
 390             "vmid" => int($task->{vmid}),
 391             "name" => $task->{hostname},
 392             "status" => $task->{state},
 393             "time" => int($task->{backuptime}),
 394             "size" => int($size),
 395             "filename" => $filename,
 396         };
 397     }
 398
 399     return $table;
 400 }
 401
 402 sub sanitize_task_list {
 403     my ($task_list) = @_;
 404     for my $task (@$task_list) {
 405         chomp $task->{msg} if $task->{msg};
 406         $task->{backuptime} = 0 if !$task->{backuptime};
 407         $task->{size} = 0 if !$task->{size};
 408         $task->{target} = 'unknown' if !$task->{target};
 409         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 410
 411         if ($task->{state} eq 'todo') {
 412             $task->{msg} = 'aborted';
 413         }
 414     }
 415 }
 416
 417 sub count_failed_tasks {
 418     my ($tasklist) = @_;
 419
 420     my $error_count = 0;
 421     for my $task (@$tasklist) {
 422         $error_count++ if $task->{state} ne 'ok';
 423     }
 424
 425     return $error_count;
 426 }
 427
 428 sub get_hostname {
 429     my $hostname = `hostname -f` || PVE::INotify::nodename();
 430     chomp $hostname;
 431     return $hostname;
 432 }
 433
 434 my $subject_template = "vzdump backup status ({{hostname}}): {{status-text}}";
 435
 436 my $body_template = <<EOT;
 437 {{error-message}}
 438 {{heading-1 "Details"}}
 439 {{table guest-table}}
 440
 441 Total running time: {{duration total-time}}
 442
 443 {{heading-1 "Logs"}}
 444 {{verbatim-monospaced logs}}
 445 EOT
 446
 447 use constant MAX_LOG_SIZE => 1024*1024;
 448
 449 sub send_notification {
 450     my ($self, $tasklist, $total_time, $err, $detail_pre, $detail_post) = @_;
 451
 452     my $opts = $self->{opts};
 453     my $mailto = $opts->{mailto};
 454     my $cmdline = $self->{cmdline};
 455     my $target = $opts->{"notification-target"};
 456     # Fall back to 'mailnotification' if 'notification-policy' is not set.
 457     # If both are set, 'notification-policy' takes precedence
 458     my $policy = $opts->{"notification-policy"} // $opts->{mailnotification} // 'always';
 459
 460     return if ($policy eq 'never');
 461
 462     sanitize_task_list($tasklist);
 463     my $error_count = count_failed_tasks($tasklist);
 464
 465     my $failed = ($error_count || $err);
 466
 467     return if (!$failed && ($policy eq 'failure'));
 468
 469     my $status_text = $failed ? 'backup failed' : 'backup successful';
 470
 471     if ($err) {
 472         if ($err =~ /\n/) {
 473             $status_text .= ": multiple problems";
 474         } else {
 475             $status_text .= ": $err";
 476             $err = undef;
 477         }
 478     }
 479
 480     my $text_log_part = "$cmdline\n\n";
 481     $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
 482     $text_log_part .= read_backup_task_logs($tasklist);
 483     $text_log_part .= $detail_post if defined($detail_post);
 484
 485     if (length($text_log_part)  > MAX_LOG_SIZE)
 486     {
 487         # Let's limit the maximum length of included logs
 488         $text_log_part = "Log output was too long to be sent. ".
 489             "See Task History for details!\n";
 490     };
 491
 492     my $notification_props = {
 493         "hostname"      => get_hostname(),
 494         "error-message" => $err,
 495         "guest-table"   => build_guest_table($tasklist),
 496         "logs"          => $text_log_part,
 497         "status-text"   => $status_text,
 498         "total-time"    => $total_time,
 499     };
 500
 501     my $notification_config = PVE::Notify::read_config();
 502
 503     if ($mailto && scalar(@$mailto)) {
 504         # <, >, @ are not allowed in endpoint names, but that is only
 505         # verified once the config is serialized. That means that
 506         # we can rely on that fact that no other endpoint with this name exists.
 507         my $endpoint_name = "<mail-to-" . join(",", @$mailto) . ">";
 508         $notification_config->add_sendmail_endpoint(
 509             $endpoint_name,
 510             $mailto,
 511             undef,
 512             undef,
 513             "vzdump backup tool");
 514
 515         my $endpoints = [$endpoint_name];
 516
 517         # Create an anonymous group containing the sendmail endpoint and the
 518         # $target endpoint, if specified
 519         if ($target) {
 520             push @$endpoints, $target;
 521         }
 522
 523         $target = "<group-$endpoint_name>";
 524         $notification_config->add_group(
 525             $target,
 526             $endpoints,
 527         );
 528     }
 529
 530     return if (!$target);
 531
 532     my $severity = $failed ? "error" : "info";
 533
 534     PVE::Notify::notify(
 535         $target,
 536         $severity,
 537         $subject_template,
 538         $body_template,
 539         $notification_props,
 540         $notification_config
 541     );
 542 };
 543
 544 sub new {
 545     my ($class, $cmdline, $opts, $skiplist) = @_;
 546
 547     mkpath $logdir;
 548
 549     check_bin ('cp');
 550     check_bin ('df');
 551     check_bin ('sendmail');
 552     check_bin ('rsync');
 553     check_bin ('tar');
 554     check_bin ('mount');
 555     check_bin ('umount');
 556     check_bin ('cstream');
 557     check_bin ('ionice');
 558
 559     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 560         check_bin ('lvcreate');
 561         check_bin ('lvs');
 562         check_bin ('lvremove');
 563     }
 564
 565     my $defaults = read_vzdump_defaults();
 566
 567     foreach my $k (keys %$defaults) {
 568         next if $k eq 'exclude-path' || $k eq 'prune-backups'; # dealt with separately
 569         if ($k eq 'dumpdir' || $k eq 'storage') {
 570             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 571                 !defined ($opts->{storage});
 572         } else {
 573             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 574         }
 575     }
 576
 577     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 578     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 579
 580     $skiplist = [] if !$skiplist;
 581     my $self = bless {
 582         cmdline => $cmdline,
 583         opts => $opts,
 584         skiplist => $skiplist,
 585     }, $class;
 586
 587     my $findexcl = $self->{findexcl} = [];
 588     if ($defaults->{'exclude-path'}) {
 589         push @$findexcl, @{$defaults->{'exclude-path'}};
 590     }
 591
 592     if ($opts->{'exclude-path'}) {
 593         push @$findexcl, @{$opts->{'exclude-path'}};
 594     }
 595
 596     if ($opts->{stdexcludes}) {
 597         push @$findexcl,
 598             '/tmp/?*',
 599             '/var/tmp/?*',
 600             '/var/run/?*.pid',
 601             ;
 602     }
 603
 604     foreach my $p (@plugins) {
 605         my $pd = $p->new($self);
 606
 607         push @{$self->{plugins}}, $pd;
 608     }
 609
 610     if (defined($opts->{storage}) && $opts->{stdout}) {
 611         die "cannot use options 'storage' and 'stdout' at the same time\n";
 612     } elsif (defined($opts->{storage}) && defined($opts->{dumpdir})) {
 613         die "cannot use options 'storage' and 'dumpdir' at the same time\n";
 614     }
 615
 616     if (my $storage = get_storage_param($opts)) {
 617         $opts->{storage} = $storage;
 618     }
 619
 620     # Enforced by the API too, but these options might come in via defaults. Drop them if necessary.
 621     if (!$opts->{storage}) {
 622         delete $opts->{$_} for qw(notes-template protected);
 623     }
 624
 625     my $errors = '';
 626     my $add_error = sub {
 627         my ($error) = @_;
 628         $errors .= "\n" if $errors;
 629         chomp($error);
 630         $errors .= $error;
 631     };
 632
 633     eval {
 634         $self->{job_init_log} = '';
 635         open my $job_init_fd, '>', \$self->{job_init_log};
 636         $self->run_hook_script('job-init', undef, $job_init_fd);
 637         close $job_init_fd;
 638
 639         PVE::Cluster::cfs_update(); # Pick up possible changes made by the hook script.
 640     };
 641     $add_error->($@) if $@;
 642
 643     if ($opts->{storage}) {
 644         my $storage_cfg = PVE::Storage::config();
 645         eval { PVE::Storage::activate_storage($storage_cfg, $opts->{storage}) };
 646         $add_error->("could not activate storage '$opts->{storage}': $@") if $@;
 647
 648         my $info = eval { storage_info ($opts->{storage}) };
 649         if (my $err = $@) {
 650             $add_error->("could not get storage information for '$opts->{storage}': $err");
 651         } else {
 652             $opts->{dumpdir} = $info->{dumpdir};
 653             $opts->{scfg} = $info->{scfg};
 654             $opts->{pbs} = $info->{pbs};
 655             $opts->{'prune-backups'} //= $info->{'prune-backups'};
 656         }
 657     } elsif ($opts->{dumpdir}) {
 658         $add_error->("dumpdir '$opts->{dumpdir}' does not exist")
 659             if ! -d $opts->{dumpdir};
 660     } else {
 661         die "internal error";
 662     }
 663
 664     $opts->{'prune-backups'} //= $defaults->{'prune-backups'};
 665
 666     # avoid triggering any remove code path if keep-all is set
 667     $opts->{remove} = 0 if $opts->{'prune-backups'}->{'keep-all'};
 668
 669     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 670         $add_error->("tmpdir '$opts->{tmpdir}' does not exist");
 671     }
 672
 673     if ($errors) {
 674         eval { $self->send_notification([], 0, $errors); };
 675         debugmsg ('err', $@) if $@;
 676         die "$errors\n";
 677     }
 678
 679     return $self;
 680 }
 681
 682 sub get_mount_info {
 683     my ($dir) = @_;
 684
 685     # Note: df 'available' can be negative, and percentage set to '-'
 686
 687     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 688
 689     my $res;
 690
 691     my $parser = sub {
 692         my $line = shift;
 693         if (my ($fsid, $fstype, undef, $mp) = $line =~
 694             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 695             $res = {
 696                 device => $fsid,
 697                 fstype => $fstype,
 698                 mountpoint => $mp,
 699             };
 700         }
 701     };
 702
 703     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 704     warn $@ if $@;
 705
 706     return $res;
 707 }
 708
 709 sub getlock {
 710     my ($self, $upid) = @_;
 711
 712     my $fh;
 713
 714     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 715
 716     die "missing UPID" if !$upid; # should not happen
 717
 718     my $SERVER_FLCK;
 719     if (!open ($SERVER_FLCK, '>>', "$lockfile")) {
 720         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 721         die "can't open lock on file '$lockfile' - $!";
 722     }
 723
 724     if (!flock ($SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 725         if (!$maxwait) {
 726             debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
 727             die "can't acquire lock '$lockfile' (wait = 0)";
 728         }
 729
 730         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 731         eval {
 732             alarm ($maxwait * 60);
 733
 734             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 735
 736             if (!flock ($SERVER_FLCK, LOCK_EX)) {
 737                 my $err = $!;
 738                 close ($SERVER_FLCK);
 739                 alarm (0);
 740                 die "$err\n";
 741             }
 742             alarm (0);
 743         };
 744         alarm (0);
 745
 746         my $err = $@;
 747
 748         if ($err) {
 749             debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
 750             die "can't acquire lock '$lockfile' - $err";
 751         }
 752
 753         debugmsg('info', "got global lock", undef, 1);
 754     }
 755
 756     PVE::Tools::file_set_contents($pidfile, $upid);
 757
 758     return $SERVER_FLCK;
 759 }
 760
 761 sub run_hook_script {
 762     my ($self, $phase, $task, $logfd) = @_;
 763
 764     my $opts = $self->{opts};
 765
 766     my $script = $opts->{script};
 767     return if !$script;
 768
 769     die "Error: The hook script '$script' does not exist.\n" if ! -f $script;
 770     die "Error: The hook script '$script' is not executable.\n" if ! -x $script;
 771
 772     my $cmd = [$script, $phase];
 773
 774     if ($task) {
 775         push @$cmd, $task->{mode};
 776         push @$cmd, $task->{vmid};
 777     }
 778
 779     local %ENV;
 780     # set immutable opts directly (so they are available in all phases)
 781     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 782     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 783
 784     foreach my $ek (qw(vmtype hostname target logfile)) {
 785         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 786     }
 787
 788     run_command ($logfd, $cmd);
 789 }
 790
 791 sub compressor_info {
 792     my ($opts) = @_;
 793     my $opt_compress = $opts->{compress};
 794
 795     if (!$opt_compress || $opt_compress eq '0') {
 796         return undef;
 797     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 798         return ('lzop', 'lzo');
 799     } elsif ($opt_compress eq 'gzip') {
 800         if ($opts->{pigz} > 0) {
 801             my $pigz_threads = $opts->{pigz};
 802             if ($pigz_threads == 1) {
 803                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 804                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 805             }
 806             return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
 807         } else {
 808             return ('gzip --rsyncable', 'gz');
 809         }
 810     } elsif ($opt_compress eq 'zstd') {
 811         my $zstd_threads = $opts->{zstd} // 1;
 812         if ($zstd_threads == 0) {
 813             my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 814             $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
 815         }
 816         return ("zstd --threads=${zstd_threads}", 'zst');
 817     } else {
 818         die "internal error - unknown compression option '$opt_compress'";
 819     }
 820 }
 821
 822 sub get_backup_file_list {
 823     my ($dir, $bkname) = @_;
 824
 825     my $bklist = [];
 826     foreach my $fn (<$dir/${bkname}-*>) {
 827         my $archive_info = eval { PVE::Storage::archive_info($fn) } // {};
 828         if ($archive_info->{is_std_name}) {
 829             my $path = "$dir/$archive_info->{filename}";
 830             my $backup = {
 831                 'path' => $path,
 832                 'ctime' => $archive_info->{ctime},
 833             };
 834             $backup->{mark} = "protected"
 835                 if -e PVE::Storage::protection_file_path($path);
 836             push @{$bklist}, $backup;
 837         }
 838     }
 839
 840     return $bklist;
 841 }
 842
 843 sub exec_backup_task {
 844     my ($self, $task) = @_;
 845
 846     my $opts = $self->{opts};
 847
 848     my $cfg = PVE::Storage::config();
 849     my $vmid = $task->{vmid};
 850     my $plugin = $task->{plugin};
 851
 852     $task->{backup_time} = time();
 853
 854     my $pbs_group_name;
 855     my $pbs_snapshot_name;
 856
 857     my $vmstarttime = time ();
 858
 859     my $logfd;
 860
 861     my $cleanup = {};
 862
 863     my $log_vm_online_again = sub {
 864         return if !defined($task->{vmstoptime});
 865         $task->{vmconttime} //= time();
 866         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 867         $delay = '<1' if $delay < 1;
 868         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 869     };
 870
 871     eval {
 872         die "unable to find VM '$vmid'\n" if !$plugin;
 873
 874         my $vmtype = $plugin->type();
 875
 876         if ($self->{opts}->{pbs}) {
 877             if ($vmtype eq 'lxc') {
 878                 $pbs_group_name = "ct/$vmid";
 879             } elsif  ($vmtype eq 'qemu') {
 880                 $pbs_group_name = "vm/$vmid";
 881             } else {
 882                 die "pbs backup not implemented for plugin type '$vmtype'\n";
 883             }
 884             my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
 885             $pbs_snapshot_name = "$pbs_group_name/$btime";
 886         }
 887
 888         # for now we deny backups of a running ha managed service in *stop* mode
 889         # as it interferes with the HA stack (started services should not stop).
 890         if ($opts->{mode} eq 'stop' &&
 891             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 892         {
 893             die "Cannot execute a backup with stop mode on a HA managed and".
 894                 " enabled Service. Use snapshot mode or disable the Service.\n";
 895         }
 896
 897         my $tmplog = "$logdir/$vmtype-$vmid.log";
 898
 899         my $bkname = "vzdump-$vmtype-$vmid";
 900         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
 901
 902         my $prune_options = $opts->{'prune-backups'};
 903
 904         my $backup_limit = 0;
 905         if (!$prune_options->{'keep-all'}) {
 906             foreach my $keep (values %{$prune_options}) {
 907                 $backup_limit += $keep;
 908             }
 909         }
 910
 911         if (($backup_limit && !$opts->{remove}) || $opts->{protected}) {
 912             my $count;
 913             my $protected_count;
 914             if (my $storeid = $opts->{storage}) {
 915                 my @backups = grep {
 916                     !$_->{subtype} || $_->{subtype} eq $vmtype
 917                 } PVE::Storage::volume_list($cfg, $storeid, $vmid, 'backup')->@*;
 918
 919                 $count = grep { !$_->{protected} } @backups;
 920                 $protected_count = scalar(@backups) - $count;
 921             } else {
 922                 $count = grep { !$_->{mark} || $_->{mark} ne "protected" } get_backup_file_list($opts->{dumpdir}, $bkname)->@*;
 923             }
 924
 925             if ($opts->{protected}) {
 926                 my $max_protected = PVE::Storage::get_max_protected_backups(
 927                     $opts->{scfg},
 928                     $opts->{storage},
 929                 );
 930                 if ($max_protected > -1 && $protected_count >= $max_protected) {
 931                     die "The number of protected backups per guest is limited to $max_protected ".
 932                         "on storage '$opts->{storage}'\n";
 933                 }
 934             } elsif ($count >= $backup_limit) {
 935                 die "There is a max backup limit of $backup_limit enforced by the target storage ".
 936                     "or the vzdump parameters. Either increase the limit or delete old backups.\n";
 937             }
 938         }
 939
 940         if (!$self->{opts}->{pbs}) {
 941             $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 942         }
 943
 944         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 945         my ($comp, $comp_ext) = compressor_info($opts);
 946         if ($comp && $comp_ext) {
 947             $ext .= ".${comp_ext}";
 948         }
 949
 950         if ($self->{opts}->{pbs}) {
 951             die "unable to pipe backup to stdout\n" if $opts->{stdout};
 952             $task->{target} = $pbs_snapshot_name;
 953         } else {
 954             if ($opts->{stdout}) {
 955                 $task->{target} = '-';
 956             } else {
 957                 $task->{target} = $task->{tmptar} = "$opts->{dumpdir}/$basename$ext";
 958                 $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 959                 unlink $task->{tmptar};
 960             }
 961         }
 962
 963         $task->{vmtype} = $vmtype;
 964
 965         my $pid = $$;
 966         if ($opts->{tmpdir}) {
 967             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp${pid}_$vmid/";
 968         } elsif ($self->{opts}->{pbs}) {
 969             $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 970         } else {
 971             # dumpdir is posix? then use it as temporary dir
 972             my $info = get_mount_info($opts->{dumpdir});
 973             if ($vmtype eq 'qemu' ||
 974                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 975                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 976             } else {
 977                 $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 978                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 979                           "using $task->{tmpdir} for temporary files", $logfd);
 980             }
 981         }
 982
 983         rmtree $task->{tmpdir};
 984         mkdir $task->{tmpdir};
 985         -d $task->{tmpdir} ||
 986             die "unable to create temporary directory '$task->{tmpdir}'";
 987
 988         $logfd = IO::File->new (">$tmplog") ||
 989             die "unable to create log file '$tmplog'";
 990
 991         $task->{dumpdir} = $opts->{dumpdir};
 992         $task->{storeid} = $opts->{storage};
 993         $task->{scfg} = $opts->{scfg};
 994         $task->{tmplog} = $tmplog;
 995
 996         unlink $task->{logfile} if defined($task->{logfile});
 997
 998         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 999         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
1000
1001         $plugin->set_logfd ($logfd);
1002
1003         # test is VM is running
1004         my ($running, $status_text) = $plugin->vm_status ($vmid);
1005
1006         debugmsg ('info', "status = ${status_text}", $logfd);
1007
1008         # lock VM (prevent config changes)
1009         $plugin->lock_vm ($vmid);
1010
1011         $cleanup->{unlock} = 1;
1012
1013         # prepare
1014
1015         my $mode = $running ? $task->{mode} : 'stop';
1016
1017         if ($mode eq 'snapshot') {
1018             my %saved_task = %$task;
1019             eval { $plugin->prepare ($task, $vmid, $mode); };
1020             if (my $err = $@) {
1021                 die $err if $err !~ m/^mode failure/;
1022                 debugmsg ('info',  $err, $logfd);
1023                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
1024                 $mode = 'suspend'; # so prepare is called again below
1025                 %$task = %saved_task;
1026             }
1027         }
1028
1029         $cleanup->{prepared} = 1;
1030
1031         $task->{mode} = $mode;
1032
1033         debugmsg ('info', "backup mode: $mode", $logfd);
1034         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)  if $opts->{bwlimit};
1035         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
1036
1037         if ($mode eq 'stop') {
1038             $plugin->prepare ($task, $vmid, $mode);
1039
1040             $self->run_hook_script ('backup-start', $task, $logfd);
1041
1042             if ($running) {
1043                 debugmsg ('info', "stopping virtual guest", $logfd);
1044                 $task->{vmstoptime} = time();
1045                 $self->run_hook_script ('pre-stop', $task, $logfd);
1046                 $plugin->stop_vm ($task, $vmid);
1047                 $cleanup->{restart} = 1;
1048             }
1049
1050
1051         } elsif ($mode eq 'suspend') {
1052             $plugin->prepare ($task, $vmid, $mode);
1053
1054             $self->run_hook_script ('backup-start', $task, $logfd);
1055
1056             if ($vmtype eq 'lxc') {
1057                 # pre-suspend rsync
1058                 $plugin->copy_data_phase1($task, $vmid);
1059             }
1060
1061             debugmsg ('info', "suspending guest", $logfd);
1062             $task->{vmstoptime} = time ();
1063             $self->run_hook_script ('pre-stop', $task, $logfd);
1064             $plugin->suspend_vm ($task, $vmid);
1065             $cleanup->{resume} = 1;
1066
1067             if ($vmtype eq 'lxc') {
1068                 # post-suspend rsync
1069                 $plugin->copy_data_phase2($task, $vmid);
1070
1071                 debugmsg ('info', "resuming guest", $logfd);
1072                 $cleanup->{resume} = 0;
1073                 $self->run_hook_script('pre-restart', $task, $logfd);
1074                 $plugin->resume_vm($task, $vmid);
1075                 $self->run_hook_script('post-restart', $task, $logfd);
1076                 $log_vm_online_again->();
1077             }
1078
1079         } elsif ($mode eq 'snapshot') {
1080             $self->run_hook_script ('backup-start', $task, $logfd);
1081
1082             my $snapshot_count = $task->{snapshot_count} || 0;
1083
1084             $self->run_hook_script ('pre-stop', $task, $logfd);
1085
1086             if ($snapshot_count > 1) {
1087                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
1088                 $task->{vmstoptime} = time ();
1089                 $plugin->suspend_vm ($task, $vmid);
1090                 $cleanup->{resume} = 1;
1091             }
1092
1093             $plugin->snapshot ($task, $vmid);
1094
1095             $self->run_hook_script ('pre-restart', $task, $logfd);
1096
1097             if ($snapshot_count > 1) {
1098                 debugmsg ('info', "resume vm", $logfd);
1099                 $cleanup->{resume} = 0;
1100                 $plugin->resume_vm ($task, $vmid);
1101                 $log_vm_online_again->();
1102             }
1103
1104             $self->run_hook_script ('post-restart', $task, $logfd);
1105
1106         } else {
1107             die "internal error - unknown mode '$mode'\n";
1108         }
1109
1110         # assemble archive image
1111         $plugin->assemble ($task, $vmid);
1112
1113         # produce archive
1114
1115         if ($opts->{stdout}) {
1116             debugmsg ('info', "sending archive to stdout", $logfd);
1117             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
1118             $self->run_hook_script ('backup-end', $task, $logfd);
1119             return;
1120         }
1121
1122         my $archive_txt = $self->{opts}->{pbs} ? 'Proxmox Backup Server' : 'vzdump';
1123         debugmsg('info', "creating $archive_txt archive '$task->{target}'", $logfd);
1124         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
1125
1126         if ($self->{opts}->{pbs}) {
1127             # size is added to task struct in guest vzdump plugins
1128         } else {
1129             rename ($task->{tmptar}, $task->{target}) ||
1130                 die "unable to rename '$task->{tmptar}' to '$task->{target}'\n";
1131
1132             # determine size
1133             $task->{size} = (-s $task->{target}) || 0;
1134             my $cs = format_size ($task->{size});
1135             debugmsg ('info', "archive file size: $cs", $logfd);
1136         }
1137
1138         # Mark as protected before pruning.
1139         if (my $storeid = $opts->{storage}) {
1140             my $volname = $opts->{pbs} ? $task->{target} : basename($task->{target});
1141             my $volid = "${storeid}:backup/${volname}";
1142
1143             if ($opts->{'notes-template'} && $opts->{'notes-template'} ne '') {
1144                 debugmsg('info', "adding notes to backup", $logfd);
1145                 my $notes = eval { $generate_notes->($opts->{'notes-template'}, $task); };
1146                 if (my $err = $@) {
1147                     debugmsg('warn', "unable to add notes - $err", $logfd);
1148                 } else {
1149                     eval { PVE::Storage::update_volume_attribute($cfg, $volid, 'notes', $notes) };
1150                     debugmsg('warn', "unable to add notes - $@", $logfd) if $@;
1151                 }
1152             }
1153
1154             if ($opts->{protected}) {
1155                 debugmsg('info', "marking backup as protected", $logfd);
1156                 eval { PVE::Storage::update_volume_attribute($cfg, $volid, 'protected', 1) };
1157                 die "unable to set protected flag - $@\n" if $@;
1158             }
1159         }
1160
1161         if ($opts->{remove}) {
1162             my $keepstr = join(', ', map { "$_=$prune_options->{$_}" } sort keys %$prune_options);
1163             debugmsg ('info', "prune older backups with retention: $keepstr", $logfd);
1164             my $pruned = 0;
1165             if (!defined($opts->{storage})) {
1166                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
1167
1168                 PVE::Storage::prune_mark_backup_group($bklist, $prune_options);
1169
1170                 foreach my $prune_entry (@{$bklist}) {
1171                     next if $prune_entry->{mark} ne 'remove';
1172                     $pruned++;
1173                     my $archive_path = $prune_entry->{path};
1174                     debugmsg ('info', "delete old backup '$archive_path'", $logfd);
1175                     PVE::Storage::archive_remove($archive_path);
1176                 }
1177             } else {
1178                 my $pruned_list = PVE::Storage::prune_backups(
1179                     $cfg,
1180                     $opts->{storage},
1181                     $prune_options,
1182                     $vmid,
1183                     $vmtype,
1184                     0,
1185                     sub { debugmsg($_[0], $_[1], $logfd) },
1186                 );
1187                 $pruned = scalar(grep { $_->{mark} eq 'remove' } $pruned_list->@*);
1188             }
1189             my $log_pruned_extra = $pruned > 0 ? " not covered by keep-retention policy" : "";
1190             debugmsg ('info', "pruned $pruned backup(s)${log_pruned_extra}", $logfd);
1191         }
1192
1193         $self->run_hook_script ('backup-end', $task, $logfd);
1194     };
1195     my $err = $@;
1196
1197     if ($plugin) {
1198         # clean-up
1199
1200         if ($cleanup->{unlock}) {
1201             eval { $plugin->unlock_vm ($vmid); };
1202             warn $@ if $@;
1203         }
1204
1205         if ($cleanup->{prepared}) {
1206             # only call cleanup when necessary (when prepare was executed)
1207             eval { $plugin->cleanup ($task, $vmid) };
1208             warn $@ if $@;
1209         }
1210
1211         eval { $plugin->set_logfd (undef); };
1212         warn $@ if $@;
1213
1214         if ($cleanup->{resume} || $cleanup->{restart}) {
1215             eval {
1216                 $self->run_hook_script ('pre-restart', $task, $logfd);
1217                 if ($cleanup->{resume}) {
1218                     debugmsg ('info', "resume vm", $logfd);
1219                     $plugin->resume_vm ($task, $vmid);
1220                 } else {
1221                     my $running = $plugin->vm_status($vmid);
1222                     if (!$running) {
1223                         debugmsg ('info', "restarting vm", $logfd);
1224                         $plugin->start_vm ($task, $vmid);
1225                     }
1226                 }
1227                 $self->run_hook_script ('post-restart', $task, $logfd);
1228             };
1229             my $err = $@;
1230             if ($err) {
1231                 warn $err;
1232             } else {
1233                 $log_vm_online_again->();
1234             }
1235         }
1236     }
1237
1238     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
1239     warn $@ if $@;
1240
1241     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
1242     warn $@ if $@;
1243
1244     my $delay = $task->{backuptime} = time () - $vmstarttime;
1245
1246     if ($err) {
1247         $task->{state} = 'err';
1248         $task->{msg} = $err;
1249         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1250         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
1251
1252         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1253
1254     } else {
1255         $task->{state} = 'ok';
1256         my $tstr = format_time ($delay);
1257         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1258         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1259     }
1260
1261     close ($logfd) if $logfd;
1262
1263     if ($task->{tmplog}) {
1264         if ($self->{opts}->{pbs}) {
1265             if ($task->{state} eq 'ok') {
1266                 eval {
1267                     PVE::Storage::PBSPlugin::run_raw_client_cmd(
1268                         $opts->{scfg},
1269                         $opts->{storage},
1270                         'upload-log',
1271                         [ $pbs_snapshot_name, $task->{tmplog} ],
1272                         errmsg => "uploading backup task log failed",
1273                         outfunc => sub {},
1274                     );
1275                 };
1276                 debugmsg('warn', "$@") if $@; # $@ contains already error prefix
1277             }
1278         } elsif ($task->{logfile}) {
1279             system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1280         }
1281     }
1282
1283     eval { $self->run_hook_script ('log-end', $task); };
1284
1285     die $err if $err && $err =~ m/^interrupted by signal$/;
1286 }
1287
1288 sub exec_backup {
1289     my ($self, $rpcenv, $authuser) = @_;
1290
1291     my $opts = $self->{opts};
1292
1293     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1294
1295     if (scalar(@{$self->{skiplist}})) {
1296         my $skip_string = join(', ', sort { $a <=> $b } @{$self->{skiplist}});
1297         debugmsg ('info', "skip external VMs: $skip_string");
1298     }
1299
1300     my $tasklist = [];
1301     my $vzdump_plugins =  {};
1302     foreach my $plugin (@{$self->{plugins}}) {
1303         my $type = $plugin->type();
1304         next if exists $vzdump_plugins->{$type};
1305         $vzdump_plugins->{$type} = $plugin;
1306     }
1307
1308     my $vmlist = PVE::Cluster::get_vmlist();
1309     my $vmids = [ sort { $a <=> $b } @{$opts->{vmids}} ];
1310     foreach my $vmid (@{$vmids}) {
1311         my $plugin;
1312         if (defined($vmlist->{ids}->{$vmid})) {
1313             my $guest_type = $vmlist->{ids}->{$vmid}->{type};
1314             $plugin = $vzdump_plugins->{$guest_type};
1315             next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all});
1316         }
1317         push @$tasklist, {
1318             mode => $opts->{mode},
1319             plugin => $plugin,
1320             state => 'todo',
1321             vmid => $vmid,
1322         };
1323     }
1324
1325     # Use in-memory files for the outer hook logs to pass them to sendmail.
1326     my $job_start_log = '';
1327     my $job_end_log = '';
1328     open my $job_start_fd, '>', \$job_start_log;
1329     open my $job_end_fd, '>', \$job_end_log;
1330
1331     my $starttime = time();
1332     my $errcount = 0;
1333     eval {
1334
1335         $self->run_hook_script ('job-start', undef, $job_start_fd);
1336
1337         foreach my $task (@$tasklist) {
1338             $self->exec_backup_task ($task);
1339             $errcount += 1 if $task->{state} ne 'ok';
1340         }
1341
1342         $self->run_hook_script ('job-end', undef, $job_end_fd);
1343     };
1344     my $err = $@;
1345
1346     if ($err) {
1347         eval { $self->run_hook_script ('job-abort', undef, $job_end_fd); };
1348         $err .= $@ if $@;
1349         debugmsg ('err', "Backup job failed - $err", undef, 1);
1350     } else {
1351         if ($errcount) {
1352             debugmsg ('info', "Backup job finished with errors", undef, 1);
1353         } else {
1354             debugmsg ('info', "Backup job finished successfully", undef, 1);
1355         }
1356     }
1357
1358     close $job_start_fd;
1359     close $job_end_fd;
1360
1361     my $totaltime = time() - $starttime;
1362
1363     eval {
1364         # otherwise $self->send_notification() will interpret it as multiple problems
1365         my $chomped_err = $err;
1366         chomp($chomped_err) if $chomped_err;
1367
1368         $self->send_notification(
1369             $tasklist,
1370             $totaltime,
1371             $chomped_err,
1372             $self->{job_init_log} . $job_start_log,
1373             $job_end_log,
1374         );
1375     };
1376     debugmsg ('err', $@) if $@;
1377
1378     die $err if $err;
1379
1380     die "job errors\n" if $errcount;
1381
1382     unlink $pidfile;
1383 }
1384
1385
1386 sub option_exists {
1387     my $key = shift;
1388     return defined($confdesc->{$key});
1389 }
1390
1391 # NOTE it might make sense to merge this and verify_vzdump_parameters(), but one
1392 # needs to adapt command_line() in guest-common's PVE/VZDump/Common.pm and detect
1393 # a second parsing attempt, because verify_vzdump_parameters() is called twice
1394 # during the update_job API call.
1395 sub parse_mailto_exclude_path {
1396     my ($param) = @_;
1397
1398     # exclude-path list need to be 0 separated or be an array
1399     if (defined($param->{'exclude-path'})) {
1400         my $expaths;
1401         if (ref($param->{'exclude-path'}) eq 'ARRAY') {
1402             $expaths = $param->{'exclude-path'};
1403         } else {
1404             $expaths = [split(/\0/, $param->{'exclude-path'} || '')];
1405         }
1406         $param->{'exclude-path'} = $expaths;
1407     }
1408
1409     if (defined($param->{mailto})) {
1410         my @mailto = PVE::Tools::split_list(extract_param($param, 'mailto'));
1411         $param->{mailto} = [ @mailto ];
1412     }
1413
1414     return;
1415 }
1416
1417 sub verify_vzdump_parameters {
1418     my ($param, $check_missing) = @_;
1419
1420     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1421         if $param->{all} && $param->{vmid};
1422
1423     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1424         if $param->{exclude} && $param->{vmid};
1425
1426     raise_param_exc({ pool => "option conflicts with option 'vmid'"})
1427         if $param->{pool} && $param->{vmid};
1428
1429     raise_param_exc({ 'prune-backups' => "option conflicts with option 'maxfiles'"})
1430         if defined($param->{'prune-backups'}) && defined($param->{maxfiles});
1431
1432     $parse_prune_backups_maxfiles->($param, 'CLI parameters');
1433     parse_performance($param);
1434
1435     if (my $template = $param->{'notes-template'}) {
1436         eval { $verify_notes_template->($template); };
1437         raise_param_exc({'notes-template' => $@}) if $@;
1438     }
1439
1440     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
1441
1442     return if !$check_missing;
1443
1444     raise_param_exc({ vmid => "property is missing"})
1445         if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
1446
1447 }
1448
1449 sub stop_running_backups {
1450     my($self) = @_;
1451
1452     my $upid = PVE::Tools::file_read_firstline($pidfile);
1453     return if !$upid;
1454
1455     my $task = PVE::Tools::upid_decode($upid);
1456
1457     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1458         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1459         kill(15, $task->{pid});
1460         # wait max 15 seconds to shut down (else, do nothing for now)
1461         my $i;
1462         for ($i = 15; $i > 0; $i--) {
1463             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1464             sleep (1);
1465         }
1466         die "stopping backup process $task->{pid} failed\n" if $i == 0;
1467     }
1468 }
1469
1470 sub get_included_guests {
1471     my ($job) = @_;
1472
1473     my $vmids = [];
1474     my $vmids_per_node = {};
1475
1476     my $vmlist = PVE::Cluster::get_vmlist();
1477
1478     if ($job->{pool}) {
1479         $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool});
1480     } elsif ($job->{vmid}) {
1481         $vmids = [ split_list($job->{vmid}) ];
1482     } elsif ($job->{all}) {
1483         # all or exclude
1484         my $exclude = check_vmids(split_list($job->{exclude}));
1485         my $excludehash = { map { $_ => 1 } @$exclude };
1486
1487         for my $id (keys %{$vmlist->{ids}}) {
1488             next if $excludehash->{$id};
1489             push @$vmids, $id;
1490         }
1491     } else {
1492         return $vmids_per_node;
1493     }
1494     $vmids = check_vmids(@$vmids);
1495
1496     for my $vmid (@$vmids) {
1497         if (defined($vmlist->{ids}->{$vmid})) {
1498             my $node = $vmlist->{ids}->{$vmid}->{node};
1499             next if (defined $job->{node} && $job->{node} ne $node);
1500
1501             push @{$vmids_per_node->{$node}}, $vmid;
1502         } else {
1503             push @{$vmids_per_node->{''}}, $vmid;
1504         }
1505     }
1506
1507     return $vmids_per_node;
1508 }
1509
1510 1;