PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Clone;
   7 use Fcntl ':flock';
   8 use File::Basename;
   9 use File::Path;
  10 use IO::File;
  11 use IO::Select;
  12 use IPC::Open3;
  13 use POSIX qw(strftime);
  14 use Time::Local;
  15
  16 use PVE::Cluster qw(cfs_read_file);
  17 use PVE::DataCenterConfig;
  18 use PVE::Exception qw(raise_param_exc);
  19 use PVE::HA::Config;
  20 use PVE::HA::Env::PVE2;
  21 use PVE::JSONSchema qw(get_standard_option);
  22 use PVE::RPCEnvironment;
  23 use PVE::Storage;
  24 use PVE::VZDump::Common;
  25 use PVE::VZDump::Plugin;
  26 use PVE::Tools qw(extract_param split_list);
  27 use PVE::API2Tools;
  28
  29 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  30
  31 my $lockfile = '/var/run/vzdump.lock';
  32 my $pidfile = '/var/run/vzdump.pid';
  33 my $logdir = '/var/log/vzdump';
  34
  35 my @plugins = qw();
  36
  37 my $confdesc = PVE::VZDump::Common::get_confdesc();
  38
  39 my $confdesc_for_defaults = Clone::clone($confdesc);
  40 delete $confdesc_for_defaults->{$_}->{requires} for qw(notes-template protected);
  41
  42 # Load available plugins
  43 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
  44 foreach my $plug (@pve_vzdump_classes) {
  45     my $filename = "/usr/share/perl5/$plug.pm";
  46     $filename =~ s!::!/!g;
  47     if (-f $filename) {
  48         eval { require $filename; };
  49         if (!$@) {
  50             $plug->import ();
  51             push @plugins, $plug;
  52         } else {
  53             die $@;
  54         }
  55     }
  56 }
  57
  58 # helper functions
  59
  60 sub debugmsg {
  61     my ($mtype, $msg, $logfd, $syslog) = @_;
  62
  63     PVE::VZDump::Plugin::debugmsg(@_);
  64 }
  65
  66 sub run_command {
  67     my ($logfd, $cmdstr, %param) = @_;
  68
  69     my $logfunc = sub {
  70         my $line = shift;
  71         debugmsg ('info', $line, $logfd);
  72     };
  73
  74     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
  75 }
  76
  77 my $verify_notes_template = sub {
  78     my ($template) = @_;
  79
  80     die "contains a line feed\n" if $template =~ /\n/;
  81
  82     my @problematic = ();
  83     while ($template =~ /\\(.)/g) {
  84         my $char = $1;
  85         push @problematic, "escape sequence '\\$char' at char " . (pos($template) - 2)
  86             if $char !~ /^[n\\]$/;
  87     }
  88
  89     while ($template =~ /\{\{([^\s{}]+)\}\}/g) {
  90         my $var = $1;
  91         push @problematic, "variable '$var' at char " . (pos($template) - length($var))
  92             if $var !~ /^(cluster|guestname|node|vmid)$/;
  93     }
  94
  95     die "found unknown: " . join(', ', @problematic) . "\n" if scalar(@problematic);
  96 };
  97
  98 my $generate_notes = sub {
  99     my ($notes_template, $task) = @_;
 100
 101     $verify_notes_template->($notes_template);
 102
 103     my $info = {
 104         cluster => PVE::Cluster::get_clinfo()->{cluster}->{name} // 'standalone node',
 105         guestname => $task->{hostname} // "VM $task->{vmid}", # is always set for CTs
 106         node => PVE::INotify::nodename(),
 107         vmid => $task->{vmid},
 108     };
 109
 110     my $unescape = sub {
 111         my ($char) = @_;
 112         return '\\' if $char eq '\\';
 113         return "\n" if $char eq 'n';
 114         die "unexpected escape character '$char'\n";
 115     };
 116
 117     $notes_template =~ s/\\(.)/$unescape->($1)/eg;
 118
 119     my $vars = join('|', keys $info->%*);
 120     $notes_template =~ s/\{\{($vars)\}\}/$info->{$1}/g;
 121
 122     return $notes_template;
 123 };
 124
 125 my sub parse_performance {
 126     my ($param) = @_;
 127
 128     if (defined(my $perf = $param->{performance})) {
 129         return if ref($perf) eq 'HASH'; # already parsed
 130         $param->{performance} = PVE::JSONSchema::parse_property_string('backup-performance', $perf);
 131     }
 132 }
 133
 134 my $parse_prune_backups_maxfiles = sub {
 135     my ($param, $kind) = @_;
 136
 137     my $maxfiles = delete $param->{maxfiles};
 138     my $prune_backups = $param->{'prune-backups'};
 139
 140     debugmsg('warn', "both 'maxfiles' and 'prune-backups' defined as ${kind} - ignoring 'maxfiles'")
 141         if defined($maxfiles) && defined($prune_backups);
 142
 143     if (defined($prune_backups)) {
 144         return if ref($prune_backups) eq 'HASH'; # already parsed
 145         $param->{'prune-backups'} = PVE::JSONSchema::parse_property_string(
 146             'prune-backups',
 147             $prune_backups
 148         );
 149     } elsif (defined($maxfiles)) {
 150         if ($maxfiles) {
 151             $param->{'prune-backups'} = { 'keep-last' => $maxfiles };
 152         } else {
 153             $param->{'prune-backups'} = { 'keep-all' => 1 };
 154         }
 155     }
 156 };
 157
 158 sub storage_info {
 159     my $storage = shift;
 160
 161     my $cfg = PVE::Storage::config();
 162     my $scfg = PVE::Storage::storage_config($cfg, $storage);
 163     my $type = $scfg->{type};
 164
 165     die "can't use storage '$storage' for backups - wrong content type\n"
 166         if (!$scfg->{content}->{backup});
 167
 168     my $info = {
 169         scfg => $scfg,
 170     };
 171
 172     $info->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'})
 173         if defined($scfg->{'prune-backups'});
 174
 175     if ($type eq 'pbs') {
 176         $info->{pbs} = 1;
 177     } else {
 178         $info->{dumpdir} = PVE::Storage::get_backup_dir($cfg, $storage);
 179     }
 180
 181     return $info;
 182 }
 183
 184 sub format_size {
 185     my $size = shift;
 186
 187     my $kb = $size / 1024;
 188
 189     if ($kb < 1024) {
 190         return int ($kb) . "KB";
 191     }
 192
 193     my $mb = $size / (1024*1024);
 194     if ($mb < 1024) {
 195         return int ($mb) . "MB";
 196     }
 197     my $gb = $mb / 1024;
 198     if ($gb < 1024) {
 199         return sprintf ("%.2fGB", $gb);
 200     }
 201     my $tb = $gb / 1024;
 202     return sprintf ("%.2fTB", $tb);
 203 }
 204
 205 sub format_time {
 206     my $seconds = shift;
 207
 208     my $hours = int ($seconds/3600);
 209     $seconds = $seconds - $hours*3600;
 210     my $min = int ($seconds/60);
 211     $seconds = $seconds - $min*60;
 212
 213     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 214 }
 215
 216 sub encode8bit {
 217     my ($str) = @_;
 218
 219     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 220
 221     return $str;
 222 }
 223
 224 sub escape_html {
 225     my ($str) = @_;
 226
 227     $str =~ s/&/&amp;/g;
 228     $str =~ s/</&lt;/g;
 229     $str =~ s/>/&gt;/g;
 230
 231     return $str;
 232 }
 233
 234 sub check_bin {
 235     my ($bin)  = @_;
 236
 237     foreach my $p (split (/:/, $ENV{PATH})) {
 238         my $fn = "$p/$bin";
 239         if (-x $fn) {
 240             return $fn;
 241         }
 242     }
 243
 244     die "unable to find command '$bin'\n";
 245 }
 246
 247 sub check_vmids {
 248     my (@vmids) = @_;
 249
 250     my $res = [];
 251     for my $vmid (sort {$a <=> $b} @vmids) {
 252         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 253         $vmid = int ($vmid); # remove leading zeros
 254         next if !$vmid;
 255         push @$res, $vmid;
 256     }
 257
 258     return $res;
 259 }
 260
 261
 262 sub read_vzdump_defaults {
 263
 264     my $fn = "/etc/vzdump.conf";
 265
 266     my $defaults = {
 267         map {
 268             my $default = $confdesc->{$_}->{default};
 269              defined($default) ? ($_ => $default) : ()
 270         } keys %$confdesc_for_defaults
 271     };
 272     $parse_prune_backups_maxfiles->($defaults, "defaults in VZDump schema");
 273     parse_performance($defaults);
 274
 275     my $raw;
 276     eval { $raw = PVE::Tools::file_get_contents($fn); };
 277     return $defaults if $@;
 278
 279     my $conf_schema = { type => 'object', properties => $confdesc_for_defaults };
 280     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 281     if (my $excludes = $res->{'exclude-path'}) {
 282         if (ref($excludes) eq 'ARRAY') {
 283             my $list = [];
 284             for my $path ($excludes->@*) {
 285                 # We still use `split_args` here to be compatible with old configs where one line
 286                 # still has multiple space separated entries.
 287                 push $list->@*, PVE::Tools::split_args($path)->@*;
 288             }
 289             $res->{'exclude-path'} = $list;
 290         } else {
 291             $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 292         }
 293     }
 294     if (defined($res->{mailto})) {
 295         my @mailto = split_list($res->{mailto});
 296         $res->{mailto} = [ @mailto ];
 297     }
 298     $parse_prune_backups_maxfiles->($res, "options in '$fn'");
 299     parse_performance($res);
 300
 301     foreach my $key (keys %$defaults) {
 302         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 303     }
 304
 305     if (defined($res->{storage}) && defined($res->{dumpdir})) {
 306         debugmsg('warn', "both 'storage' and 'dumpdir' defined in '$fn' - ignoring 'dumpdir'");
 307         delete $res->{dumpdir};
 308     }
 309
 310     return $res;
 311 }
 312
 313 use constant MAX_MAIL_SIZE => 1024*1024;
 314 sub sendmail {
 315     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 316
 317     my $opts = $self->{opts};
 318
 319     my $mailto = $opts->{mailto};
 320
 321     return if !($mailto && scalar(@$mailto));
 322
 323     my $cmdline = $self->{cmdline};
 324
 325     my $ecount = 0;
 326     foreach my $task (@$tasklist) {
 327         $ecount++ if $task->{state} ne 'ok';
 328         chomp $task->{msg} if $task->{msg};
 329         $task->{backuptime} = 0 if !$task->{backuptime};
 330         $task->{size} = 0 if !$task->{size};
 331         $task->{target} = 'unknown' if !$task->{target};
 332         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 333
 334         if ($task->{state} eq 'todo') {
 335             $task->{msg} = 'aborted';
 336         }
 337     }
 338
 339     my $notify = $opts->{mailnotification} || 'always';
 340     return if (!$ecount && !$err && ($notify eq 'failure'));
 341
 342     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 343     if ($err) {
 344         if ($err =~ /\n/) {
 345             $stat .= ": multiple problems";
 346         } else {
 347             $stat .= ": $err";
 348             $err = undef;
 349         }
 350     }
 351
 352     my $hostname = `hostname -f` || PVE::INotify::nodename();
 353     chomp $hostname;
 354
 355     # text part
 356     my $text = $err ? "$err\n\n" : '';
 357     my $namelength = 20;
 358     $text .= sprintf (
 359         "%-10s %-${namelength}s %-6s %10s %10s  %s\n",
 360         qw(VMID NAME STATUS TIME SIZE FILENAME)
 361     );
 362     foreach my $task (@$tasklist) {
 363         my $name = substr($task->{hostname}, 0, $namelength);
 364         my $successful = $task->{state} eq 'ok';
 365         my $size = $successful ? format_size ($task->{size}) : 0;
 366         my $filename = $successful ? $task->{target} : '-';
 367         my $size_fmt = $successful ? "%10s": "%8.2fMB";
 368         $text .= sprintf(
 369             "%-10s %-${namelength}s %-6s %10s $size_fmt  %s\n",
 370             $task->{vmid},
 371             $name,
 372             $task->{state},
 373             format_time($task->{backuptime}),
 374             $size,
 375             $filename,
 376         );
 377     }
 378
 379     my $text_log_part;
 380     $text_log_part .= "\nDetailed backup logs:\n\n";
 381     $text_log_part .= "$cmdline\n\n";
 382
 383     $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
 384     foreach my $task (@$tasklist) {
 385         my $vmid = $task->{vmid};
 386         my $log = $task->{tmplog};
 387         if (!$log) {
 388             $text_log_part .= "$vmid: no log available\n\n";
 389             next;
 390         }
 391         if (open (my $TMP, '<', "$log")) {
 392             while (my $line = <$TMP>) {
 393                 next if $line =~ /^status: \d+/; # not useful in mails
 394                 $text_log_part .= encode8bit ("$vmid: $line");
 395             }
 396             close ($TMP);
 397         } else {
 398             $text_log_part .= "$vmid: Could not open log file\n\n";
 399         }
 400         $text_log_part .= "\n";
 401     }
 402     $text_log_part .= $detail_post if defined($detail_post);
 403
 404     # html part
 405     my $html = "<html><body>\n";
 406     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 407     $html .= "<table border=1 cellpadding=3>\n";
 408     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 409
 410     my $ssize = 0;
 411     foreach my $task (@$tasklist) {
 412         my $vmid = $task->{vmid};
 413         my $name = $task->{hostname};
 414
 415         if  ($task->{state} eq 'ok') {
 416             $ssize += $task->{size};
 417
 418             $html .= sprintf (
 419                 "<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 420                 $vmid,
 421                 $name,
 422                 format_time($task->{backuptime}),
 423                 format_size ($task->{size}),
 424                 escape_html ($task->{target}),
 425             );
 426         } else {
 427             $html .= sprintf (
 428                 "<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 429                 $vmid,
 430                 $name,
 431                 format_time($task->{backuptime}),
 432                 escape_html ($task->{msg}),
 433             );
 434         }
 435     }
 436
 437     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 438  format_time ($totaltime), format_size ($ssize));
 439
 440     $html .= "\n</table><br><br>\n";
 441     my $html_log_part;
 442     $html_log_part .= "Detailed backup logs:<br /><br />\n";
 443     $html_log_part .= "<pre>\n";
 444     $html_log_part .= escape_html($cmdline) . "\n\n";
 445
 446     $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 447     foreach my $task (@$tasklist) {
 448         my $vmid = $task->{vmid};
 449         my $log = $task->{tmplog};
 450         if (!$log) {
 451             $html_log_part .= "$vmid: no log available\n\n";
 452             next;
 453         }
 454         if (open (my $TMP, '<', "$log")) {
 455             while (my $line = <$TMP>) {
 456                 next if $line =~ /^status: \d+/; # not useful in mails
 457                 if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 458                     $html_log_part .= encode8bit ("$vmid: <font color=red>".
 459                         escape_html ($line) . "</font>");
 460                 } else {
 461                     $html_log_part .= encode8bit ("$vmid: " . escape_html ($line));
 462                 }
 463             }
 464             close ($TMP);
 465         } else {
 466             $html_log_part .= "$vmid: Could not open log file\n\n";
 467         }
 468         $html_log_part .= "\n";
 469     }
 470     $html_log_part .= escape_html($detail_post) if defined($detail_post);
 471     $html_log_part .= "</pre>";
 472     my $html_end = "\n</body></html>\n";
 473     # end html part
 474
 475     if (length($text) + length($text_log_part) +
 476         length($html) + length($html_log_part) +
 477         length($html_end) < MAX_MAIL_SIZE)
 478     {
 479         $html .= $html_log_part;
 480         $html .= $html_end;
 481         $text .= $text_log_part;
 482     } else {
 483         my $msg = "Log output was too long to be sent by mail. ".
 484             "See Task History for details!\n";
 485         $text .= $msg;
 486         $html .= "<p>$msg</p>";
 487         $html .= $html_end;
 488     }
 489
 490     my $subject = "vzdump backup status ($hostname) : $stat";
 491
 492     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 493     my $mailfrom = $dcconf->{email_from} || "root";
 494
 495     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 496 };
 497
 498 sub new {
 499     my ($class, $cmdline, $opts, $skiplist) = @_;
 500
 501     mkpath $logdir;
 502
 503     check_bin ('cp');
 504     check_bin ('df');
 505     check_bin ('sendmail');
 506     check_bin ('rsync');
 507     check_bin ('tar');
 508     check_bin ('mount');
 509     check_bin ('umount');
 510     check_bin ('cstream');
 511     check_bin ('ionice');
 512
 513     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 514         check_bin ('lvcreate');
 515         check_bin ('lvs');
 516         check_bin ('lvremove');
 517     }
 518
 519     my $defaults = read_vzdump_defaults();
 520
 521     foreach my $k (keys %$defaults) {
 522         next if $k eq 'exclude-path' || $k eq 'prune-backups'; # dealt with separately
 523         if ($k eq 'dumpdir' || $k eq 'storage') {
 524             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 525                 !defined ($opts->{storage});
 526         } else {
 527             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 528         }
 529     }
 530
 531     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 532     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 533
 534     $skiplist = [] if !$skiplist;
 535     my $self = bless {
 536         cmdline => $cmdline,
 537         opts => $opts,
 538         skiplist => $skiplist,
 539     }, $class;
 540
 541     my $findexcl = $self->{findexcl} = [];
 542     if ($defaults->{'exclude-path'}) {
 543         push @$findexcl, @{$defaults->{'exclude-path'}};
 544     }
 545
 546     if ($opts->{'exclude-path'}) {
 547         push @$findexcl, @{$opts->{'exclude-path'}};
 548     }
 549
 550     if ($opts->{stdexcludes}) {
 551         push @$findexcl,
 552             '/tmp/?*',
 553             '/var/tmp/?*',
 554             '/var/run/?*.pid',
 555             ;
 556     }
 557
 558     foreach my $p (@plugins) {
 559         my $pd = $p->new($self);
 560
 561         push @{$self->{plugins}}, $pd;
 562     }
 563
 564     if (defined($opts->{storage}) && $opts->{stdout}) {
 565         die "cannot use options 'storage' and 'stdout' at the same time\n";
 566     } elsif (defined($opts->{storage}) && defined($opts->{dumpdir})) {
 567         die "cannot use options 'storage' and 'dumpdir' at the same time\n";
 568     }
 569
 570     if (!$opts->{dumpdir} && !$opts->{storage}) {
 571         $opts->{storage} = 'local';
 572     }
 573
 574     # Enforced by the API too, but these options might come in via defaults. Drop them if necessary.
 575     if (!$opts->{storage}) {
 576         delete $opts->{$_} for qw(notes-template protected);
 577     }
 578
 579     my $errors = '';
 580     my $add_error = sub {
 581         my ($error) = @_;
 582         $errors .= "\n" if $errors;
 583         chomp($error);
 584         $errors .= $error;
 585     };
 586
 587     eval {
 588         $self->{job_init_log} = '';
 589         open my $job_init_fd, '>', \$self->{job_init_log};
 590         $self->run_hook_script('job-init', undef, $job_init_fd);
 591         close $job_init_fd;
 592
 593         PVE::Cluster::cfs_update(); # Pick up possible changes made by the hook script.
 594     };
 595     $add_error->($@) if $@;
 596
 597     if ($opts->{storage}) {
 598         my $storage_cfg = PVE::Storage::config();
 599         eval { PVE::Storage::activate_storage($storage_cfg, $opts->{storage}) };
 600         $add_error->("could not activate storage '$opts->{storage}': $@") if $@;
 601
 602         my $info = eval { storage_info ($opts->{storage}) };
 603         if (my $err = $@) {
 604             $add_error->("could not get storage information for '$opts->{storage}': $err");
 605         } else {
 606             $opts->{dumpdir} = $info->{dumpdir};
 607             $opts->{scfg} = $info->{scfg};
 608             $opts->{pbs} = $info->{pbs};
 609             $opts->{'prune-backups'} //= $info->{'prune-backups'};
 610         }
 611     } elsif ($opts->{dumpdir}) {
 612         $add_error->("dumpdir '$opts->{dumpdir}' does not exist")
 613             if ! -d $opts->{dumpdir};
 614     } else {
 615         die "internal error";
 616     }
 617
 618     $opts->{'prune-backups'} //= $defaults->{'prune-backups'};
 619
 620     # avoid triggering any remove code path if keep-all is set
 621     $opts->{remove} = 0 if $opts->{'prune-backups'}->{'keep-all'};
 622
 623     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 624         $add_error->("tmpdir '$opts->{tmpdir}' does not exist");
 625     }
 626
 627     if ($errors) {
 628         eval { $self->sendmail([], 0, $errors); };
 629         debugmsg ('err', $@) if $@;
 630         die "$errors\n";
 631     }
 632
 633     return $self;
 634 }
 635
 636 sub get_mount_info {
 637     my ($dir) = @_;
 638
 639     # Note: df 'available' can be negative, and percentage set to '-'
 640
 641     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 642
 643     my $res;
 644
 645     my $parser = sub {
 646         my $line = shift;
 647         if (my ($fsid, $fstype, undef, $mp) = $line =~
 648             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 649             $res = {
 650                 device => $fsid,
 651                 fstype => $fstype,
 652                 mountpoint => $mp,
 653             };
 654         }
 655     };
 656
 657     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 658     warn $@ if $@;
 659
 660     return $res;
 661 }
 662
 663 sub getlock {
 664     my ($self, $upid) = @_;
 665
 666     my $fh;
 667
 668     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 669
 670     die "missing UPID" if !$upid; # should not happen
 671
 672     my $SERVER_FLCK;
 673     if (!open ($SERVER_FLCK, '>>', "$lockfile")) {
 674         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 675         die "can't open lock on file '$lockfile' - $!";
 676     }
 677
 678     if (!flock ($SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 679         if (!$maxwait) {
 680             debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
 681             die "can't acquire lock '$lockfile' (wait = 0)";
 682         }
 683
 684         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 685         eval {
 686             alarm ($maxwait * 60);
 687
 688             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 689
 690             if (!flock ($SERVER_FLCK, LOCK_EX)) {
 691                 my $err = $!;
 692                 close ($SERVER_FLCK);
 693                 alarm (0);
 694                 die "$err\n";
 695             }
 696             alarm (0);
 697         };
 698         alarm (0);
 699
 700         my $err = $@;
 701
 702         if ($err) {
 703             debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
 704             die "can't acquire lock '$lockfile' - $err";
 705         }
 706
 707         debugmsg('info', "got global lock", undef, 1);
 708     }
 709
 710     PVE::Tools::file_set_contents($pidfile, $upid);
 711
 712     return $SERVER_FLCK;
 713 }
 714
 715 sub run_hook_script {
 716     my ($self, $phase, $task, $logfd) = @_;
 717
 718     my $opts = $self->{opts};
 719
 720     my $script = $opts->{script};
 721     return if !$script;
 722
 723     die "Error: The hook script '$script' does not exist.\n" if ! -f $script;
 724     die "Error: The hook script '$script' is not executable.\n" if ! -x $script;
 725
 726     my $cmd = [$script, $phase];
 727
 728     if ($task) {
 729         push @$cmd, $task->{mode};
 730         push @$cmd, $task->{vmid};
 731     }
 732
 733     local %ENV;
 734     # set immutable opts directly (so they are available in all phases)
 735     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 736     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 737
 738     foreach my $ek (qw(vmtype hostname target logfile)) {
 739         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 740     }
 741
 742     run_command ($logfd, $cmd);
 743 }
 744
 745 sub compressor_info {
 746     my ($opts) = @_;
 747     my $opt_compress = $opts->{compress};
 748
 749     if (!$opt_compress || $opt_compress eq '0') {
 750         return undef;
 751     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 752         return ('lzop', 'lzo');
 753     } elsif ($opt_compress eq 'gzip') {
 754         if ($opts->{pigz} > 0) {
 755             my $pigz_threads = $opts->{pigz};
 756             if ($pigz_threads == 1) {
 757                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 758                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 759             }
 760             return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
 761         } else {
 762             return ('gzip --rsyncable', 'gz');
 763         }
 764     } elsif ($opt_compress eq 'zstd') {
 765         my $zstd_threads = $opts->{zstd} // 1;
 766         if ($zstd_threads == 0) {
 767             my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 768             $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
 769         }
 770         return ("zstd --threads=${zstd_threads}", 'zst');
 771     } else {
 772         die "internal error - unknown compression option '$opt_compress'";
 773     }
 774 }
 775
 776 sub get_backup_file_list {
 777     my ($dir, $bkname) = @_;
 778
 779     my $bklist = [];
 780     foreach my $fn (<$dir/${bkname}-*>) {
 781         my $archive_info = eval { PVE::Storage::archive_info($fn) } // {};
 782         if ($archive_info->{is_std_name}) {
 783             my $path = "$dir/$archive_info->{filename}";
 784             my $backup = {
 785                 'path' => $path,
 786                 'ctime' => $archive_info->{ctime},
 787             };
 788             $backup->{mark} = "protected"
 789                 if -e PVE::Storage::protection_file_path($path);
 790             push @{$bklist}, $backup;
 791         }
 792     }
 793
 794     return $bklist;
 795 }
 796
 797 sub exec_backup_task {
 798     my ($self, $task) = @_;
 799
 800     my $opts = $self->{opts};
 801
 802     my $cfg = PVE::Storage::config();
 803     my $vmid = $task->{vmid};
 804     my $plugin = $task->{plugin};
 805
 806     $task->{backup_time} = time();
 807
 808     my $pbs_group_name;
 809     my $pbs_snapshot_name;
 810
 811     my $vmstarttime = time ();
 812
 813     my $logfd;
 814
 815     my $cleanup = {};
 816
 817     my $log_vm_online_again = sub {
 818         return if !defined($task->{vmstoptime});
 819         $task->{vmconttime} //= time();
 820         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 821         $delay = '<1' if $delay < 1;
 822         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 823     };
 824
 825     eval {
 826         die "unable to find VM '$vmid'\n" if !$plugin;
 827
 828         my $vmtype = $plugin->type();
 829
 830         if ($self->{opts}->{pbs}) {
 831             if ($vmtype eq 'lxc') {
 832                 $pbs_group_name = "ct/$vmid";
 833             } elsif  ($vmtype eq 'qemu') {
 834                 $pbs_group_name = "vm/$vmid";
 835             } else {
 836                 die "pbs backup not implemented for plugin type '$vmtype'\n";
 837             }
 838             my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
 839             $pbs_snapshot_name = "$pbs_group_name/$btime";
 840         }
 841
 842         # for now we deny backups of a running ha managed service in *stop* mode
 843         # as it interferes with the HA stack (started services should not stop).
 844         if ($opts->{mode} eq 'stop' &&
 845             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 846         {
 847             die "Cannot execute a backup with stop mode on a HA managed and".
 848                 " enabled Service. Use snapshot mode or disable the Service.\n";
 849         }
 850
 851         my $tmplog = "$logdir/$vmtype-$vmid.log";
 852
 853         my $bkname = "vzdump-$vmtype-$vmid";
 854         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
 855
 856         my $prune_options = $opts->{'prune-backups'};
 857
 858         my $backup_limit = 0;
 859         if (!$prune_options->{'keep-all'}) {
 860             foreach my $keep (values %{$prune_options}) {
 861                 $backup_limit += $keep;
 862             }
 863         }
 864
 865         if (($backup_limit && !$opts->{remove}) || $opts->{protected}) {
 866             my $count;
 867             my $protected_count;
 868             if (my $storeid = $opts->{storage}) {
 869                 my @backups = grep {
 870                     !$_->{subtype} || $_->{subtype} eq $vmtype
 871                 } PVE::Storage::volume_list($cfg, $storeid, $vmid, 'backup')->@*;
 872
 873                 $count = grep { !$_->{protected} } @backups;
 874                 $protected_count = scalar(@backups) - $count;
 875             } else {
 876                 $count = grep { !$_->{mark} || $_->{mark} ne "protected" } get_backup_file_list($opts->{dumpdir}, $bkname)->@*;
 877             }
 878
 879             if ($opts->{protected}) {
 880                 my $max_protected = PVE::Storage::get_max_protected_backups(
 881                     $opts->{scfg},
 882                     $opts->{storage},
 883                 );
 884                 if ($max_protected > -1 && $protected_count >= $max_protected) {
 885                     die "The number of protected backups per guest is limited to $max_protected ".
 886                         "on storage '$opts->{storage}'\n";
 887                 }
 888             } elsif ($count >= $backup_limit) {
 889                 die "There is a max backup limit of $backup_limit enforced by the target storage ".
 890                     "or the vzdump parameters. Either increase the limit or delete old backups.\n";
 891             }
 892         }
 893
 894         if (!$self->{opts}->{pbs}) {
 895             $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 896         }
 897
 898         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 899         my ($comp, $comp_ext) = compressor_info($opts);
 900         if ($comp && $comp_ext) {
 901             $ext .= ".${comp_ext}";
 902         }
 903
 904         if ($self->{opts}->{pbs}) {
 905             die "unable to pipe backup to stdout\n" if $opts->{stdout};
 906             $task->{target} = $pbs_snapshot_name;
 907         } else {
 908             if ($opts->{stdout}) {
 909                 $task->{target} = '-';
 910             } else {
 911                 $task->{target} = $task->{tmptar} = "$opts->{dumpdir}/$basename$ext";
 912                 $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 913                 unlink $task->{tmptar};
 914             }
 915         }
 916
 917         $task->{vmtype} = $vmtype;
 918
 919         my $pid = $$;
 920         if ($opts->{tmpdir}) {
 921             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp${pid}_$vmid/";
 922         } elsif ($self->{opts}->{pbs}) {
 923             $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 924         } else {
 925             # dumpdir is posix? then use it as temporary dir
 926             my $info = get_mount_info($opts->{dumpdir});
 927             if ($vmtype eq 'qemu' ||
 928                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 929                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 930             } else {
 931                 $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 932                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 933                           "using $task->{tmpdir} for temporary files", $logfd);
 934             }
 935         }
 936
 937         rmtree $task->{tmpdir};
 938         mkdir $task->{tmpdir};
 939         -d $task->{tmpdir} ||
 940             die "unable to create temporary directory '$task->{tmpdir}'";
 941
 942         $logfd = IO::File->new (">$tmplog") ||
 943             die "unable to create log file '$tmplog'";
 944
 945         $task->{dumpdir} = $opts->{dumpdir};
 946         $task->{storeid} = $opts->{storage};
 947         $task->{scfg} = $opts->{scfg};
 948         $task->{tmplog} = $tmplog;
 949
 950         unlink $task->{logfile} if defined($task->{logfile});
 951
 952         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 953         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
 954
 955         $plugin->set_logfd ($logfd);
 956
 957         # test is VM is running
 958         my ($running, $status_text) = $plugin->vm_status ($vmid);
 959
 960         debugmsg ('info', "status = ${status_text}", $logfd);
 961
 962         # lock VM (prevent config changes)
 963         $plugin->lock_vm ($vmid);
 964
 965         $cleanup->{unlock} = 1;
 966
 967         # prepare
 968
 969         my $mode = $running ? $task->{mode} : 'stop';
 970
 971         if ($mode eq 'snapshot') {
 972             my %saved_task = %$task;
 973             eval { $plugin->prepare ($task, $vmid, $mode); };
 974             if (my $err = $@) {
 975                 die $err if $err !~ m/^mode failure/;
 976                 debugmsg ('info',  $err, $logfd);
 977                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 978                 $mode = 'suspend'; # so prepare is called again below
 979                 %$task = %saved_task;
 980             }
 981         }
 982
 983         $cleanup->{prepared} = 1;
 984
 985         $task->{mode} = $mode;
 986
 987         debugmsg ('info', "backup mode: $mode", $logfd);
 988         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)  if $opts->{bwlimit};
 989         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 990
 991         if ($mode eq 'stop') {
 992             $plugin->prepare ($task, $vmid, $mode);
 993
 994             $self->run_hook_script ('backup-start', $task, $logfd);
 995
 996             if ($running) {
 997                 debugmsg ('info', "stopping virtual guest", $logfd);
 998                 $task->{vmstoptime} = time();
 999                 $self->run_hook_script ('pre-stop', $task, $logfd);
1000                 $plugin->stop_vm ($task, $vmid);
1001                 $cleanup->{restart} = 1;
1002             }
1003
1004
1005         } elsif ($mode eq 'suspend') {
1006             $plugin->prepare ($task, $vmid, $mode);
1007
1008             $self->run_hook_script ('backup-start', $task, $logfd);
1009
1010             if ($vmtype eq 'lxc') {
1011                 # pre-suspend rsync
1012                 $plugin->copy_data_phase1($task, $vmid);
1013             }
1014
1015             debugmsg ('info', "suspending guest", $logfd);
1016             $task->{vmstoptime} = time ();
1017             $self->run_hook_script ('pre-stop', $task, $logfd);
1018             $plugin->suspend_vm ($task, $vmid);
1019             $cleanup->{resume} = 1;
1020
1021             if ($vmtype eq 'lxc') {
1022                 # post-suspend rsync
1023                 $plugin->copy_data_phase2($task, $vmid);
1024
1025                 debugmsg ('info', "resuming guest", $logfd);
1026                 $cleanup->{resume} = 0;
1027                 $self->run_hook_script('pre-restart', $task, $logfd);
1028                 $plugin->resume_vm($task, $vmid);
1029                 $self->run_hook_script('post-restart', $task, $logfd);
1030                 $log_vm_online_again->();
1031             }
1032
1033         } elsif ($mode eq 'snapshot') {
1034             $self->run_hook_script ('backup-start', $task, $logfd);
1035
1036             my $snapshot_count = $task->{snapshot_count} || 0;
1037
1038             $self->run_hook_script ('pre-stop', $task, $logfd);
1039
1040             if ($snapshot_count > 1) {
1041                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
1042                 $task->{vmstoptime} = time ();
1043                 $plugin->suspend_vm ($task, $vmid);
1044                 $cleanup->{resume} = 1;
1045             }
1046
1047             $plugin->snapshot ($task, $vmid);
1048
1049             $self->run_hook_script ('pre-restart', $task, $logfd);
1050
1051             if ($snapshot_count > 1) {
1052                 debugmsg ('info', "resume vm", $logfd);
1053                 $cleanup->{resume} = 0;
1054                 $plugin->resume_vm ($task, $vmid);
1055                 $log_vm_online_again->();
1056             }
1057
1058             $self->run_hook_script ('post-restart', $task, $logfd);
1059
1060         } else {
1061             die "internal error - unknown mode '$mode'\n";
1062         }
1063
1064         # assemble archive image
1065         $plugin->assemble ($task, $vmid);
1066
1067         # produce archive
1068
1069         if ($opts->{stdout}) {
1070             debugmsg ('info', "sending archive to stdout", $logfd);
1071             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
1072             $self->run_hook_script ('backup-end', $task, $logfd);
1073             return;
1074         }
1075
1076         my $archive_txt = $self->{opts}->{pbs} ? 'Proxmox Backup Server' : 'vzdump';
1077         debugmsg('info', "creating $archive_txt archive '$task->{target}'", $logfd);
1078         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
1079
1080         if ($self->{opts}->{pbs}) {
1081             # size is added to task struct in guest vzdump plugins
1082         } else {
1083             rename ($task->{tmptar}, $task->{target}) ||
1084                 die "unable to rename '$task->{tmptar}' to '$task->{target}'\n";
1085
1086             # determine size
1087             $task->{size} = (-s $task->{target}) || 0;
1088             my $cs = format_size ($task->{size});
1089             debugmsg ('info', "archive file size: $cs", $logfd);
1090         }
1091
1092         # Mark as protected before pruning.
1093         if (my $storeid = $opts->{storage}) {
1094             my $volname = $opts->{pbs} ? $task->{target} : basename($task->{target});
1095             my $volid = "${storeid}:backup/${volname}";
1096
1097             if ($opts->{'notes-template'} && $opts->{'notes-template'} ne '') {
1098                 debugmsg('info', "adding notes to backup", $logfd);
1099                 my $notes = eval { $generate_notes->($opts->{'notes-template'}, $task); };
1100                 if (my $err = $@) {
1101                     debugmsg('warn', "unable to add notes - $err", $logfd);
1102                 } else {
1103                     eval { PVE::Storage::update_volume_attribute($cfg, $volid, 'notes', $notes) };
1104                     debugmsg('warn', "unable to add notes - $@", $logfd) if $@;
1105                 }
1106             }
1107
1108             if ($opts->{protected}) {
1109                 debugmsg('info', "marking backup as protected", $logfd);
1110                 eval { PVE::Storage::update_volume_attribute($cfg, $volid, 'protected', 1) };
1111                 die "unable to set protected flag - $@\n" if $@;
1112             }
1113         }
1114
1115         if ($opts->{remove}) {
1116             my $keepstr = join(', ', map { "$_=$prune_options->{$_}" } sort keys %$prune_options);
1117             debugmsg ('info', "prune older backups with retention: $keepstr", $logfd);
1118             my $pruned = 0;
1119             if (!defined($opts->{storage})) {
1120                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
1121
1122                 PVE::Storage::prune_mark_backup_group($bklist, $prune_options);
1123
1124                 foreach my $prune_entry (@{$bklist}) {
1125                     next if $prune_entry->{mark} ne 'remove';
1126                     $pruned++;
1127                     my $archive_path = $prune_entry->{path};
1128                     debugmsg ('info', "delete old backup '$archive_path'", $logfd);
1129                     PVE::Storage::archive_remove($archive_path);
1130                 }
1131             } else {
1132                 my $pruned_list = PVE::Storage::prune_backups(
1133                     $cfg,
1134                     $opts->{storage},
1135                     $prune_options,
1136                     $vmid,
1137                     $vmtype,
1138                     0,
1139                     sub { debugmsg($_[0], $_[1], $logfd) },
1140                 );
1141                 $pruned = scalar(grep { $_->{mark} eq 'remove' } $pruned_list->@*);
1142             }
1143             my $log_pruned_extra = $pruned > 0 ? " not covered by keep-retention policy" : "";
1144             debugmsg ('info', "pruned $pruned backup(s)${log_pruned_extra}", $logfd);
1145         }
1146
1147         $self->run_hook_script ('backup-end', $task, $logfd);
1148     };
1149     my $err = $@;
1150
1151     if ($plugin) {
1152         # clean-up
1153
1154         if ($cleanup->{unlock}) {
1155             eval { $plugin->unlock_vm ($vmid); };
1156             warn $@ if $@;
1157         }
1158
1159         if ($cleanup->{prepared}) {
1160             # only call cleanup when necessary (when prepare was executed)
1161             eval { $plugin->cleanup ($task, $vmid) };
1162             warn $@ if $@;
1163         }
1164
1165         eval { $plugin->set_logfd (undef); };
1166         warn $@ if $@;
1167
1168         if ($cleanup->{resume} || $cleanup->{restart}) {
1169             eval {
1170                 $self->run_hook_script ('pre-restart', $task, $logfd);
1171                 if ($cleanup->{resume}) {
1172                     debugmsg ('info', "resume vm", $logfd);
1173                     $plugin->resume_vm ($task, $vmid);
1174                 } else {
1175                     my $running = $plugin->vm_status($vmid);
1176                     if (!$running) {
1177                         debugmsg ('info', "restarting vm", $logfd);
1178                         $plugin->start_vm ($task, $vmid);
1179                     }
1180                 }
1181                 $self->run_hook_script ('post-restart', $task, $logfd);
1182             };
1183             my $err = $@;
1184             if ($err) {
1185                 warn $err;
1186             } else {
1187                 $log_vm_online_again->();
1188             }
1189         }
1190     }
1191
1192     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
1193     warn $@ if $@;
1194
1195     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
1196     warn $@ if $@;
1197
1198     my $delay = $task->{backuptime} = time () - $vmstarttime;
1199
1200     if ($err) {
1201         $task->{state} = 'err';
1202         $task->{msg} = $err;
1203         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1204         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
1205
1206         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1207
1208     } else {
1209         $task->{state} = 'ok';
1210         my $tstr = format_time ($delay);
1211         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1212         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1213     }
1214
1215     close ($logfd) if $logfd;
1216
1217     if ($task->{tmplog}) {
1218         if ($self->{opts}->{pbs}) {
1219             if ($task->{state} eq 'ok') {
1220                 eval {
1221                     PVE::Storage::PBSPlugin::run_raw_client_cmd(
1222                         $opts->{scfg},
1223                         $opts->{storage},
1224                         'upload-log',
1225                         [ $pbs_snapshot_name, $task->{tmplog} ],
1226                         errmsg => "uploading backup task log failed",
1227                         outfunc => sub {},
1228                     );
1229                 };
1230                 debugmsg('warn', "$@") if $@; # $@ contains already error prefix
1231             }
1232         } elsif ($task->{logfile}) {
1233             system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1234         }
1235     }
1236
1237     eval { $self->run_hook_script ('log-end', $task); };
1238
1239     die $err if $err && $err =~ m/^interrupted by signal$/;
1240 }
1241
1242 sub exec_backup {
1243     my ($self, $rpcenv, $authuser) = @_;
1244
1245     my $opts = $self->{opts};
1246
1247     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1248
1249     if (scalar(@{$self->{skiplist}})) {
1250         my $skip_string = join(', ', sort { $a <=> $b } @{$self->{skiplist}});
1251         debugmsg ('info', "skip external VMs: $skip_string");
1252     }
1253
1254     my $tasklist = [];
1255     my $vzdump_plugins =  {};
1256     foreach my $plugin (@{$self->{plugins}}) {
1257         my $type = $plugin->type();
1258         next if exists $vzdump_plugins->{$type};
1259         $vzdump_plugins->{$type} = $plugin;
1260     }
1261
1262     my $vmlist = PVE::Cluster::get_vmlist();
1263     my $vmids = [ sort { $a <=> $b } @{$opts->{vmids}} ];
1264     foreach my $vmid (@{$vmids}) {
1265         my $plugin;
1266         if (defined($vmlist->{ids}->{$vmid})) {
1267             my $guest_type = $vmlist->{ids}->{$vmid}->{type};
1268             $plugin = $vzdump_plugins->{$guest_type};
1269             next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all});
1270         }
1271         push @$tasklist, {
1272             mode => $opts->{mode},
1273             plugin => $plugin,
1274             state => 'todo',
1275             vmid => $vmid,
1276         };
1277     }
1278
1279     # Use in-memory files for the outer hook logs to pass them to sendmail.
1280     my $job_start_log = '';
1281     my $job_end_log = '';
1282     open my $job_start_fd, '>', \$job_start_log;
1283     open my $job_end_fd, '>', \$job_end_log;
1284
1285     my $starttime = time();
1286     my $errcount = 0;
1287     eval {
1288
1289         $self->run_hook_script ('job-start', undef, $job_start_fd);
1290
1291         foreach my $task (@$tasklist) {
1292             $self->exec_backup_task ($task);
1293             $errcount += 1 if $task->{state} ne 'ok';
1294         }
1295
1296         $self->run_hook_script ('job-end', undef, $job_end_fd);
1297     };
1298     my $err = $@;
1299
1300     if ($err) {
1301         eval { $self->run_hook_script ('job-abort', undef, $job_end_fd); };
1302         $err .= $@ if $@;
1303         debugmsg ('err', "Backup job failed - $err", undef, 1);
1304     } else {
1305         if ($errcount) {
1306             debugmsg ('info', "Backup job finished with errors", undef, 1);
1307         } else {
1308             debugmsg ('info', "Backup job finished successfully", undef, 1);
1309         }
1310     }
1311
1312     close $job_start_fd;
1313     close $job_end_fd;
1314
1315     my $totaltime = time() - $starttime;
1316
1317     eval {
1318         # otherwise $self->sendmail() will interpret it as multiple problems
1319         my $chomped_err = $err;
1320         chomp($chomped_err) if $chomped_err;
1321
1322         $self->sendmail(
1323             $tasklist,
1324             $totaltime,
1325             $chomped_err,
1326             $self->{job_init_log} . $job_start_log,
1327             $job_end_log,
1328         );
1329     };
1330     debugmsg ('err', $@) if $@;
1331
1332     die $err if $err;
1333
1334     die "job errors\n" if $errcount;
1335
1336     unlink $pidfile;
1337 }
1338
1339
1340 sub option_exists {
1341     my $key = shift;
1342     return defined($confdesc->{$key});
1343 }
1344
1345 # NOTE it might make sense to merge this and verify_vzdump_parameters(), but one
1346 # needs to adapt command_line() in guest-common's PVE/VZDump/Common.pm and detect
1347 # a second parsing attempt, because verify_vzdump_parameters() is called twice
1348 # during the update_job API call.
1349 sub parse_mailto_exclude_path {
1350     my ($param) = @_;
1351
1352     # exclude-path list need to be 0 separated or be an array
1353     if (defined($param->{'exclude-path'})) {
1354         my $expaths;
1355         if (ref($param->{'exclude-path'}) eq 'ARRAY') {
1356             $expaths = $param->{'exclude-path'};
1357         } else {
1358             $expaths = [split(/\0/, $param->{'exclude-path'} || '')];
1359         }
1360         $param->{'exclude-path'} = $expaths;
1361     }
1362
1363     if (defined($param->{mailto})) {
1364         my @mailto = PVE::Tools::split_list(extract_param($param, 'mailto'));
1365         $param->{mailto} = [ @mailto ];
1366     }
1367
1368     return;
1369 }
1370
1371 sub verify_vzdump_parameters {
1372     my ($param, $check_missing) = @_;
1373
1374     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1375         if $param->{all} && $param->{vmid};
1376
1377     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1378         if $param->{exclude} && $param->{vmid};
1379
1380     raise_param_exc({ pool => "option conflicts with option 'vmid'"})
1381         if $param->{pool} && $param->{vmid};
1382
1383     raise_param_exc({ 'prune-backups' => "option conflicts with option 'maxfiles'"})
1384         if defined($param->{'prune-backups'}) && defined($param->{maxfiles});
1385
1386     $parse_prune_backups_maxfiles->($param, 'CLI parameters');
1387     parse_performance($param);
1388
1389     if (my $template = $param->{'notes-template'}) {
1390         eval { $verify_notes_template->($template); };
1391         raise_param_exc({'notes-template' => $@}) if $@;
1392     }
1393
1394     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
1395
1396     return if !$check_missing;
1397
1398     raise_param_exc({ vmid => "property is missing"})
1399         if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
1400
1401 }
1402
1403 sub stop_running_backups {
1404     my($self) = @_;
1405
1406     my $upid = PVE::Tools::file_read_firstline($pidfile);
1407     return if !$upid;
1408
1409     my $task = PVE::Tools::upid_decode($upid);
1410
1411     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1412         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1413         kill(15, $task->{pid});
1414         # wait max 15 seconds to shut down (else, do nothing for now)
1415         my $i;
1416         for ($i = 15; $i > 0; $i--) {
1417             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1418             sleep (1);
1419         }
1420         die "stopping backup process $task->{pid} failed\n" if $i == 0;
1421     }
1422 }
1423
1424 sub get_included_guests {
1425     my ($job) = @_;
1426
1427     my $vmids = [];
1428     my $vmids_per_node = {};
1429
1430     my $vmlist = PVE::Cluster::get_vmlist();
1431
1432     if ($job->{pool}) {
1433         $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool});
1434     } elsif ($job->{vmid}) {
1435         $vmids = [ split_list($job->{vmid}) ];
1436     } elsif ($job->{all}) {
1437         # all or exclude
1438         my $exclude = check_vmids(split_list($job->{exclude}));
1439         my $excludehash = { map { $_ => 1 } @$exclude };
1440
1441         for my $id (keys %{$vmlist->{ids}}) {
1442             next if $excludehash->{$id};
1443             push @$vmids, $id;
1444         }
1445     } else {
1446         return $vmids_per_node;
1447     }
1448     $vmids = check_vmids(@$vmids);
1449
1450     for my $vmid (@$vmids) {
1451         if (defined($vmlist->{ids}->{$vmid})) {
1452             my $node = $vmlist->{ids}->{$vmid}->{node};
1453             next if (defined $job->{node} && $job->{node} ne $node);
1454
1455             push @{$vmids_per_node->{$node}}, $vmid;
1456         } else {
1457             push @{$vmids_per_node->{''}}, $vmid;
1458         }
1459     }
1460
1461     return $vmids_per_node;
1462 }
1463
1464 1;