PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Basename;
   8 use File::Path;
   9 use IO::File;
  10 use IO::Select;
  11 use IPC::Open3;
  12 use POSIX qw(strftime);
  13 use Time::Local;
  14
  15 use PVE::Cluster qw(cfs_read_file);
  16 use PVE::DataCenterConfig;
  17 use PVE::Exception qw(raise_param_exc);
  18 use PVE::HA::Config;
  19 use PVE::HA::Env::PVE2;
  20 use PVE::JSONSchema qw(get_standard_option);
  21 use PVE::RPCEnvironment;
  22 use PVE::Storage;
  23 use PVE::VZDump::Common;
  24 use PVE::VZDump::Plugin;
  25 use PVE::Tools qw(extract_param split_list);
  26 use PVE::API2Tools;
  27
  28 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  29
  30 my $lockfile = '/var/run/vzdump.lock';
  31 my $pidfile = '/var/run/vzdump.pid';
  32 my $logdir = '/var/log/vzdump';
  33
  34 my @plugins = qw();
  35
  36 my $confdesc = PVE::VZDump::Common::get_confdesc();
  37
  38 # Load available plugins
  39 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
  40 foreach my $plug (@pve_vzdump_classes) {
  41     my $filename = "/usr/share/perl5/$plug.pm";
  42     $filename =~ s!::!/!g;
  43     if (-f $filename) {
  44         eval { require $filename; };
  45         if (!$@) {
  46             $plug->import ();
  47             push @plugins, $plug;
  48         } else {
  49             die $@;
  50         }
  51     }
  52 }
  53
  54 # helper functions
  55
  56 sub debugmsg {
  57     my ($mtype, $msg, $logfd, $syslog) = @_;
  58
  59     PVE::VZDump::Plugin::debugmsg(@_);
  60 }
  61
  62 sub run_command {
  63     my ($logfd, $cmdstr, %param) = @_;
  64
  65     my $logfunc = sub {
  66         my $line = shift;
  67         debugmsg ('info', $line, $logfd);
  68     };
  69
  70     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
  71 }
  72
  73 my $generate_notes = sub {
  74     my ($notes_template, $task) = @_;
  75
  76     my $info = {
  77         cluster => PVE::Cluster::get_clinfo()->{cluster}->{name},
  78         guestname => $task->{hostname},
  79         node => PVE::INotify::nodename(),
  80         vmid => $task->{vmid},
  81     };
  82
  83     my $unescape = sub {
  84         my ($char) = @_;
  85         return '\\' if $char eq '\\';
  86         return "\n" if $char eq 'n';
  87         die "unexpected escape character '$char'\n";
  88     };
  89
  90     $notes_template =~ s/\\(.)/$unescape->($1)/eg;
  91
  92     my $vars = join('|', keys $info->%*);
  93     $notes_template =~ s/\{\{($vars)\}\}/\Q$info->{$1}\E/g;
  94
  95     die "unexpected variable name '$1'" if $notes_template =~ m/\{\{([^\s]+)\}\}/;
  96
  97     return $notes_template;
  98 };
  99
 100 my $parse_prune_backups_maxfiles = sub {
 101     my ($param, $kind) = @_;
 102
 103     my $maxfiles = delete $param->{maxfiles};
 104     my $prune_backups = $param->{'prune-backups'};
 105
 106     debugmsg('warn', "both 'maxfiles' and 'prune-backups' defined as ${kind} - ignoring 'maxfiles'")
 107         if defined($maxfiles) && defined($prune_backups);
 108
 109     if (defined($prune_backups)) {
 110         return if ref($prune_backups) eq 'HASH'; # already parsed
 111         $param->{'prune-backups'} = PVE::JSONSchema::parse_property_string(
 112             'prune-backups',
 113             $prune_backups
 114         );
 115     } elsif (defined($maxfiles)) {
 116         if ($maxfiles) {
 117             $param->{'prune-backups'} = { 'keep-last' => $maxfiles };
 118         } else {
 119             $param->{'prune-backups'} = { 'keep-all' => 1 };
 120         }
 121     }
 122 };
 123
 124 sub storage_info {
 125     my $storage = shift;
 126
 127     my $cfg = PVE::Storage::config();
 128     my $scfg = PVE::Storage::storage_config($cfg, $storage);
 129     my $type = $scfg->{type};
 130
 131     die "can't use storage '$storage' for backups - wrong content type\n"
 132         if (!$scfg->{content}->{backup});
 133
 134     my $info = {
 135         scfg => $scfg,
 136     };
 137
 138     $info->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'})
 139         if defined($scfg->{'prune-backups'});
 140
 141     if ($type eq 'pbs') {
 142         $info->{pbs} = 1;
 143     } else {
 144         $info->{dumpdir} = PVE::Storage::get_backup_dir($cfg, $storage);
 145     }
 146
 147     return $info;
 148 }
 149
 150 sub format_size {
 151     my $size = shift;
 152
 153     my $kb = $size / 1024;
 154
 155     if ($kb < 1024) {
 156         return int ($kb) . "KB";
 157     }
 158
 159     my $mb = $size / (1024*1024);
 160     if ($mb < 1024) {
 161         return int ($mb) . "MB";
 162     }
 163     my $gb = $mb / 1024;
 164     if ($gb < 1024) {
 165         return sprintf ("%.2fGB", $gb);
 166     }
 167     my $tb = $gb / 1024;
 168     return sprintf ("%.2fTB", $tb);
 169 }
 170
 171 sub format_time {
 172     my $seconds = shift;
 173
 174     my $hours = int ($seconds/3600);
 175     $seconds = $seconds - $hours*3600;
 176     my $min = int ($seconds/60);
 177     $seconds = $seconds - $min*60;
 178
 179     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 180 }
 181
 182 sub encode8bit {
 183     my ($str) = @_;
 184
 185     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 186
 187     return $str;
 188 }
 189
 190 sub escape_html {
 191     my ($str) = @_;
 192
 193     $str =~ s/&/&amp;/g;
 194     $str =~ s/</&lt;/g;
 195     $str =~ s/>/&gt;/g;
 196
 197     return $str;
 198 }
 199
 200 sub check_bin {
 201     my ($bin)  = @_;
 202
 203     foreach my $p (split (/:/, $ENV{PATH})) {
 204         my $fn = "$p/$bin";
 205         if (-x $fn) {
 206             return $fn;
 207         }
 208     }
 209
 210     die "unable to find command '$bin'\n";
 211 }
 212
 213 sub check_vmids {
 214     my (@vmids) = @_;
 215
 216     my $res = [];
 217     for my $vmid (sort {$a <=> $b} @vmids) {
 218         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 219         $vmid = int ($vmid); # remove leading zeros
 220         next if !$vmid;
 221         push @$res, $vmid;
 222     }
 223
 224     return $res;
 225 }
 226
 227
 228 sub read_vzdump_defaults {
 229
 230     my $fn = "/etc/vzdump.conf";
 231
 232     my $defaults = {
 233         map {
 234             my $default = $confdesc->{$_}->{default};
 235              defined($default) ? ($_ => $default) : ()
 236         } keys %$confdesc
 237     };
 238     $parse_prune_backups_maxfiles->($defaults, "defaults in VZDump schema");
 239
 240     my $raw;
 241     eval { $raw = PVE::Tools::file_get_contents($fn); };
 242     return $defaults if $@;
 243
 244     my $conf_schema = { type => 'object', properties => $confdesc, };
 245     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 246     if (my $excludes = $res->{'exclude-path'}) {
 247         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 248     }
 249     if (defined($res->{mailto})) {
 250         my @mailto = split_list($res->{mailto});
 251         $res->{mailto} = [ @mailto ];
 252     }
 253     $parse_prune_backups_maxfiles->($res, "options in '$fn'");
 254
 255     foreach my $key (keys %$defaults) {
 256         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 257     }
 258
 259     if (defined($res->{storage}) && defined($res->{dumpdir})) {
 260         debugmsg('warn', "both 'storage' and 'dumpdir' defined in '$fn' - ignoring 'dumpdir'");
 261         delete $res->{dumpdir};
 262     }
 263
 264     return $res;
 265 }
 266
 267 use constant MAX_MAIL_SIZE => 1024*1024;
 268 sub sendmail {
 269     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 270
 271     my $opts = $self->{opts};
 272
 273     my $mailto = $opts->{mailto};
 274
 275     return if !($mailto && scalar(@$mailto));
 276
 277     my $cmdline = $self->{cmdline};
 278
 279     my $ecount = 0;
 280     foreach my $task (@$tasklist) {
 281         $ecount++ if $task->{state} ne 'ok';
 282         chomp $task->{msg} if $task->{msg};
 283         $task->{backuptime} = 0 if !$task->{backuptime};
 284         $task->{size} = 0 if !$task->{size};
 285         $task->{target} = 'unknown' if !$task->{target};
 286         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 287
 288         if ($task->{state} eq 'todo') {
 289             $task->{msg} = 'aborted';
 290         }
 291     }
 292
 293     my $notify = $opts->{mailnotification} || 'always';
 294     return if (!$ecount && !$err && ($notify eq 'failure'));
 295
 296     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 297     if ($err) {
 298         if ($err =~ /\n/) {
 299             $stat .= ": multiple problems";
 300         } else {
 301             $stat .= ": $err";
 302             $err = undef;
 303         }
 304     }
 305
 306     my $hostname = `hostname -f` || PVE::INotify::nodename();
 307     chomp $hostname;
 308
 309     # text part
 310     my $text = $err ? "$err\n\n" : '';
 311     my $namelength = 20;
 312     $text .= sprintf (
 313         "%-10s %-${namelength}s %-6s %10s %10s  %s\n",
 314         qw(VMID NAME STATUS TIME SIZE FILENAME)
 315     );
 316     foreach my $task (@$tasklist) {
 317         my $name = substr($task->{hostname}, 0, $namelength);
 318         my $successful = $task->{state} eq 'ok';
 319         my $size = $successful ? format_size ($task->{size}) : 0;
 320         my $filename = $successful ? $task->{target} : '-';
 321         my $size_fmt = $successful ? "%10s": "%8.2fMB";
 322         $text .= sprintf(
 323             "%-10s %-${namelength}s %-6s %10s $size_fmt  %s\n",
 324             $task->{vmid},
 325             $name,
 326             $task->{state},
 327             format_time($task->{backuptime}),
 328             $size,
 329             $filename,
 330         );
 331     }
 332
 333     my $text_log_part;
 334     $text_log_part .= "\nDetailed backup logs:\n\n";
 335     $text_log_part .= "$cmdline\n\n";
 336
 337     $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
 338     foreach my $task (@$tasklist) {
 339         my $vmid = $task->{vmid};
 340         my $log = $task->{tmplog};
 341         if (!$log) {
 342             $text_log_part .= "$vmid: no log available\n\n";
 343             next;
 344         }
 345         if (open (my $TMP, '<', "$log")) {
 346             while (my $line = <$TMP>) {
 347                 next if $line =~ /^status: \d+/; # not useful in mails
 348                 $text_log_part .= encode8bit ("$vmid: $line");
 349             }
 350             close ($TMP);
 351         } else {
 352             $text_log_part .= "$vmid: Could not open log file\n\n";
 353         }
 354         $text_log_part .= "\n";
 355     }
 356     $text_log_part .= $detail_post if defined($detail_post);
 357
 358     # html part
 359     my $html = "<html><body>\n";
 360     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 361     $html .= "<table border=1 cellpadding=3>\n";
 362     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 363
 364     my $ssize = 0;
 365     foreach my $task (@$tasklist) {
 366         my $vmid = $task->{vmid};
 367         my $name = $task->{hostname};
 368
 369         if  ($task->{state} eq 'ok') {
 370             $ssize += $task->{size};
 371
 372             $html .= sprintf (
 373                 "<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 374                 $vmid,
 375                 $name,
 376                 format_time($task->{backuptime}),
 377                 format_size ($task->{size}),
 378                 escape_html ($task->{target}),
 379             );
 380         } else {
 381             $html .= sprintf (
 382                 "<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 383                 $vmid,
 384                 $name,
 385                 format_time($task->{backuptime}),
 386                 escape_html ($task->{msg}),
 387             );
 388         }
 389     }
 390
 391     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 392  format_time ($totaltime), format_size ($ssize));
 393
 394     $html .= "\n</table><br><br>\n";
 395     my $html_log_part;
 396     $html_log_part .= "Detailed backup logs:<br /><br />\n";
 397     $html_log_part .= "<pre>\n";
 398     $html_log_part .= escape_html($cmdline) . "\n\n";
 399
 400     $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 401     foreach my $task (@$tasklist) {
 402         my $vmid = $task->{vmid};
 403         my $log = $task->{tmplog};
 404         if (!$log) {
 405             $html_log_part .= "$vmid: no log available\n\n";
 406             next;
 407         }
 408         if (open (my $TMP, '<', "$log")) {
 409             while (my $line = <$TMP>) {
 410                 next if $line =~ /^status: \d+/; # not useful in mails
 411                 if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 412                     $html_log_part .= encode8bit ("$vmid: <font color=red>".
 413                         escape_html ($line) . "</font>");
 414                 } else {
 415                     $html_log_part .= encode8bit ("$vmid: " . escape_html ($line));
 416                 }
 417             }
 418             close ($TMP);
 419         } else {
 420             $html_log_part .= "$vmid: Could not open log file\n\n";
 421         }
 422         $html_log_part .= "\n";
 423     }
 424     $html_log_part .= escape_html($detail_post) if defined($detail_post);
 425     $html_log_part .= "</pre>";
 426     my $html_end = "\n</body></html>\n";
 427     # end html part
 428
 429     if (length($text) + length($text_log_part) +
 430         length($html) + length($html_log_part) +
 431         length($html_end) < MAX_MAIL_SIZE)
 432     {
 433         $html .= $html_log_part;
 434         $html .= $html_end;
 435         $text .= $text_log_part;
 436     } else {
 437         my $msg = "Log output was too long to be sent by mail. ".
 438             "See Task History for details!\n";
 439         $text .= $msg;
 440         $html .= "<p>$msg</p>";
 441         $html .= $html_end;
 442     }
 443
 444     my $subject = "vzdump backup status ($hostname) : $stat";
 445
 446     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 447     my $mailfrom = $dcconf->{email_from} || "root";
 448
 449     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 450 };
 451
 452 sub new {
 453     my ($class, $cmdline, $opts, $skiplist) = @_;
 454
 455     mkpath $logdir;
 456
 457     check_bin ('cp');
 458     check_bin ('df');
 459     check_bin ('sendmail');
 460     check_bin ('rsync');
 461     check_bin ('tar');
 462     check_bin ('mount');
 463     check_bin ('umount');
 464     check_bin ('cstream');
 465     check_bin ('ionice');
 466
 467     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 468         check_bin ('lvcreate');
 469         check_bin ('lvs');
 470         check_bin ('lvremove');
 471     }
 472
 473     my $defaults = read_vzdump_defaults();
 474
 475     foreach my $k (keys %$defaults) {
 476         next if $k eq 'exclude-path' || $k eq 'prune-backups'; # dealt with separately
 477         if ($k eq 'dumpdir' || $k eq 'storage') {
 478             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 479                 !defined ($opts->{storage});
 480         } else {
 481             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 482         }
 483     }
 484
 485     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 486     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 487
 488     $skiplist = [] if !$skiplist;
 489     my $self = bless {
 490         cmdline => $cmdline,
 491         opts => $opts,
 492         skiplist => $skiplist,
 493     }, $class;
 494
 495     my $findexcl = $self->{findexcl} = [];
 496     if ($defaults->{'exclude-path'}) {
 497         push @$findexcl, @{$defaults->{'exclude-path'}};
 498     }
 499
 500     if ($opts->{'exclude-path'}) {
 501         push @$findexcl, @{$opts->{'exclude-path'}};
 502     }
 503
 504     if ($opts->{stdexcludes}) {
 505         push @$findexcl,
 506             '/tmp/?*',
 507             '/var/tmp/?*',
 508             '/var/run/?*.pid',
 509             ;
 510     }
 511
 512     foreach my $p (@plugins) {
 513         my $pd = $p->new($self);
 514
 515         push @{$self->{plugins}}, $pd;
 516     }
 517
 518     if (defined($opts->{storage}) && $opts->{stdout}) {
 519         die "cannot use options 'storage' and 'stdout' at the same time\n";
 520     } elsif (defined($opts->{storage}) && defined($opts->{dumpdir})) {
 521         die "cannot use options 'storage' and 'dumpdir' at the same time\n";
 522     }
 523
 524     if (!$opts->{dumpdir} && !$opts->{storage}) {
 525         $opts->{storage} = 'local';
 526     }
 527
 528     my $errors = '';
 529     my $add_error = sub {
 530         my ($error) = @_;
 531         $errors .= "\n" if $errors;
 532         chomp($error);
 533         $errors .= $error;
 534     };
 535
 536     eval {
 537         $self->{job_init_log} = '';
 538         open my $job_init_fd, '>', \$self->{job_init_log};
 539         $self->run_hook_script('job-init', undef, $job_init_fd);
 540         close $job_init_fd;
 541
 542         PVE::Cluster::cfs_update(); # Pick up possible changes made by the hook script.
 543     };
 544     $add_error->($@) if $@;
 545
 546     if ($opts->{storage}) {
 547         my $storage_cfg = PVE::Storage::config();
 548         eval { PVE::Storage::activate_storage($storage_cfg, $opts->{storage}) };
 549         $add_error->("could not activate storage '$opts->{storage}': $@") if $@;
 550
 551         my $info = eval { storage_info ($opts->{storage}) };
 552         if (my $err = $@) {
 553             $add_error->("could not get storage information for '$opts->{storage}': $err");
 554         } else {
 555             $opts->{dumpdir} = $info->{dumpdir};
 556             $opts->{scfg} = $info->{scfg};
 557             $opts->{pbs} = $info->{pbs};
 558             $opts->{'prune-backups'} //= $info->{'prune-backups'};
 559         }
 560     } elsif ($opts->{dumpdir}) {
 561         $add_error->("dumpdir '$opts->{dumpdir}' does not exist")
 562             if ! -d $opts->{dumpdir};
 563     } else {
 564         die "internal error";
 565     }
 566
 567     $opts->{'prune-backups'} //= $defaults->{'prune-backups'};
 568
 569     # avoid triggering any remove code path if keep-all is set
 570     $opts->{remove} = 0 if $opts->{'prune-backups'}->{'keep-all'};
 571
 572     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 573         $add_error->("tmpdir '$opts->{tmpdir}' does not exist");
 574     }
 575
 576     if ($errors) {
 577         eval { $self->sendmail([], 0, $errors); };
 578         debugmsg ('err', $@) if $@;
 579         die "$errors\n";
 580     }
 581
 582     return $self;
 583 }
 584
 585 sub get_mount_info {
 586     my ($dir) = @_;
 587
 588     # Note: df 'available' can be negative, and percentage set to '-'
 589
 590     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 591
 592     my $res;
 593
 594     my $parser = sub {
 595         my $line = shift;
 596         if (my ($fsid, $fstype, undef, $mp) = $line =~
 597             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 598             $res = {
 599                 device => $fsid,
 600                 fstype => $fstype,
 601                 mountpoint => $mp,
 602             };
 603         }
 604     };
 605
 606     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 607     warn $@ if $@;
 608
 609     return $res;
 610 }
 611
 612 sub getlock {
 613     my ($self, $upid) = @_;
 614
 615     my $fh;
 616
 617     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 618
 619     die "missing UPID" if !$upid; # should not happen
 620
 621     my $SERVER_FLCK;
 622     if (!open ($SERVER_FLCK, '>>', "$lockfile")) {
 623         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 624         die "can't open lock on file '$lockfile' - $!";
 625     }
 626
 627     if (!flock ($SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 628         if (!$maxwait) {
 629             debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
 630             die "can't acquire lock '$lockfile' (wait = 0)";
 631         }
 632
 633         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 634         eval {
 635             alarm ($maxwait * 60);
 636
 637             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 638
 639             if (!flock ($SERVER_FLCK, LOCK_EX)) {
 640                 my $err = $!;
 641                 close ($SERVER_FLCK);
 642                 alarm (0);
 643                 die "$err\n";
 644             }
 645             alarm (0);
 646         };
 647         alarm (0);
 648
 649         my $err = $@;
 650
 651         if ($err) {
 652             debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
 653             die "can't acquire lock '$lockfile' - $err";
 654         }
 655
 656         debugmsg('info', "got global lock", undef, 1);
 657     }
 658
 659     PVE::Tools::file_set_contents($pidfile, $upid);
 660
 661     return $SERVER_FLCK;
 662 }
 663
 664 sub run_hook_script {
 665     my ($self, $phase, $task, $logfd) = @_;
 666
 667     my $opts = $self->{opts};
 668
 669     my $script = $opts->{script};
 670     return if !$script;
 671
 672     die "Error: The hook script '$script' does not exist.\n" if ! -f $script;
 673     die "Error: The hook script '$script' is not executable.\n" if ! -x $script;
 674
 675     my $cmd = [$script, $phase];
 676
 677     if ($task) {
 678         push @$cmd, $task->{mode};
 679         push @$cmd, $task->{vmid};
 680     }
 681
 682     local %ENV;
 683     # set immutable opts directly (so they are available in all phases)
 684     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 685     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 686
 687     foreach my $ek (qw(vmtype hostname target logfile)) {
 688         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 689     }
 690
 691     run_command ($logfd, $cmd);
 692 }
 693
 694 sub compressor_info {
 695     my ($opts) = @_;
 696     my $opt_compress = $opts->{compress};
 697
 698     if (!$opt_compress || $opt_compress eq '0') {
 699         return undef;
 700     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 701         return ('lzop', 'lzo');
 702     } elsif ($opt_compress eq 'gzip') {
 703         if ($opts->{pigz} > 0) {
 704             my $pigz_threads = $opts->{pigz};
 705             if ($pigz_threads == 1) {
 706                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 707                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 708             }
 709             return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
 710         } else {
 711             return ('gzip --rsyncable', 'gz');
 712         }
 713     } elsif ($opt_compress eq 'zstd') {
 714         my $zstd_threads = $opts->{zstd} // 1;
 715         if ($zstd_threads == 0) {
 716             my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 717             $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
 718         }
 719         return ("zstd --rsyncable --threads=${zstd_threads}", 'zst');
 720     } else {
 721         die "internal error - unknown compression option '$opt_compress'";
 722     }
 723 }
 724
 725 sub get_backup_file_list {
 726     my ($dir, $bkname) = @_;
 727
 728     my $bklist = [];
 729     foreach my $fn (<$dir/${bkname}-*>) {
 730         my $archive_info = eval { PVE::Storage::archive_info($fn) } // {};
 731         if ($archive_info->{is_std_name}) {
 732             my $path = "$dir/$archive_info->{filename}";
 733             my $backup = {
 734                 'path' => $path,
 735                 'ctime' => $archive_info->{ctime},
 736             };
 737             $backup->{mark} = "protected"
 738                 if -e PVE::Storage::protection_file_path($path);
 739             push @{$bklist}, $backup;
 740         }
 741     }
 742
 743     return $bklist;
 744 }
 745
 746 sub exec_backup_task {
 747     my ($self, $task) = @_;
 748
 749     my $opts = $self->{opts};
 750
 751     my $cfg = PVE::Storage::config();
 752     my $vmid = $task->{vmid};
 753     my $plugin = $task->{plugin};
 754
 755     $task->{backup_time} = time();
 756
 757     my $pbs_group_name;
 758     my $pbs_snapshot_name;
 759
 760     my $vmstarttime = time ();
 761
 762     my $logfd;
 763
 764     my $cleanup = {};
 765
 766     my $log_vm_online_again = sub {
 767         return if !defined($task->{vmstoptime});
 768         $task->{vmconttime} //= time();
 769         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 770         $delay = '<1' if $delay < 1;
 771         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 772     };
 773
 774     eval {
 775         die "unable to find VM '$vmid'\n" if !$plugin;
 776
 777         my $vmtype = $plugin->type();
 778
 779         if ($self->{opts}->{pbs}) {
 780             if ($vmtype eq 'lxc') {
 781                 $pbs_group_name = "ct/$vmid";
 782             } elsif  ($vmtype eq 'qemu') {
 783                 $pbs_group_name = "vm/$vmid";
 784             } else {
 785                 die "pbs backup not implemented for plugin type '$vmtype'\n";
 786             }
 787             my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
 788             $pbs_snapshot_name = "$pbs_group_name/$btime";
 789         }
 790
 791         # for now we deny backups of a running ha managed service in *stop* mode
 792         # as it interferes with the HA stack (started services should not stop).
 793         if ($opts->{mode} eq 'stop' &&
 794             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 795         {
 796             die "Cannot execute a backup with stop mode on a HA managed and".
 797                 " enabled Service. Use snapshot mode or disable the Service.\n";
 798         }
 799
 800         my $tmplog = "$logdir/$vmtype-$vmid.log";
 801
 802         my $bkname = "vzdump-$vmtype-$vmid";
 803         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
 804
 805         my $prune_options = $opts->{'prune-backups'};
 806
 807         my $backup_limit = 0;
 808         if (!$prune_options->{'keep-all'}) {
 809             foreach my $keep (values %{$prune_options}) {
 810                 $backup_limit += $keep;
 811             }
 812         }
 813
 814         if (($backup_limit && !$opts->{remove}) || $opts->{protected}) {
 815             my $count;
 816             my $protected_count;
 817             if (my $storeid = $opts->{storage}) {
 818                 my @backups = grep {
 819                     !$_->{subtype} || $_->{subtype} eq $vmtype
 820                 } PVE::Storage::volume_list($cfg, $storeid, $vmid, 'backup')->@*;
 821
 822                 $count = grep { !$_->{protected} } @backups;
 823                 $protected_count = scalar(@backups) - $count;
 824             } else {
 825                 $count = grep { !$_->{mark} || $_->{mark} ne "protected" } get_backup_file_list($opts->{dumpdir}, $bkname)->@*;
 826             }
 827
 828             if ($opts->{protected}) {
 829                 my $max_protected = PVE::Storage::get_max_protected_backups(
 830                     $opts->{scfg},
 831                     $opts->{storage},
 832                 );
 833                 if ($max_protected > -1 && $protected_count >= $max_protected) {
 834                     die "The number of protected backups per guest is limited to $max_protected ".
 835                         "on storage '$opts->{storage}'\n";
 836                 }
 837             } elsif ($count >= $backup_limit) {
 838                 die "There is a max backup limit of $backup_limit enforced by the target storage ".
 839                     "or the vzdump parameters. Either increase the limit or delete old backups.\n";
 840             }
 841         }
 842
 843         if (!$self->{opts}->{pbs}) {
 844             $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 845         }
 846
 847         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 848         my ($comp, $comp_ext) = compressor_info($opts);
 849         if ($comp && $comp_ext) {
 850             $ext .= ".${comp_ext}";
 851         }
 852
 853         if ($self->{opts}->{pbs}) {
 854             die "unable to pipe backup to stdout\n" if $opts->{stdout};
 855             $task->{target} = $pbs_snapshot_name;
 856         } else {
 857             if ($opts->{stdout}) {
 858                 $task->{target} = '-';
 859             } else {
 860                 $task->{target} = $task->{tmptar} = "$opts->{dumpdir}/$basename$ext";
 861                 $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 862                 unlink $task->{tmptar};
 863             }
 864         }
 865
 866         $task->{vmtype} = $vmtype;
 867
 868         my $pid = $$;
 869         if ($opts->{tmpdir}) {
 870             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp${pid}_$vmid/";
 871         } elsif ($self->{opts}->{pbs}) {
 872             $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 873         } else {
 874             # dumpdir is posix? then use it as temporary dir
 875             my $info = get_mount_info($opts->{dumpdir});
 876             if ($vmtype eq 'qemu' ||
 877                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 878                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 879             } else {
 880                 $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 881                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 882                           "using $task->{tmpdir} for temporary files", $logfd);
 883             }
 884         }
 885
 886         rmtree $task->{tmpdir};
 887         mkdir $task->{tmpdir};
 888         -d $task->{tmpdir} ||
 889             die "unable to create temporary directory '$task->{tmpdir}'";
 890
 891         $logfd = IO::File->new (">$tmplog") ||
 892             die "unable to create log file '$tmplog'";
 893
 894         $task->{dumpdir} = $opts->{dumpdir};
 895         $task->{storeid} = $opts->{storage};
 896         $task->{scfg} = $opts->{scfg};
 897         $task->{tmplog} = $tmplog;
 898
 899         unlink $task->{logfile} if defined($task->{logfile});
 900
 901         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 902         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
 903
 904         $plugin->set_logfd ($logfd);
 905
 906         # test is VM is running
 907         my ($running, $status_text) = $plugin->vm_status ($vmid);
 908
 909         debugmsg ('info', "status = ${status_text}", $logfd);
 910
 911         # lock VM (prevent config changes)
 912         $plugin->lock_vm ($vmid);
 913
 914         $cleanup->{unlock} = 1;
 915
 916         # prepare
 917
 918         my $mode = $running ? $task->{mode} : 'stop';
 919
 920         if ($mode eq 'snapshot') {
 921             my %saved_task = %$task;
 922             eval { $plugin->prepare ($task, $vmid, $mode); };
 923             if (my $err = $@) {
 924                 die $err if $err !~ m/^mode failure/;
 925                 debugmsg ('info',  $err, $logfd);
 926                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 927                 $mode = 'suspend'; # so prepare is called again below
 928                 %$task = %saved_task;
 929             }
 930         }
 931
 932         $cleanup->{prepared} = 1;
 933
 934         $task->{mode} = $mode;
 935
 936         debugmsg ('info', "backup mode: $mode", $logfd);
 937         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)  if $opts->{bwlimit};
 938         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 939
 940         if ($mode eq 'stop') {
 941             $plugin->prepare ($task, $vmid, $mode);
 942
 943             $self->run_hook_script ('backup-start', $task, $logfd);
 944
 945             if ($running) {
 946                 debugmsg ('info', "stopping virtual guest", $logfd);
 947                 $task->{vmstoptime} = time();
 948                 $self->run_hook_script ('pre-stop', $task, $logfd);
 949                 $plugin->stop_vm ($task, $vmid);
 950                 $cleanup->{restart} = 1;
 951             }
 952
 953
 954         } elsif ($mode eq 'suspend') {
 955             $plugin->prepare ($task, $vmid, $mode);
 956
 957             $self->run_hook_script ('backup-start', $task, $logfd);
 958
 959             if ($vmtype eq 'lxc') {
 960                 # pre-suspend rsync
 961                 $plugin->copy_data_phase1($task, $vmid);
 962             }
 963
 964             debugmsg ('info', "suspending guest", $logfd);
 965             $task->{vmstoptime} = time ();
 966             $self->run_hook_script ('pre-stop', $task, $logfd);
 967             $plugin->suspend_vm ($task, $vmid);
 968             $cleanup->{resume} = 1;
 969
 970             if ($vmtype eq 'lxc') {
 971                 # post-suspend rsync
 972                 $plugin->copy_data_phase2($task, $vmid);
 973
 974                 debugmsg ('info', "resuming guest", $logfd);
 975                 $cleanup->{resume} = 0;
 976                 $self->run_hook_script('pre-restart', $task, $logfd);
 977                 $plugin->resume_vm($task, $vmid);
 978                 $self->run_hook_script('post-restart', $task, $logfd);
 979                 $log_vm_online_again->();
 980             }
 981
 982         } elsif ($mode eq 'snapshot') {
 983             $self->run_hook_script ('backup-start', $task, $logfd);
 984
 985             my $snapshot_count = $task->{snapshot_count} || 0;
 986
 987             $self->run_hook_script ('pre-stop', $task, $logfd);
 988
 989             if ($snapshot_count > 1) {
 990                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 991                 $task->{vmstoptime} = time ();
 992                 $plugin->suspend_vm ($task, $vmid);
 993                 $cleanup->{resume} = 1;
 994             }
 995
 996             $plugin->snapshot ($task, $vmid);
 997
 998             $self->run_hook_script ('pre-restart', $task, $logfd);
 999
1000             if ($snapshot_count > 1) {
1001                 debugmsg ('info', "resume vm", $logfd);
1002                 $cleanup->{resume} = 0;
1003                 $plugin->resume_vm ($task, $vmid);
1004                 $log_vm_online_again->();
1005             }
1006
1007             $self->run_hook_script ('post-restart', $task, $logfd);
1008
1009         } else {
1010             die "internal error - unknown mode '$mode'\n";
1011         }
1012
1013         # assemble archive image
1014         $plugin->assemble ($task, $vmid);
1015
1016         # produce archive
1017
1018         if ($opts->{stdout}) {
1019             debugmsg ('info', "sending archive to stdout", $logfd);
1020             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
1021             $self->run_hook_script ('backup-end', $task, $logfd);
1022             return;
1023         }
1024
1025         my $archive_txt = $self->{opts}->{pbs} ? 'Proxmox Backup Server' : 'vzdump';
1026         debugmsg('info', "creating $archive_txt archive '$task->{target}'", $logfd);
1027         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
1028
1029         if ($self->{opts}->{pbs}) {
1030             # size is added to task struct in guest vzdump plugins
1031         } else {
1032             rename ($task->{tmptar}, $task->{target}) ||
1033                 die "unable to rename '$task->{tmptar}' to '$task->{target}'\n";
1034
1035             # determine size
1036             $task->{size} = (-s $task->{target}) || 0;
1037             my $cs = format_size ($task->{size});
1038             debugmsg ('info', "archive file size: $cs", $logfd);
1039         }
1040
1041         # Mark as protected before pruning.
1042         if (my $storeid = $opts->{storage}) {
1043             my $volname = $opts->{pbs} ? $task->{target} : basename($task->{target});
1044             my $volid = "${storeid}:backup/${volname}";
1045
1046             if ($opts->{'notes-template'} && $opts->{'notes-template'} ne '') {
1047                 debugmsg('info', "adding notes to backup", $logfd);
1048                 my $notes = eval { $generate_notes->($opts->{'notes-template'}, $task); };
1049                 if (my $err = $@) {
1050                     debugmsg('warn', "unable to add notes - $err", $logfd);
1051                 } else {
1052                     eval { PVE::Storage::update_volume_attribute($cfg, $volid, 'notes', $notes) };
1053                     debugmsg('warn', "unable to add notes - $@", $logfd) if $@;
1054                 }
1055             }
1056
1057             if ($opts->{protected}) {
1058                 debugmsg('info', "marking backup as protected", $logfd);
1059                 eval { PVE::Storage::update_volume_attribute($cfg, $volid, 'protected', 1) };
1060                 die "unable to set protected flag - $@\n" if $@;
1061             }
1062         }
1063
1064         if ($opts->{remove}) {
1065             my $keepstr = join(', ', map { "$_=$prune_options->{$_}" } sort keys %$prune_options);
1066             debugmsg ('info', "prune older backups with retention: $keepstr", $logfd);
1067             my $pruned = 0;
1068             if (!defined($opts->{storage})) {
1069                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
1070
1071                 PVE::Storage::prune_mark_backup_group($bklist, $prune_options);
1072
1073                 foreach my $prune_entry (@{$bklist}) {
1074                     next if $prune_entry->{mark} ne 'remove';
1075                     $pruned++;
1076                     my $archive_path = $prune_entry->{path};
1077                     debugmsg ('info', "delete old backup '$archive_path'", $logfd);
1078                     PVE::Storage::archive_remove($archive_path);
1079                 }
1080             } else {
1081                 my $pruned_list = PVE::Storage::prune_backups(
1082                     $cfg,
1083                     $opts->{storage},
1084                     $prune_options,
1085                     $vmid,
1086                     $vmtype,
1087                     0,
1088                     sub { debugmsg($_[0], $_[1], $logfd) },
1089                 );
1090                 $pruned = scalar(grep { $_->{mark} eq 'remove' } $pruned_list->@*);
1091             }
1092             my $log_pruned_extra = $pruned > 0 ? " not covered by keep-retention policy" : "";
1093             debugmsg ('info', "pruned $pruned backup(s)${log_pruned_extra}", $logfd);
1094         }
1095
1096         $self->run_hook_script ('backup-end', $task, $logfd);
1097     };
1098     my $err = $@;
1099
1100     if ($plugin) {
1101         # clean-up
1102
1103         if ($cleanup->{unlock}) {
1104             eval { $plugin->unlock_vm ($vmid); };
1105             warn $@ if $@;
1106         }
1107
1108         if ($cleanup->{prepared}) {
1109             # only call cleanup when necessary (when prepare was executed)
1110             eval { $plugin->cleanup ($task, $vmid) };
1111             warn $@ if $@;
1112         }
1113
1114         eval { $plugin->set_logfd (undef); };
1115         warn $@ if $@;
1116
1117         if ($cleanup->{resume} || $cleanup->{restart}) {
1118             eval {
1119                 $self->run_hook_script ('pre-restart', $task, $logfd);
1120                 if ($cleanup->{resume}) {
1121                     debugmsg ('info', "resume vm", $logfd);
1122                     $plugin->resume_vm ($task, $vmid);
1123                 } else {
1124                     my $running = $plugin->vm_status($vmid);
1125                     if (!$running) {
1126                         debugmsg ('info', "restarting vm", $logfd);
1127                         $plugin->start_vm ($task, $vmid);
1128                     }
1129                 }
1130                 $self->run_hook_script ('post-restart', $task, $logfd);
1131             };
1132             my $err = $@;
1133             if ($err) {
1134                 warn $err;
1135             } else {
1136                 $log_vm_online_again->();
1137             }
1138         }
1139     }
1140
1141     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
1142     warn $@ if $@;
1143
1144     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
1145     warn $@ if $@;
1146
1147     my $delay = $task->{backuptime} = time () - $vmstarttime;
1148
1149     if ($err) {
1150         $task->{state} = 'err';
1151         $task->{msg} = $err;
1152         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1153         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
1154
1155         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1156
1157     } else {
1158         $task->{state} = 'ok';
1159         my $tstr = format_time ($delay);
1160         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1161         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1162     }
1163
1164     close ($logfd) if $logfd;
1165
1166     if ($task->{tmplog}) {
1167         if ($self->{opts}->{pbs}) {
1168             if ($task->{state} eq 'ok') {
1169                 eval {
1170                     PVE::Storage::PBSPlugin::run_raw_client_cmd(
1171                         $opts->{scfg},
1172                         $opts->{storage},
1173                         'upload-log',
1174                         [ $pbs_snapshot_name, $task->{tmplog} ],
1175                         errmsg => "uploading backup task log failed",
1176                         outfunc => sub {},
1177                     );
1178                 };
1179                 debugmsg('warn', "$@") if $@; # $@ contains already error prefix
1180             }
1181         } elsif ($task->{logfile}) {
1182             system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1183         }
1184     }
1185
1186     eval { $self->run_hook_script ('log-end', $task); };
1187
1188     die $err if $err && $err =~ m/^interrupted by signal$/;
1189 }
1190
1191 sub exec_backup {
1192     my ($self, $rpcenv, $authuser) = @_;
1193
1194     my $opts = $self->{opts};
1195
1196     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1197
1198     if (scalar(@{$self->{skiplist}})) {
1199         my $skip_string = join(', ', sort { $a <=> $b } @{$self->{skiplist}});
1200         debugmsg ('info', "skip external VMs: $skip_string");
1201     }
1202
1203     my $tasklist = [];
1204     my $vzdump_plugins =  {};
1205     foreach my $plugin (@{$self->{plugins}}) {
1206         my $type = $plugin->type();
1207         next if exists $vzdump_plugins->{$type};
1208         $vzdump_plugins->{$type} = $plugin;
1209     }
1210
1211     my $vmlist = PVE::Cluster::get_vmlist();
1212     my $vmids = [ sort { $a <=> $b } @{$opts->{vmids}} ];
1213     foreach my $vmid (@{$vmids}) {
1214         my $plugin;
1215         if (defined($vmlist->{ids}->{$vmid})) {
1216             my $guest_type = $vmlist->{ids}->{$vmid}->{type};
1217             $plugin = $vzdump_plugins->{$guest_type};
1218             next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all});
1219         }
1220         push @$tasklist, {
1221             mode => $opts->{mode},
1222             plugin => $plugin,
1223             state => 'todo',
1224             vmid => $vmid,
1225         };
1226     }
1227
1228     # Use in-memory files for the outer hook logs to pass them to sendmail.
1229     my $job_start_log = '';
1230     my $job_end_log = '';
1231     open my $job_start_fd, '>', \$job_start_log;
1232     open my $job_end_fd, '>', \$job_end_log;
1233
1234     my $starttime = time();
1235     my $errcount = 0;
1236     eval {
1237
1238         $self->run_hook_script ('job-start', undef, $job_start_fd);
1239
1240         foreach my $task (@$tasklist) {
1241             $self->exec_backup_task ($task);
1242             $errcount += 1 if $task->{state} ne 'ok';
1243         }
1244
1245         $self->run_hook_script ('job-end', undef, $job_end_fd);
1246     };
1247     my $err = $@;
1248
1249     $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1250
1251     if ($err) {
1252         debugmsg ('err', "Backup job failed - $err", undef, 1);
1253     } else {
1254         if ($errcount) {
1255             debugmsg ('info', "Backup job finished with errors", undef, 1);
1256         } else {
1257             debugmsg ('info', "Backup job finished successfully", undef, 1);
1258         }
1259     }
1260
1261     close $job_start_fd;
1262     close $job_end_fd;
1263
1264     my $totaltime = time() - $starttime;
1265
1266     eval {
1267         $self->sendmail(
1268             $tasklist,
1269             $totaltime,
1270             undef,
1271             $self->{job_init_log} . $job_start_log,
1272             $job_end_log,
1273         );
1274     };
1275     debugmsg ('err', $@) if $@;
1276
1277     die $err if $err;
1278
1279     die "job errors\n" if $errcount;
1280
1281     unlink $pidfile;
1282 }
1283
1284
1285 sub option_exists {
1286     my $key = shift;
1287     return defined($confdesc->{$key});
1288 }
1289
1290 # NOTE it might make sense to merge this and verify_vzdump_parameters(), but one
1291 # needs to adapt command_line() in guest-common's PVE/VZDump/Common.pm and detect
1292 # a second parsing attempt, because verify_vzdump_parameters() is called twice
1293 # during the update_job API call.
1294 sub parse_mailto_exclude_path {
1295     my ($param) = @_;
1296
1297     # exclude-path list need to be 0 separated
1298     if (defined($param->{'exclude-path'})) {
1299         my @expaths = split(/\0/, $param->{'exclude-path'} || '');
1300         $param->{'exclude-path'} = [ @expaths ];
1301     }
1302
1303     if (defined($param->{mailto})) {
1304         my @mailto = PVE::Tools::split_list(extract_param($param, 'mailto'));
1305         $param->{mailto} = [ @mailto ];
1306     }
1307
1308     return;
1309 }
1310
1311 sub verify_vzdump_parameters {
1312     my ($param, $check_missing) = @_;
1313
1314     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1315         if $param->{all} && $param->{vmid};
1316
1317     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1318         if $param->{exclude} && $param->{vmid};
1319
1320     raise_param_exc({ pool => "option conflicts with option 'vmid'"})
1321         if $param->{pool} && $param->{vmid};
1322
1323     raise_param_exc({ 'prune-backups' => "option conflicts with option 'maxfiles'"})
1324         if defined($param->{'prune-backups'}) && defined($param->{maxfiles});
1325
1326     $parse_prune_backups_maxfiles->($param, 'CLI parameters');
1327
1328     raise_param_exc({'notes-template' => "contains a line feed"})
1329         if $param->{'notes-template'} && $param->{'notes-template'} =~ m/\n/;
1330
1331     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
1332
1333     return if !$check_missing;
1334
1335     raise_param_exc({ vmid => "property is missing"})
1336         if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
1337
1338 }
1339
1340 sub stop_running_backups {
1341     my($self) = @_;
1342
1343     my $upid = PVE::Tools::file_read_firstline($pidfile);
1344     return if !$upid;
1345
1346     my $task = PVE::Tools::upid_decode($upid);
1347
1348     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1349         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1350         kill(15, $task->{pid});
1351         # wait max 15 seconds to shut down (else, do nothing for now)
1352         my $i;
1353         for ($i = 15; $i > 0; $i--) {
1354             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1355             sleep (1);
1356         }
1357         die "stopping backup process $task->{pid} failed\n" if $i == 0;
1358     }
1359 }
1360
1361 sub get_included_guests {
1362     my ($job) = @_;
1363
1364     my $vmids = [];
1365     my $vmids_per_node = {};
1366
1367     my $vmlist = PVE::Cluster::get_vmlist();
1368
1369     if ($job->{pool}) {
1370         $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool});
1371     } elsif ($job->{vmid}) {
1372         $vmids = [ split_list($job->{vmid}) ];
1373     } elsif ($job->{all}) {
1374         # all or exclude
1375         my $exclude = check_vmids(split_list($job->{exclude}));
1376         my $excludehash = { map { $_ => 1 } @$exclude };
1377
1378         for my $id (keys %{$vmlist->{ids}}) {
1379             next if $excludehash->{$id};
1380             push @$vmids, $id;
1381         }
1382     } else {
1383         return $vmids_per_node;
1384     }
1385     $vmids = check_vmids(@$vmids);
1386
1387     for my $vmid (@$vmids) {
1388         if (defined($vmlist->{ids}->{$vmid})) {
1389             my $node = $vmlist->{ids}->{$vmid}->{node};
1390             next if (defined $job->{node} && $job->{node} ne $node);
1391
1392             push @{$vmids_per_node->{$node}}, $vmid;
1393         } else {
1394             push @{$vmids_per_node->{''}}, $vmid;
1395         }
1396     }
1397
1398     return $vmids_per_node;
1399 }
1400
1401 1;