PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Path;
   8 use IO::File;
   9 use IO::Select;
  10 use IPC::Open3;
  11 use POSIX qw(strftime);
  12 use Time::Local;
  13
  14 use PVE::Cluster qw(cfs_read_file);
  15 use PVE::DataCenterConfig;
  16 use PVE::Exception qw(raise_param_exc);
  17 use PVE::HA::Config;
  18 use PVE::HA::Env::PVE2;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::RPCEnvironment;
  21 use PVE::Storage;
  22 use PVE::VZDump::Common;
  23 use PVE::VZDump::Plugin;
  24 use PVE::Tools qw(extract_param split_list);
  25 use PVE::API2Tools;
  26
  27 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  28
  29 my $lockfile = '/var/run/vzdump.lock';
  30 my $pidfile = '/var/run/vzdump.pid';
  31 my $logdir = '/var/log/vzdump';
  32
  33 my @plugins = qw();
  34
  35 my $confdesc = PVE::VZDump::Common::get_confdesc();
  36
  37 # Load available plugins
  38 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
  39 foreach my $plug (@pve_vzdump_classes) {
  40     my $filename = "/usr/share/perl5/$plug.pm";
  41     $filename =~ s!::!/!g;
  42     if (-f $filename) {
  43         eval { require $filename; };
  44         if (!$@) {
  45             $plug->import ();
  46             push @plugins, $plug;
  47         } else {
  48             die $@;
  49         }
  50     }
  51 }
  52
  53 # helper functions
  54
  55 sub debugmsg {
  56     my ($mtype, $msg, $logfd, $syslog) = @_;
  57
  58     PVE::VZDump::Plugin::debugmsg(@_);
  59 }
  60
  61 sub run_command {
  62     my ($logfd, $cmdstr, %param) = @_;
  63
  64     my $logfunc = sub {
  65         my $line = shift;
  66         debugmsg ('info', $line, $logfd);
  67     };
  68
  69     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
  70 }
  71
  72 my $parse_prune_backups_maxfiles = sub {
  73     my ($param, $kind) = @_;
  74
  75     my $maxfiles = delete $param->{maxfiles};
  76     my $prune_backups = $param->{'prune-backups'};
  77
  78     debugmsg('warn', "both 'maxfiles' and 'prune-backups' defined as ${kind} - ignoring 'maxfiles'")
  79         if defined($maxfiles) && defined($prune_backups);
  80
  81     if (defined($prune_backups)) {
  82         return if ref($prune_backups) eq 'HASH'; # already parsed
  83         $param->{'prune-backups'} = PVE::JSONSchema::parse_property_string(
  84             'prune-backups',
  85             $prune_backups
  86         );
  87     } elsif (defined($maxfiles)) {
  88         if ($maxfiles) {
  89             $param->{'prune-backups'} = { 'keep-last' => $maxfiles };
  90         } else {
  91             $param->{'prune-backups'} = { 'keep-all' => 1 };
  92         }
  93     }
  94 };
  95
  96 sub storage_info {
  97     my $storage = shift;
  98
  99     my $cfg = PVE::Storage::config();
 100     my $scfg = PVE::Storage::storage_config($cfg, $storage);
 101     my $type = $scfg->{type};
 102
 103     die "can't use storage '$storage' for backups - wrong content type\n"
 104         if (!$scfg->{content}->{backup});
 105
 106     my $info = {
 107         scfg => $scfg,
 108     };
 109
 110     $info->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'})
 111         if defined($scfg->{'prune-backups'});
 112
 113     if ($type eq 'pbs') {
 114         $info->{pbs} = 1;
 115     } else {
 116         $info->{dumpdir} = PVE::Storage::get_backup_dir($cfg, $storage);
 117     }
 118
 119     return $info;
 120 }
 121
 122 sub format_size {
 123     my $size = shift;
 124
 125     my $kb = $size / 1024;
 126
 127     if ($kb < 1024) {
 128         return int ($kb) . "KB";
 129     }
 130
 131     my $mb = $size / (1024*1024);
 132     if ($mb < 1024) {
 133         return int ($mb) . "MB";
 134     }
 135     my $gb = $mb / 1024;
 136     if ($gb < 1024) {
 137         return sprintf ("%.2fGB", $gb);
 138     }
 139     my $tb = $gb / 1024;
 140     return sprintf ("%.2fTB", $tb);
 141 }
 142
 143 sub format_time {
 144     my $seconds = shift;
 145
 146     my $hours = int ($seconds/3600);
 147     $seconds = $seconds - $hours*3600;
 148     my $min = int ($seconds/60);
 149     $seconds = $seconds - $min*60;
 150
 151     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 152 }
 153
 154 sub encode8bit {
 155     my ($str) = @_;
 156
 157     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 158
 159     return $str;
 160 }
 161
 162 sub escape_html {
 163     my ($str) = @_;
 164
 165     $str =~ s/&/&amp;/g;
 166     $str =~ s/</&lt;/g;
 167     $str =~ s/>/&gt;/g;
 168
 169     return $str;
 170 }
 171
 172 sub check_bin {
 173     my ($bin)  = @_;
 174
 175     foreach my $p (split (/:/, $ENV{PATH})) {
 176         my $fn = "$p/$bin";
 177         if (-x $fn) {
 178             return $fn;
 179         }
 180     }
 181
 182     die "unable to find command '$bin'\n";
 183 }
 184
 185 sub check_vmids {
 186     my (@vmids) = @_;
 187
 188     my $res = [];
 189     for my $vmid (sort {$a <=> $b} @vmids) {
 190         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 191         $vmid = int ($vmid); # remove leading zeros
 192         next if !$vmid;
 193         push @$res, $vmid;
 194     }
 195
 196     return $res;
 197 }
 198
 199
 200 sub read_vzdump_defaults {
 201
 202     my $fn = "/etc/vzdump.conf";
 203
 204     my $defaults = {
 205         map {
 206             my $default = $confdesc->{$_}->{default};
 207              defined($default) ? ($_ => $default) : ()
 208         } keys %$confdesc
 209     };
 210     $parse_prune_backups_maxfiles->($defaults, "defaults in VZDump schema");
 211
 212     my $raw;
 213     eval { $raw = PVE::Tools::file_get_contents($fn); };
 214     return $defaults if $@;
 215
 216     my $conf_schema = { type => 'object', properties => $confdesc, };
 217     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 218     if (my $excludes = $res->{'exclude-path'}) {
 219         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 220     }
 221     if (defined($res->{mailto})) {
 222         my @mailto = split_list($res->{mailto});
 223         $res->{mailto} = [ @mailto ];
 224     }
 225     $parse_prune_backups_maxfiles->($res, "options in '$fn'");
 226
 227     foreach my $key (keys %$defaults) {
 228         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 229     }
 230
 231     if (defined($res->{storage}) && defined($res->{dumpdir})) {
 232         debugmsg('warn', "both 'storage' and 'dumpdir' defined in '$fn' - ignoring 'dumpdir'");
 233         delete $res->{dumpdir};
 234     }
 235
 236     return $res;
 237 }
 238
 239 use constant MAX_MAIL_SIZE => 1024*1024;
 240 sub sendmail {
 241     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 242
 243     my $opts = $self->{opts};
 244
 245     my $mailto = $opts->{mailto};
 246
 247     return if !($mailto && scalar(@$mailto));
 248
 249     my $cmdline = $self->{cmdline};
 250
 251     my $ecount = 0;
 252     foreach my $task (@$tasklist) {
 253         $ecount++ if $task->{state} ne 'ok';
 254         chomp $task->{msg} if $task->{msg};
 255         $task->{backuptime} = 0 if !$task->{backuptime};
 256         $task->{size} = 0 if !$task->{size};
 257         $task->{target} = 'unknown' if !$task->{target};
 258         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 259
 260         if ($task->{state} eq 'todo') {
 261             $task->{msg} = 'aborted';
 262         }
 263     }
 264
 265     my $notify = $opts->{mailnotification} || 'always';
 266     return if (!$ecount && !$err && ($notify eq 'failure'));
 267
 268     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 269     if ($err) {
 270         if ($err =~ /\n/) {
 271             $stat .= ": multiple problems";
 272         } else {
 273             $stat .= ": $err";
 274             $err = undef;
 275         }
 276     }
 277
 278     my $hostname = `hostname -f` || PVE::INotify::nodename();
 279     chomp $hostname;
 280
 281     # text part
 282     my $text = $err ? "$err\n\n" : '';
 283     my $namelength = 20;
 284     $text .= sprintf (
 285         "%-10s %-${namelength}s %-6s %10s %10s  %s\n",
 286         qw(VMID NAME STATUS TIME SIZE FILENAME)
 287     );
 288     foreach my $task (@$tasklist) {
 289         my $name = substr($task->{hostname}, 0, $namelength);
 290         my $successful = $task->{state} eq 'ok';
 291         my $size = $successful ? format_size ($task->{size}) : 0;
 292         my $filename = $successful ? $task->{target} : '-';
 293         my $size_fmt = $successful ? "%10s": "%8.2fMB";
 294         $text .= sprintf(
 295             "%-10s %-${namelength}s %-6s %10s $size_fmt  %s\n",
 296             $task->{vmid},
 297             $name,
 298             $task->{state},
 299             format_time($task->{backuptime}),
 300             $size,
 301             $filename,
 302         );
 303     }
 304
 305     my $text_log_part;
 306     $text_log_part .= "\nDetailed backup logs:\n\n";
 307     $text_log_part .= "$cmdline\n\n";
 308
 309     $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
 310     foreach my $task (@$tasklist) {
 311         my $vmid = $task->{vmid};
 312         my $log = $task->{tmplog};
 313         if (!$log) {
 314             $text_log_part .= "$vmid: no log available\n\n";
 315             next;
 316         }
 317         if (open (my $TMP, '<', "$log")) {
 318             while (my $line = <$TMP>) {
 319                 next if $line =~ /^status: \d+/; # not useful in mails
 320                 $text_log_part .= encode8bit ("$vmid: $line");
 321             }
 322             close ($TMP);
 323         } else {
 324             $text_log_part .= "$vmid: Could not open log file\n\n";
 325         }
 326         $text_log_part .= "\n";
 327     }
 328     $text_log_part .= $detail_post if defined($detail_post);
 329
 330     # html part
 331     my $html = "<html><body>\n";
 332     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 333     $html .= "<table border=1 cellpadding=3>\n";
 334     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 335
 336     my $ssize = 0;
 337     foreach my $task (@$tasklist) {
 338         my $vmid = $task->{vmid};
 339         my $name = $task->{hostname};
 340
 341         if  ($task->{state} eq 'ok') {
 342             $ssize += $task->{size};
 343
 344             $html .= sprintf (
 345                 "<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 346                 $vmid,
 347                 $name,
 348                 format_time($task->{backuptime}),
 349                 format_size ($task->{size}),
 350                 escape_html ($task->{target}),
 351             );
 352         } else {
 353             $html .= sprintf (
 354                 "<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 355                 $vmid,
 356                 $name,
 357                 format_time($task->{backuptime}),
 358                 escape_html ($task->{msg}),
 359             );
 360         }
 361     }
 362
 363     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 364  format_time ($totaltime), format_size ($ssize));
 365
 366     $html .= "\n</table><br><br>\n";
 367     my $html_log_part;
 368     $html_log_part .= "Detailed backup logs:<br /><br />\n";
 369     $html_log_part .= "<pre>\n";
 370     $html_log_part .= escape_html($cmdline) . "\n\n";
 371
 372     $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 373     foreach my $task (@$tasklist) {
 374         my $vmid = $task->{vmid};
 375         my $log = $task->{tmplog};
 376         if (!$log) {
 377             $html_log_part .= "$vmid: no log available\n\n";
 378             next;
 379         }
 380         if (open (my $TMP, '<', "$log")) {
 381             while (my $line = <$TMP>) {
 382                 next if $line =~ /^status: \d+/; # not useful in mails
 383                 if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 384                     $html_log_part .= encode8bit ("$vmid: <font color=red>".
 385                         escape_html ($line) . "</font>");
 386                 } else {
 387                     $html_log_part .= encode8bit ("$vmid: " . escape_html ($line));
 388                 }
 389             }
 390             close ($TMP);
 391         } else {
 392             $html_log_part .= "$vmid: Could not open log file\n\n";
 393         }
 394         $html_log_part .= "\n";
 395     }
 396     $html_log_part .= escape_html($detail_post) if defined($detail_post);
 397     $html_log_part .= "</pre>";
 398     my $html_end = "\n</body></html>\n";
 399     # end html part
 400
 401     if (length($text) + length($text_log_part) +
 402         length($html) + length($html_log_part) +
 403         length($html_end) < MAX_MAIL_SIZE)
 404     {
 405         $html .= $html_log_part;
 406         $html .= $html_end;
 407         $text .= $text_log_part;
 408     } else {
 409         my $msg = "Log output was too long to be sent by mail. ".
 410             "See Task History for details!\n";
 411         $text .= $msg;
 412         $html .= "<p>$msg</p>";
 413         $html .= $html_end;
 414     }
 415
 416     my $subject = "vzdump backup status ($hostname) : $stat";
 417
 418     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 419     my $mailfrom = $dcconf->{email_from} || "root";
 420
 421     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 422 };
 423
 424 sub new {
 425     my ($class, $cmdline, $opts, $skiplist) = @_;
 426
 427     mkpath $logdir;
 428
 429     check_bin ('cp');
 430     check_bin ('df');
 431     check_bin ('sendmail');
 432     check_bin ('rsync');
 433     check_bin ('tar');
 434     check_bin ('mount');
 435     check_bin ('umount');
 436     check_bin ('cstream');
 437     check_bin ('ionice');
 438
 439     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 440         check_bin ('lvcreate');
 441         check_bin ('lvs');
 442         check_bin ('lvremove');
 443     }
 444
 445     my $defaults = read_vzdump_defaults();
 446
 447     foreach my $k (keys %$defaults) {
 448         next if $k eq 'exclude-path' || $k eq 'prune-backups'; # dealt with separately
 449         if ($k eq 'dumpdir' || $k eq 'storage') {
 450             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 451                 !defined ($opts->{storage});
 452         } else {
 453             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 454         }
 455     }
 456
 457     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 458     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 459
 460     $skiplist = [] if !$skiplist;
 461     my $self = bless {
 462         cmdline => $cmdline,
 463         opts => $opts,
 464         skiplist => $skiplist,
 465     }, $class;
 466
 467     my $findexcl = $self->{findexcl} = [];
 468     if ($defaults->{'exclude-path'}) {
 469         push @$findexcl, @{$defaults->{'exclude-path'}};
 470     }
 471
 472     if ($opts->{'exclude-path'}) {
 473         push @$findexcl, @{$opts->{'exclude-path'}};
 474     }
 475
 476     if ($opts->{stdexcludes}) {
 477         push @$findexcl,
 478             '/tmp/?*',
 479             '/var/tmp/?*',
 480             '/var/run/?*.pid',
 481             ;
 482     }
 483
 484     foreach my $p (@plugins) {
 485         my $pd = $p->new($self);
 486
 487         push @{$self->{plugins}}, $pd;
 488     }
 489
 490     if (defined($opts->{storage}) && $opts->{stdout}) {
 491         die "cannot use options 'storage' and 'stdout' at the same time\n";
 492     } elsif (defined($opts->{storage}) && defined($opts->{dumpdir})) {
 493         die "cannot use options 'storage' and 'dumpdir' at the same time\n";
 494     }
 495
 496     if (!$opts->{dumpdir} && !$opts->{storage}) {
 497         $opts->{storage} = 'local';
 498     }
 499
 500     $self->{job_init_log} = '';
 501     open my $job_init_fd, '>', \$self->{job_init_log};
 502     $self->run_hook_script('job-init', undef, $job_init_fd);
 503     close $job_init_fd;
 504
 505     PVE::Cluster::cfs_update(); # Pick up possible changes made by the hook script.
 506
 507     my $errors = '';
 508
 509     if ($opts->{storage}) {
 510         my $storage_cfg = PVE::Storage::config();
 511         eval { PVE::Storage::activate_storage($storage_cfg, $opts->{storage}) };
 512         if (my $err = $@) {
 513             chomp($err);
 514             $errors .= "could not activate storage '$opts->{storage}': $err";
 515         }
 516
 517         my $info = eval { storage_info ($opts->{storage}) };
 518         if (my $err = $@) {
 519             chomp($err);
 520             $errors .= "could not get storage information for '$opts->{storage}': $err";
 521         } else {
 522             $opts->{dumpdir} = $info->{dumpdir};
 523             $opts->{scfg} = $info->{scfg};
 524             $opts->{pbs} = $info->{pbs};
 525             $opts->{'prune-backups'} //= $info->{'prune-backups'};
 526         }
 527     } elsif ($opts->{dumpdir}) {
 528         $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
 529             if ! -d $opts->{dumpdir};
 530     } else {
 531         die "internal error";
 532     }
 533
 534     $opts->{'prune-backups'} //= $defaults->{'prune-backups'};
 535
 536     # avoid triggering any remove code path if keep-all is set
 537     $opts->{remove} = 0 if $opts->{'prune-backups'}->{'keep-all'};
 538
 539     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 540         $errors .= "\n" if $errors;
 541         $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
 542     }
 543
 544     if ($errors) {
 545         eval { $self->sendmail([], 0, $errors); };
 546         debugmsg ('err', $@) if $@;
 547         die "$errors\n";
 548     }
 549
 550     return $self;
 551 }
 552
 553 sub get_mount_info {
 554     my ($dir) = @_;
 555
 556     # Note: df 'available' can be negative, and percentage set to '-'
 557
 558     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 559
 560     my $res;
 561
 562     my $parser = sub {
 563         my $line = shift;
 564         if (my ($fsid, $fstype, undef, $mp) = $line =~
 565             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 566             $res = {
 567                 device => $fsid,
 568                 fstype => $fstype,
 569                 mountpoint => $mp,
 570             };
 571         }
 572     };
 573
 574     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 575     warn $@ if $@;
 576
 577     return $res;
 578 }
 579
 580 sub getlock {
 581     my ($self, $upid) = @_;
 582
 583     my $fh;
 584
 585     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 586
 587     die "missing UPID" if !$upid; # should not happen
 588
 589     my $SERVER_FLCK;
 590     if (!open ($SERVER_FLCK, '>>', "$lockfile")) {
 591         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 592         die "can't open lock on file '$lockfile' - $!";
 593     }
 594
 595     if (!flock ($SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 596         if (!$maxwait) {
 597             debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
 598             die "can't acquire lock '$lockfile' (wait = 0)";
 599         }
 600
 601         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 602         eval {
 603             alarm ($maxwait * 60);
 604
 605             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 606
 607             if (!flock ($SERVER_FLCK, LOCK_EX)) {
 608                 my $err = $!;
 609                 close ($SERVER_FLCK);
 610                 alarm (0);
 611                 die "$err\n";
 612             }
 613             alarm (0);
 614         };
 615         alarm (0);
 616
 617         my $err = $@;
 618
 619         if ($err) {
 620             debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
 621             die "can't acquire lock '$lockfile' - $err";
 622         }
 623
 624         debugmsg('info', "got global lock", undef, 1);
 625     }
 626
 627     PVE::Tools::file_set_contents($pidfile, $upid);
 628
 629     return $SERVER_FLCK;
 630 }
 631
 632 sub run_hook_script {
 633     my ($self, $phase, $task, $logfd) = @_;
 634
 635     my $opts = $self->{opts};
 636
 637     my $script = $opts->{script};
 638     return if !$script;
 639
 640     if (!-x $script) {
 641         die "The hook script '$script' is not executable.\n";
 642     }
 643
 644     my $cmd = [$script, $phase];
 645
 646     if ($task) {
 647         push @$cmd, $task->{mode};
 648         push @$cmd, $task->{vmid};
 649     }
 650
 651     local %ENV;
 652     # set immutable opts directly (so they are available in all phases)
 653     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 654     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 655
 656     foreach my $ek (qw(vmtype hostname target logfile)) {
 657         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 658     }
 659
 660     run_command ($logfd, $cmd);
 661 }
 662
 663 sub compressor_info {
 664     my ($opts) = @_;
 665     my $opt_compress = $opts->{compress};
 666
 667     if (!$opt_compress || $opt_compress eq '0') {
 668         return undef;
 669     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 670         return ('lzop', 'lzo');
 671     } elsif ($opt_compress eq 'gzip') {
 672         if ($opts->{pigz} > 0) {
 673             my $pigz_threads = $opts->{pigz};
 674             if ($pigz_threads == 1) {
 675                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 676                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 677             }
 678             return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
 679         } else {
 680             return ('gzip --rsyncable', 'gz');
 681         }
 682     } elsif ($opt_compress eq 'zstd') {
 683         my $zstd_threads = $opts->{zstd} // 1;
 684         if ($zstd_threads == 0) {
 685             my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 686             $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
 687         }
 688         return ("zstd --rsyncable --threads=${zstd_threads}", 'zst');
 689     } else {
 690         die "internal error - unknown compression option '$opt_compress'";
 691     }
 692 }
 693
 694 sub get_backup_file_list {
 695     my ($dir, $bkname, $exclude_fn) = @_;
 696
 697     my $bklist = [];
 698     foreach my $fn (<$dir/${bkname}-*>) {
 699         next if $exclude_fn && $fn eq $exclude_fn;
 700
 701         my $archive_info = eval { PVE::Storage::archive_info($fn) } // {};
 702         if ($archive_info->{is_std_name}) {
 703             my $filename = $archive_info->{filename};
 704             my $backup = {
 705                 'path' => "$dir/$filename",
 706                 'ctime' => $archive_info->{ctime},
 707             };
 708             push @{$bklist}, $backup;
 709         }
 710     }
 711
 712     return $bklist;
 713 }
 714
 715 sub exec_backup_task {
 716     my ($self, $task) = @_;
 717
 718     my $opts = $self->{opts};
 719
 720     my $cfg = PVE::Storage::config();
 721     my $vmid = $task->{vmid};
 722     my $plugin = $task->{plugin};
 723
 724     $task->{backup_time} = time();
 725
 726     my $pbs_group_name;
 727     my $pbs_snapshot_name;
 728
 729     my $vmstarttime = time ();
 730
 731     my $logfd;
 732
 733     my $cleanup = {};
 734
 735     my $log_vm_online_again = sub {
 736         return if !defined($task->{vmstoptime});
 737         $task->{vmconttime} //= time();
 738         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 739         $delay = '<1' if $delay < 1;
 740         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 741     };
 742
 743     eval {
 744         die "unable to find VM '$vmid'\n" if !$plugin;
 745
 746         my $vmtype = $plugin->type();
 747
 748         if ($self->{opts}->{pbs}) {
 749             if ($vmtype eq 'lxc') {
 750                 $pbs_group_name = "ct/$vmid";
 751             } elsif  ($vmtype eq 'qemu') {
 752                 $pbs_group_name = "vm/$vmid";
 753             } else {
 754                 die "pbs backup not implemented for plugin type '$vmtype'\n";
 755             }
 756             my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
 757             $pbs_snapshot_name = "$pbs_group_name/$btime";
 758         }
 759
 760         # for now we deny backups of a running ha managed service in *stop* mode
 761         # as it interferes with the HA stack (started services should not stop).
 762         if ($opts->{mode} eq 'stop' &&
 763             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 764         {
 765             die "Cannot execute a backup with stop mode on a HA managed and".
 766                 " enabled Service. Use snapshot mode or disable the Service.\n";
 767         }
 768
 769         my $tmplog = "$logdir/$vmtype-$vmid.log";
 770
 771         my $bkname = "vzdump-$vmtype-$vmid";
 772         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
 773
 774         my $prune_options = $opts->{'prune-backups'};
 775
 776         my $backup_limit = 0;
 777         if (!$prune_options->{'keep-all'}) {
 778             foreach my $keep (values %{$prune_options}) {
 779                 $backup_limit += $keep;
 780             }
 781         }
 782
 783         if ($backup_limit && !$opts->{remove}) {
 784             my $count;
 785             if ($self->{opts}->{pbs}) {
 786                 my $res = PVE::Storage::PBSPlugin::run_client_cmd($opts->{scfg}, $opts->{storage}, 'snapshots', $pbs_group_name);
 787                 $count = scalar(@$res);
 788             } else {
 789                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 790                 $count = scalar(@$bklist);
 791             }
 792             die "There is a max backup limit of $backup_limit enforced by the".
 793                 " target storage or the vzdump parameters.".
 794                 " Either increase the limit or delete old backup(s).\n"
 795                 if $count >= $backup_limit;
 796         }
 797
 798         if (!$self->{opts}->{pbs}) {
 799             $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 800         }
 801
 802         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 803         my ($comp, $comp_ext) = compressor_info($opts);
 804         if ($comp && $comp_ext) {
 805             $ext .= ".${comp_ext}";
 806         }
 807
 808         if ($self->{opts}->{pbs}) {
 809             die "unable to pipe backup to stdout\n" if $opts->{stdout};
 810             $task->{target} = $pbs_snapshot_name;
 811         } else {
 812             if ($opts->{stdout}) {
 813                 $task->{target} = '-';
 814             } else {
 815                 $task->{target} = $task->{tmptar} = "$opts->{dumpdir}/$basename$ext";
 816                 $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 817                 unlink $task->{tmptar};
 818             }
 819         }
 820
 821         $task->{vmtype} = $vmtype;
 822
 823         my $pid = $$;
 824         if ($opts->{tmpdir}) {
 825             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp${pid}_$vmid/";
 826         } elsif ($self->{opts}->{pbs}) {
 827             $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 828         } else {
 829             # dumpdir is posix? then use it as temporary dir
 830             my $info = get_mount_info($opts->{dumpdir});
 831             if ($vmtype eq 'qemu' ||
 832                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 833                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 834             } else {
 835                 $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 836                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 837                           "using $task->{tmpdir} for temporary files", $logfd);
 838             }
 839         }
 840
 841         rmtree $task->{tmpdir};
 842         mkdir $task->{tmpdir};
 843         -d $task->{tmpdir} ||
 844             die "unable to create temporary directory '$task->{tmpdir}'";
 845
 846         $logfd = IO::File->new (">$tmplog") ||
 847             die "unable to create log file '$tmplog'";
 848
 849         $task->{dumpdir} = $opts->{dumpdir};
 850         $task->{storeid} = $opts->{storage};
 851         $task->{scfg} = $opts->{scfg};
 852         $task->{tmplog} = $tmplog;
 853
 854         unlink $task->{logfile} if defined($task->{logfile});
 855
 856         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 857         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
 858
 859         $plugin->set_logfd ($logfd);
 860
 861         # test is VM is running
 862         my ($running, $status_text) = $plugin->vm_status ($vmid);
 863
 864         debugmsg ('info', "status = ${status_text}", $logfd);
 865
 866         # lock VM (prevent config changes)
 867         $plugin->lock_vm ($vmid);
 868
 869         $cleanup->{unlock} = 1;
 870
 871         # prepare
 872
 873         my $mode = $running ? $task->{mode} : 'stop';
 874
 875         if ($mode eq 'snapshot') {
 876             my %saved_task = %$task;
 877             eval { $plugin->prepare ($task, $vmid, $mode); };
 878             if (my $err = $@) {
 879                 die $err if $err !~ m/^mode failure/;
 880                 debugmsg ('info',  $err, $logfd);
 881                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 882                 $mode = 'suspend'; # so prepare is called again below
 883                 %$task = %saved_task;
 884             }
 885         }
 886
 887         $cleanup->{prepared} = 1;
 888
 889         $task->{mode} = $mode;
 890
 891         debugmsg ('info', "backup mode: $mode", $logfd);
 892         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)  if $opts->{bwlimit};
 893         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 894
 895         if ($mode eq 'stop') {
 896             $plugin->prepare ($task, $vmid, $mode);
 897
 898             $self->run_hook_script ('backup-start', $task, $logfd);
 899
 900             if ($running) {
 901                 debugmsg ('info', "stopping virtual guest", $logfd);
 902                 $task->{vmstoptime} = time();
 903                 $self->run_hook_script ('pre-stop', $task, $logfd);
 904                 $plugin->stop_vm ($task, $vmid);
 905                 $cleanup->{restart} = 1;
 906             }
 907
 908
 909         } elsif ($mode eq 'suspend') {
 910             $plugin->prepare ($task, $vmid, $mode);
 911
 912             $self->run_hook_script ('backup-start', $task, $logfd);
 913
 914             if ($vmtype eq 'lxc') {
 915                 # pre-suspend rsync
 916                 $plugin->copy_data_phase1($task, $vmid);
 917             }
 918
 919             debugmsg ('info', "suspending guest", $logfd);
 920             $task->{vmstoptime} = time ();
 921             $self->run_hook_script ('pre-stop', $task, $logfd);
 922             $plugin->suspend_vm ($task, $vmid);
 923             $cleanup->{resume} = 1;
 924
 925             if ($vmtype eq 'lxc') {
 926                 # post-suspend rsync
 927                 $plugin->copy_data_phase2($task, $vmid);
 928
 929                 debugmsg ('info', "resuming guest", $logfd);
 930                 $cleanup->{resume} = 0;
 931                 $self->run_hook_script('pre-restart', $task, $logfd);
 932                 $plugin->resume_vm($task, $vmid);
 933                 $self->run_hook_script('post-restart', $task, $logfd);
 934                 $log_vm_online_again->();
 935             }
 936
 937         } elsif ($mode eq 'snapshot') {
 938             $self->run_hook_script ('backup-start', $task, $logfd);
 939
 940             my $snapshot_count = $task->{snapshot_count} || 0;
 941
 942             $self->run_hook_script ('pre-stop', $task, $logfd);
 943
 944             if ($snapshot_count > 1) {
 945                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 946                 $task->{vmstoptime} = time ();
 947                 $plugin->suspend_vm ($task, $vmid);
 948                 $cleanup->{resume} = 1;
 949             }
 950
 951             $plugin->snapshot ($task, $vmid);
 952
 953             $self->run_hook_script ('pre-restart', $task, $logfd);
 954
 955             if ($snapshot_count > 1) {
 956                 debugmsg ('info', "resume vm", $logfd);
 957                 $cleanup->{resume} = 0;
 958                 $plugin->resume_vm ($task, $vmid);
 959                 $log_vm_online_again->();
 960             }
 961
 962             $self->run_hook_script ('post-restart', $task, $logfd);
 963
 964         } else {
 965             die "internal error - unknown mode '$mode'\n";
 966         }
 967
 968         # assemble archive image
 969         $plugin->assemble ($task, $vmid);
 970
 971         # produce archive
 972
 973         if ($opts->{stdout}) {
 974             debugmsg ('info', "sending archive to stdout", $logfd);
 975             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 976             $self->run_hook_script ('backup-end', $task, $logfd);
 977             return;
 978         }
 979
 980         my $archive_txt = $self->{opts}->{pbs} ? 'Proxmox Backup Server' : 'vzdump';
 981         debugmsg('info', "creating $archive_txt archive '$task->{target}'", $logfd);
 982         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 983
 984         if ($self->{opts}->{pbs}) {
 985             # size is added to task struct in guest vzdump plugins
 986         } else {
 987             rename ($task->{tmptar}, $task->{target}) ||
 988                 die "unable to rename '$task->{tmptar}' to '$task->{target}'\n";
 989
 990             # determine size
 991             $task->{size} = (-s $task->{target}) || 0;
 992             my $cs = format_size ($task->{size});
 993             debugmsg ('info', "archive file size: $cs", $logfd);
 994         }
 995
 996         if ($opts->{remove}) {
 997             my $keepstr = join(', ', map { "$_=$prune_options->{$_}" } sort keys %$prune_options);
 998             debugmsg ('info', "prune older backups with retention: $keepstr", $logfd);
 999             my $pruned = 0;
1000             if (!defined($opts->{storage})) {
1001                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
1002
1003                 for my $prune_entry ($bklist->@*) {
1004                     if (-e PVE::Storage::protection_file_path($prune_entry->{path})) {
1005                         $prune_entry->{mark} = 'protected';
1006                     }
1007                 }
1008
1009                 PVE::Storage::prune_mark_backup_group($bklist, $prune_options);
1010
1011                 foreach my $prune_entry (@{$bklist}) {
1012                     next if $prune_entry->{mark} ne 'remove';
1013                     $pruned++;
1014                     my $archive_path = $prune_entry->{path};
1015                     debugmsg ('info', "delete old backup '$archive_path'", $logfd);
1016                     PVE::Storage::archive_remove($archive_path);
1017                 }
1018             } else {
1019                 my $pruned_list = PVE::Storage::prune_backups(
1020                     $cfg,
1021                     $opts->{storage},
1022                     $prune_options,
1023                     $vmid,
1024                     $vmtype,
1025                     0,
1026                     sub { debugmsg($_[0], $_[1], $logfd) },
1027                 );
1028                 $pruned = scalar(grep { $_->{mark} eq 'remove' } $pruned_list->@*);
1029             }
1030             my $log_pruned_extra = $pruned > 0 ? " not covered by keep-retention policy" : "";
1031             debugmsg ('info', "pruned $pruned backup(s)${log_pruned_extra}", $logfd);
1032         }
1033
1034         $self->run_hook_script ('backup-end', $task, $logfd);
1035     };
1036     my $err = $@;
1037
1038     if ($plugin) {
1039         # clean-up
1040
1041         if ($cleanup->{unlock}) {
1042             eval { $plugin->unlock_vm ($vmid); };
1043             warn $@ if $@;
1044         }
1045
1046         if ($cleanup->{prepared}) {
1047             # only call cleanup when necessary (when prepare was executed)
1048             eval { $plugin->cleanup ($task, $vmid) };
1049             warn $@ if $@;
1050         }
1051
1052         eval { $plugin->set_logfd (undef); };
1053         warn $@ if $@;
1054
1055         if ($cleanup->{resume} || $cleanup->{restart}) {
1056             eval {
1057                 $self->run_hook_script ('pre-restart', $task, $logfd);
1058                 if ($cleanup->{resume}) {
1059                     debugmsg ('info', "resume vm", $logfd);
1060                     $plugin->resume_vm ($task, $vmid);
1061                 } else {
1062                     my $running = $plugin->vm_status($vmid);
1063                     if (!$running) {
1064                         debugmsg ('info', "restarting vm", $logfd);
1065                         $plugin->start_vm ($task, $vmid);
1066                     }
1067                 }
1068                 $self->run_hook_script ('post-restart', $task, $logfd);
1069             };
1070             my $err = $@;
1071             if ($err) {
1072                 warn $err;
1073             } else {
1074                 $log_vm_online_again->();
1075             }
1076         }
1077     }
1078
1079     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
1080     warn $@ if $@;
1081
1082     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
1083     warn $@ if $@;
1084
1085     my $delay = $task->{backuptime} = time () - $vmstarttime;
1086
1087     if ($err) {
1088         $task->{state} = 'err';
1089         $task->{msg} = $err;
1090         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1091         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
1092
1093         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1094
1095     } else {
1096         $task->{state} = 'ok';
1097         my $tstr = format_time ($delay);
1098         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1099         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1100     }
1101
1102     close ($logfd) if $logfd;
1103
1104     if ($task->{tmplog}) {
1105         if ($self->{opts}->{pbs}) {
1106             if ($task->{state} eq 'ok') {
1107                 eval {
1108                     PVE::Storage::PBSPlugin::run_raw_client_cmd(
1109                         $opts->{scfg},
1110                         $opts->{storage},
1111                         'upload-log',
1112                         [ $pbs_snapshot_name, $task->{tmplog} ],
1113                         errmsg => "uploading backup task log failed",
1114                         outfunc => sub {},
1115                     );
1116                 };
1117                 debugmsg('warn', "$@") if $@; # $@ contains already error prefix
1118             }
1119         } elsif ($task->{logfile}) {
1120             system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1121         }
1122     }
1123
1124     eval { $self->run_hook_script ('log-end', $task); };
1125
1126     die $err if $err && $err =~ m/^interrupted by signal$/;
1127 }
1128
1129 sub exec_backup {
1130     my ($self, $rpcenv, $authuser) = @_;
1131
1132     my $opts = $self->{opts};
1133
1134     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1135
1136     if (scalar(@{$self->{skiplist}})) {
1137         my $skip_string = join(', ', sort { $a <=> $b } @{$self->{skiplist}});
1138         debugmsg ('info', "skip external VMs: $skip_string");
1139     }
1140
1141     my $tasklist = [];
1142     my $vzdump_plugins =  {};
1143     foreach my $plugin (@{$self->{plugins}}) {
1144         my $type = $plugin->type();
1145         next if exists $vzdump_plugins->{$type};
1146         $vzdump_plugins->{$type} = $plugin;
1147     }
1148
1149     my $vmlist = PVE::Cluster::get_vmlist();
1150     my $vmids = [ sort { $a <=> $b } @{$opts->{vmids}} ];
1151     foreach my $vmid (@{$vmids}) {
1152         my $plugin;
1153         if (defined($vmlist->{ids}->{$vmid})) {
1154             my $guest_type = $vmlist->{ids}->{$vmid}->{type};
1155             $plugin = $vzdump_plugins->{$guest_type};
1156             next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all});
1157         }
1158         push @$tasklist, {
1159             mode => $opts->{mode},
1160             plugin => $plugin,
1161             state => 'todo',
1162             vmid => $vmid,
1163         };
1164     }
1165
1166     # Use in-memory files for the outer hook logs to pass them to sendmail.
1167     my $job_start_log = '';
1168     my $job_end_log = '';
1169     open my $job_start_fd, '>', \$job_start_log;
1170     open my $job_end_fd, '>', \$job_end_log;
1171
1172     my $starttime = time();
1173     my $errcount = 0;
1174     eval {
1175
1176         $self->run_hook_script ('job-start', undef, $job_start_fd);
1177
1178         foreach my $task (@$tasklist) {
1179             $self->exec_backup_task ($task);
1180             $errcount += 1 if $task->{state} ne 'ok';
1181         }
1182
1183         $self->run_hook_script ('job-end', undef, $job_end_fd);
1184     };
1185     my $err = $@;
1186
1187     $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1188
1189     if ($err) {
1190         debugmsg ('err', "Backup job failed - $err", undef, 1);
1191     } else {
1192         if ($errcount) {
1193             debugmsg ('info', "Backup job finished with errors", undef, 1);
1194         } else {
1195             debugmsg ('info', "Backup job finished successfully", undef, 1);
1196         }
1197     }
1198
1199     close $job_start_fd;
1200     close $job_end_fd;
1201
1202     my $totaltime = time() - $starttime;
1203
1204     eval {
1205         $self->sendmail(
1206             $tasklist,
1207             $totaltime,
1208             undef,
1209             $self->{job_init_log} . $job_start_log,
1210             $job_end_log,
1211         );
1212     };
1213     debugmsg ('err', $@) if $@;
1214
1215     die $err if $err;
1216
1217     die "job errors\n" if $errcount;
1218
1219     unlink $pidfile;
1220 }
1221
1222
1223 sub option_exists {
1224     my $key = shift;
1225     return defined($confdesc->{$key});
1226 }
1227
1228 # NOTE it might make sense to merge this and verify_vzdump_parameters(), but one
1229 # needs to adapt command_line() in guest-common's PVE/VZDump/Common.pm and detect
1230 # a second parsing attempt, because verify_vzdump_parameters() is called twice
1231 # during the update_job API call.
1232 sub parse_mailto_exclude_path {
1233     my ($param) = @_;
1234
1235     # exclude-path list need to be 0 separated
1236     if (defined($param->{'exclude-path'})) {
1237         my @expaths = split(/\0/, $param->{'exclude-path'} || '');
1238         $param->{'exclude-path'} = [ @expaths ];
1239     }
1240
1241     if (defined($param->{mailto})) {
1242         my @mailto = PVE::Tools::split_list(extract_param($param, 'mailto'));
1243         $param->{mailto} = [ @mailto ];
1244     }
1245
1246     return;
1247 }
1248
1249 sub verify_vzdump_parameters {
1250     my ($param, $check_missing) = @_;
1251
1252     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1253         if $param->{all} && $param->{vmid};
1254
1255     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1256         if $param->{exclude} && $param->{vmid};
1257
1258     raise_param_exc({ pool => "option conflicts with option 'vmid'"})
1259         if $param->{pool} && $param->{vmid};
1260
1261     raise_param_exc({ 'prune-backups' => "option conflicts with option 'maxfiles'"})
1262         if defined($param->{'prune-backups'}) && defined($param->{maxfiles});
1263
1264     $parse_prune_backups_maxfiles->($param, 'CLI parameters');
1265
1266     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
1267
1268     return if !$check_missing;
1269
1270     raise_param_exc({ vmid => "property is missing"})
1271         if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
1272
1273 }
1274
1275 sub stop_running_backups {
1276     my($self) = @_;
1277
1278     my $upid = PVE::Tools::file_read_firstline($pidfile);
1279     return if !$upid;
1280
1281     my $task = PVE::Tools::upid_decode($upid);
1282
1283     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1284         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1285         kill(15, $task->{pid});
1286         # wait max 15 seconds to shut down (else, do nothing for now)
1287         my $i;
1288         for ($i = 15; $i > 0; $i--) {
1289             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1290             sleep (1);
1291         }
1292         die "stopping backup process $task->{pid} failed\n" if $i == 0;
1293     }
1294 }
1295
1296 sub get_included_guests {
1297     my ($job) = @_;
1298
1299     my $vmids = [];
1300     my $vmids_per_node = {};
1301
1302     my $vmlist = PVE::Cluster::get_vmlist();
1303
1304     if ($job->{pool}) {
1305         $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool});
1306     } elsif ($job->{vmid}) {
1307         $vmids = [ split_list($job->{vmid}) ];
1308     } elsif ($job->{all}) {
1309         # all or exclude
1310         my $exclude = check_vmids(split_list($job->{exclude}));
1311         my $excludehash = { map { $_ => 1 } @$exclude };
1312
1313         for my $id (keys %{$vmlist->{ids}}) {
1314             next if $excludehash->{$id};
1315             push @$vmids, $id;
1316         }
1317     } else {
1318         return $vmids_per_node;
1319     }
1320     $vmids = check_vmids(@$vmids);
1321
1322     for my $vmid (@$vmids) {
1323         if (defined($vmlist->{ids}->{$vmid})) {
1324             my $node = $vmlist->{ids}->{$vmid}->{node};
1325             next if (defined $job->{node} && $job->{node} ne $node);
1326
1327             push @{$vmids_per_node->{$node}}, $vmid;
1328         } else {
1329             push @{$vmids_per_node->{''}}, $vmid;
1330         }
1331     }
1332
1333     return $vmids_per_node;
1334 }
1335
1336 1;