PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Path;
   8 use IO::File;
   9 use IO::Select;
  10 use IPC::Open3;
  11 use POSIX qw(strftime);
  12 use Time::Local;
  13
  14 use PVE::Cluster qw(cfs_read_file);
  15 use PVE::DataCenterConfig;
  16 use PVE::Exception qw(raise_param_exc);
  17 use PVE::HA::Config;
  18 use PVE::HA::Env::PVE2;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::RPCEnvironment;
  21 use PVE::Storage;
  22 use PVE::VZDump::Common;
  23 use PVE::VZDump::Plugin;
  24 use PVE::Tools qw(extract_param split_list);
  25 use PVE::API2Tools;
  26
  27 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  28
  29 my $lockfile = '/var/run/vzdump.lock';
  30 my $pidfile = '/var/run/vzdump.pid';
  31 my $logdir = '/var/log/vzdump';
  32
  33 my @plugins = qw();
  34
  35 my $confdesc = PVE::VZDump::Common::get_confdesc();
  36
  37 # Load available plugins
  38 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
  39 foreach my $plug (@pve_vzdump_classes) {
  40     my $filename = "/usr/share/perl5/$plug.pm";
  41     $filename =~ s!::!/!g;
  42     if (-f $filename) {
  43         eval { require $filename; };
  44         if (!$@) {
  45             $plug->import ();
  46             push @plugins, $plug;
  47         } else {
  48             die $@;
  49         }
  50     }
  51 }
  52
  53 # helper functions
  54
  55 sub debugmsg {
  56     my ($mtype, $msg, $logfd, $syslog) = @_;
  57
  58     PVE::VZDump::Plugin::debugmsg(@_);
  59 }
  60
  61 sub run_command {
  62     my ($logfd, $cmdstr, %param) = @_;
  63
  64     my $logfunc = sub {
  65         my $line = shift;
  66         debugmsg ('info', $line, $logfd);
  67     };
  68
  69     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
  70 }
  71
  72 my $parse_prune_backups_maxfiles = sub {
  73     my ($param, $kind) = @_;
  74
  75     my $maxfiles = delete $param->{maxfiles};
  76     my $prune_backups = $param->{'prune-backups'};
  77
  78     debugmsg('warn', "both 'maxfiles' and 'prune-backups' defined as ${kind} - ignoring 'maxfiles'")
  79         if defined($maxfiles) && defined($prune_backups);
  80
  81     if (defined($prune_backups)) {
  82         return if ref($prune_backups) eq 'HASH'; # already parsed
  83         $param->{'prune-backups'} = PVE::JSONSchema::parse_property_string(
  84             'prune-backups',
  85             $prune_backups
  86         );
  87     } elsif (defined($maxfiles)) {
  88         if ($maxfiles) {
  89             $param->{'prune-backups'} = { 'keep-last' => $maxfiles };
  90         } else {
  91             $param->{'prune-backups'} = { 'keep-all' => 1 };
  92         }
  93     }
  94 };
  95
  96 sub storage_info {
  97     my $storage = shift;
  98
  99     my $cfg = PVE::Storage::config();
 100     my $scfg = PVE::Storage::storage_config($cfg, $storage);
 101     my $type = $scfg->{type};
 102
 103     die "can't use storage '$storage' for backups - wrong content type\n"
 104         if (!$scfg->{content}->{backup});
 105
 106     my $info = {
 107         scfg => $scfg,
 108     };
 109
 110     $info->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'})
 111         if defined($scfg->{'prune-backups'});
 112
 113     if ($type eq 'pbs') {
 114         $info->{pbs} = 1;
 115     } else {
 116         $info->{dumpdir} = PVE::Storage::get_backup_dir($cfg, $storage);
 117     }
 118
 119     return $info;
 120 }
 121
 122 sub format_size {
 123     my $size = shift;
 124
 125     my $kb = $size / 1024;
 126
 127     if ($kb < 1024) {
 128         return int ($kb) . "KB";
 129     }
 130
 131     my $mb = $size / (1024*1024);
 132     if ($mb < 1024) {
 133         return int ($mb) . "MB";
 134     }
 135     my $gb = $mb / 1024;
 136     if ($gb < 1024) {
 137         return sprintf ("%.2fGB", $gb);
 138     }
 139     my $tb = $gb / 1024;
 140     return sprintf ("%.2fTB", $tb);
 141 }
 142
 143 sub format_time {
 144     my $seconds = shift;
 145
 146     my $hours = int ($seconds/3600);
 147     $seconds = $seconds - $hours*3600;
 148     my $min = int ($seconds/60);
 149     $seconds = $seconds - $min*60;
 150
 151     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 152 }
 153
 154 sub encode8bit {
 155     my ($str) = @_;
 156
 157     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 158
 159     return $str;
 160 }
 161
 162 sub escape_html {
 163     my ($str) = @_;
 164
 165     $str =~ s/&/&amp;/g;
 166     $str =~ s/</&lt;/g;
 167     $str =~ s/>/&gt;/g;
 168
 169     return $str;
 170 }
 171
 172 sub check_bin {
 173     my ($bin)  = @_;
 174
 175     foreach my $p (split (/:/, $ENV{PATH})) {
 176         my $fn = "$p/$bin";
 177         if (-x $fn) {
 178             return $fn;
 179         }
 180     }
 181
 182     die "unable to find command '$bin'\n";
 183 }
 184
 185 sub check_vmids {
 186     my (@vmids) = @_;
 187
 188     my $res = [];
 189     for my $vmid (sort {$a <=> $b} @vmids) {
 190         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 191         $vmid = int ($vmid); # remove leading zeros
 192         next if !$vmid;
 193         push @$res, $vmid;
 194     }
 195
 196     return $res;
 197 }
 198
 199
 200 sub read_vzdump_defaults {
 201
 202     my $fn = "/etc/vzdump.conf";
 203
 204     my $defaults = {
 205         map {
 206             my $default = $confdesc->{$_}->{default};
 207              defined($default) ? ($_ => $default) : ()
 208         } keys %$confdesc
 209     };
 210     $parse_prune_backups_maxfiles->($defaults, "defaults in VZDump schema");
 211
 212     my $raw;
 213     eval { $raw = PVE::Tools::file_get_contents($fn); };
 214     return $defaults if $@;
 215
 216     my $conf_schema = { type => 'object', properties => $confdesc, };
 217     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 218     if (my $excludes = $res->{'exclude-path'}) {
 219         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 220     }
 221     if (defined($res->{mailto})) {
 222         my @mailto = split_list($res->{mailto});
 223         $res->{mailto} = [ @mailto ];
 224     }
 225     $parse_prune_backups_maxfiles->($res, "options in '$fn'");
 226
 227     foreach my $key (keys %$defaults) {
 228         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 229     }
 230
 231     if (defined($res->{storage}) && defined($res->{dumpdir})) {
 232         debugmsg('warn', "both 'storage' and 'dumpdir' defined in '$fn' - ignoring 'dumpdir'");
 233         delete $res->{dumpdir};
 234     }
 235
 236     return $res;
 237 }
 238
 239 use constant MAX_MAIL_SIZE => 1024*1024;
 240 sub sendmail {
 241     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 242
 243     my $opts = $self->{opts};
 244
 245     my $mailto = $opts->{mailto};
 246
 247     return if !($mailto && scalar(@$mailto));
 248
 249     my $cmdline = $self->{cmdline};
 250
 251     my $ecount = 0;
 252     foreach my $task (@$tasklist) {
 253         $ecount++ if $task->{state} ne 'ok';
 254         chomp $task->{msg} if $task->{msg};
 255         $task->{backuptime} = 0 if !$task->{backuptime};
 256         $task->{size} = 0 if !$task->{size};
 257         $task->{target} = 'unknown' if !$task->{target};
 258         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 259
 260         if ($task->{state} eq 'todo') {
 261             $task->{msg} = 'aborted';
 262         }
 263     }
 264
 265     my $notify = $opts->{mailnotification} || 'always';
 266     return if (!$ecount && !$err && ($notify eq 'failure'));
 267
 268     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 269     if ($err) {
 270         if ($err =~ /\n/) {
 271             $stat .= ": multiple problems";
 272         } else {
 273             $stat .= ": $err";
 274             $err = undef;
 275         }
 276     }
 277
 278     my $hostname = `hostname -f` || PVE::INotify::nodename();
 279     chomp $hostname;
 280
 281     # text part
 282     my $text = $err ? "$err\n\n" : '';
 283     my $namelength = 20;
 284     $text .= sprintf (
 285         "%-10s %-${namelength}s %-6s %10s %10s  %s\n",
 286         qw(VMID NAME STATUS TIME SIZE FILENAME)
 287     );
 288     foreach my $task (@$tasklist) {
 289         my $name = substr($task->{hostname}, 0, $namelength);
 290         my $successful = $task->{state} eq 'ok';
 291         my $size = $successful ? format_size ($task->{size}) : 0;
 292         my $filename = $successful ? $task->{target} : '-';
 293         my $size_fmt = $successful ? "%10s": "%8.2fMB";
 294         $text .= sprintf(
 295             "%-10s %-${namelength}s %-6s %10s $size_fmt  %s\n",
 296             $task->{vmid},
 297             $name,
 298             $task->{state},
 299             format_time($task->{backuptime}),
 300             $size,
 301             $filename,
 302         );
 303     }
 304
 305     my $text_log_part;
 306     $text_log_part .= "\nDetailed backup logs:\n\n";
 307     $text_log_part .= "$cmdline\n\n";
 308
 309     $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
 310     foreach my $task (@$tasklist) {
 311         my $vmid = $task->{vmid};
 312         my $log = $task->{tmplog};
 313         if (!$log) {
 314             $text_log_part .= "$vmid: no log available\n\n";
 315             next;
 316         }
 317         if (open (my $TMP, '<', "$log")) {
 318             while (my $line = <$TMP>) {
 319                 next if $line =~ /^status: \d+/; # not useful in mails
 320                 $text_log_part .= encode8bit ("$vmid: $line");
 321             }
 322             close ($TMP);
 323         } else {
 324             $text_log_part .= "$vmid: Could not open log file\n\n";
 325         }
 326         $text_log_part .= "\n";
 327     }
 328     $text_log_part .= $detail_post if defined($detail_post);
 329
 330     # html part
 331     my $html = "<html><body>\n";
 332     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 333     $html .= "<table border=1 cellpadding=3>\n";
 334     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 335
 336     my $ssize = 0;
 337     foreach my $task (@$tasklist) {
 338         my $vmid = $task->{vmid};
 339         my $name = $task->{hostname};
 340
 341         if  ($task->{state} eq 'ok') {
 342             $ssize += $task->{size};
 343
 344             $html .= sprintf (
 345                 "<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 346                 $vmid,
 347                 $name,
 348                 format_time($task->{backuptime}),
 349                 format_size ($task->{size}),
 350                 escape_html ($task->{target}),
 351             );
 352         } else {
 353             $html .= sprintf (
 354                 "<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 355                 $vmid,
 356                 $name,
 357                 format_time($task->{backuptime}),
 358                 escape_html ($task->{msg}),
 359             );
 360         }
 361     }
 362
 363     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 364  format_time ($totaltime), format_size ($ssize));
 365
 366     $html .= "\n</table><br><br>\n";
 367     my $html_log_part;
 368     $html_log_part .= "Detailed backup logs:<br /><br />\n";
 369     $html_log_part .= "<pre>\n";
 370     $html_log_part .= escape_html($cmdline) . "\n\n";
 371
 372     $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 373     foreach my $task (@$tasklist) {
 374         my $vmid = $task->{vmid};
 375         my $log = $task->{tmplog};
 376         if (!$log) {
 377             $html_log_part .= "$vmid: no log available\n\n";
 378             next;
 379         }
 380         if (open (my $TMP, '<', "$log")) {
 381             while (my $line = <$TMP>) {
 382                 next if $line =~ /^status: \d+/; # not useful in mails
 383                 if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 384                     $html_log_part .= encode8bit ("$vmid: <font color=red>".
 385                         escape_html ($line) . "</font>");
 386                 } else {
 387                     $html_log_part .= encode8bit ("$vmid: " . escape_html ($line));
 388                 }
 389             }
 390             close ($TMP);
 391         } else {
 392             $html_log_part .= "$vmid: Could not open log file\n\n";
 393         }
 394         $html_log_part .= "\n";
 395     }
 396     $html_log_part .= escape_html($detail_post) if defined($detail_post);
 397     $html_log_part .= "</pre>";
 398     my $html_end = "\n</body></html>\n";
 399     # end html part
 400
 401     if (length($text) + length($text_log_part) +
 402         length($html) + length($html_log_part) +
 403         length($html_end) < MAX_MAIL_SIZE)
 404     {
 405         $html .= $html_log_part;
 406         $html .= $html_end;
 407         $text .= $text_log_part;
 408     } else {
 409         my $msg = "Log output was too long to be sent by mail. ".
 410             "See Task History for details!\n";
 411         $text .= $msg;
 412         $html .= "<p>$msg</p>";
 413         $html .= $html_end;
 414     }
 415
 416     my $subject = "vzdump backup status ($hostname) : $stat";
 417
 418     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 419     my $mailfrom = $dcconf->{email_from} || "root";
 420
 421     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 422 };
 423
 424 sub new {
 425     my ($class, $cmdline, $opts, $skiplist) = @_;
 426
 427     mkpath $logdir;
 428
 429     check_bin ('cp');
 430     check_bin ('df');
 431     check_bin ('sendmail');
 432     check_bin ('rsync');
 433     check_bin ('tar');
 434     check_bin ('mount');
 435     check_bin ('umount');
 436     check_bin ('cstream');
 437     check_bin ('ionice');
 438
 439     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 440         check_bin ('lvcreate');
 441         check_bin ('lvs');
 442         check_bin ('lvremove');
 443     }
 444
 445     my $defaults = read_vzdump_defaults();
 446
 447     foreach my $k (keys %$defaults) {
 448         next if $k eq 'exclude-path' || $k eq 'prune-backups'; # dealt with separately
 449         if ($k eq 'dumpdir' || $k eq 'storage') {
 450             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 451                 !defined ($opts->{storage});
 452         } else {
 453             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 454         }
 455     }
 456
 457     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 458     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 459
 460     $skiplist = [] if !$skiplist;
 461     my $self = bless {
 462         cmdline => $cmdline,
 463         opts => $opts,
 464         skiplist => $skiplist,
 465     }, $class;
 466
 467     my $findexcl = $self->{findexcl} = [];
 468     if ($defaults->{'exclude-path'}) {
 469         push @$findexcl, @{$defaults->{'exclude-path'}};
 470     }
 471
 472     if ($opts->{'exclude-path'}) {
 473         push @$findexcl, @{$opts->{'exclude-path'}};
 474     }
 475
 476     if ($opts->{stdexcludes}) {
 477         push @$findexcl,
 478             '/tmp/?*',
 479             '/var/tmp/?*',
 480             '/var/run/?*.pid',
 481             ;
 482     }
 483
 484     foreach my $p (@plugins) {
 485         my $pd = $p->new($self);
 486
 487         push @{$self->{plugins}}, $pd;
 488     }
 489
 490     if (defined($opts->{storage}) && $opts->{stdout}) {
 491         die "cannot use options 'storage' and 'stdout' at the same time\n";
 492     } elsif (defined($opts->{storage}) && defined($opts->{dumpdir})) {
 493         die "cannot use options 'storage' and 'dumpdir' at the same time\n";
 494     }
 495
 496     if (!$opts->{dumpdir} && !$opts->{storage}) {
 497         $opts->{storage} = 'local';
 498     }
 499
 500     $self->{job_init_log} = '';
 501     open my $job_init_fd, '>', \$self->{job_init_log};
 502     $self->run_hook_script('job-init', undef, $job_init_fd);
 503     close $job_init_fd;
 504
 505     PVE::Cluster::cfs_update(); # Pick up possible changes made by the hook script.
 506
 507     my $errors = '';
 508
 509     if ($opts->{storage}) {
 510         my $storage_cfg = PVE::Storage::config();
 511         eval { PVE::Storage::activate_storage($storage_cfg, $opts->{storage}) };
 512         if (my $err = $@) {
 513             chomp($err);
 514             $errors .= "could not activate storage '$opts->{storage}': $err";
 515         }
 516
 517         my $info = eval { storage_info ($opts->{storage}) };
 518         if (my $err = $@) {
 519             chomp($err);
 520             $errors .= "could not get storage information for '$opts->{storage}': $err";
 521         } else {
 522             $opts->{dumpdir} = $info->{dumpdir};
 523             $opts->{scfg} = $info->{scfg};
 524             $opts->{pbs} = $info->{pbs};
 525             $opts->{'prune-backups'} //= $info->{'prune-backups'};
 526         }
 527     } elsif ($opts->{dumpdir}) {
 528         $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
 529             if ! -d $opts->{dumpdir};
 530     } else {
 531         die "internal error";
 532     }
 533
 534     $opts->{'prune-backups'} //= $defaults->{'prune-backups'};
 535
 536     # avoid triggering any remove code path if keep-all is set
 537     $opts->{remove} = 0 if $opts->{'prune-backups'}->{'keep-all'};
 538
 539     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 540         $errors .= "\n" if $errors;
 541         $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
 542     }
 543
 544     if ($errors) {
 545         eval { $self->sendmail([], 0, $errors); };
 546         debugmsg ('err', $@) if $@;
 547         die "$errors\n";
 548     }
 549
 550     return $self;
 551 }
 552
 553 sub get_mount_info {
 554     my ($dir) = @_;
 555
 556     # Note: df 'available' can be negative, and percentage set to '-'
 557
 558     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 559
 560     my $res;
 561
 562     my $parser = sub {
 563         my $line = shift;
 564         if (my ($fsid, $fstype, undef, $mp) = $line =~
 565             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 566             $res = {
 567                 device => $fsid,
 568                 fstype => $fstype,
 569                 mountpoint => $mp,
 570             };
 571         }
 572     };
 573
 574     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 575     warn $@ if $@;
 576
 577     return $res;
 578 }
 579
 580 sub getlock {
 581     my ($self, $upid) = @_;
 582
 583     my $fh;
 584
 585     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 586
 587     die "missing UPID" if !$upid; # should not happen
 588
 589     my $SERVER_FLCK;
 590     if (!open ($SERVER_FLCK, '>>', "$lockfile")) {
 591         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 592         die "can't open lock on file '$lockfile' - $!";
 593     }
 594
 595     if (!flock ($SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 596         if (!$maxwait) {
 597             debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
 598             die "can't acquire lock '$lockfile' (wait = 0)";
 599         }
 600
 601         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 602         eval {
 603             alarm ($maxwait * 60);
 604
 605             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 606
 607             if (!flock ($SERVER_FLCK, LOCK_EX)) {
 608                 my $err = $!;
 609                 close ($SERVER_FLCK);
 610                 alarm (0);
 611                 die "$err\n";
 612             }
 613             alarm (0);
 614         };
 615         alarm (0);
 616
 617         my $err = $@;
 618
 619         if ($err) {
 620             debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
 621             die "can't acquire lock '$lockfile' - $err";
 622         }
 623
 624         debugmsg('info', "got global lock", undef, 1);
 625     }
 626
 627     PVE::Tools::file_set_contents($pidfile, $upid);
 628
 629     return $SERVER_FLCK;
 630 }
 631
 632 sub run_hook_script {
 633     my ($self, $phase, $task, $logfd) = @_;
 634
 635     my $opts = $self->{opts};
 636
 637     my $script = $opts->{script};
 638     return if !$script;
 639
 640     if (!-x $script) {
 641         die "The hook script '$script' is not executable.\n";
 642     }
 643
 644     my $cmd = [$script, $phase];
 645
 646     if ($task) {
 647         push @$cmd, $task->{mode};
 648         push @$cmd, $task->{vmid};
 649     }
 650
 651     local %ENV;
 652     # set immutable opts directly (so they are available in all phases)
 653     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 654     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 655
 656     foreach my $ek (qw(vmtype hostname target logfile)) {
 657         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 658     }
 659
 660     run_command ($logfd, $cmd);
 661 }
 662
 663 sub compressor_info {
 664     my ($opts) = @_;
 665     my $opt_compress = $opts->{compress};
 666
 667     if (!$opt_compress || $opt_compress eq '0') {
 668         return undef;
 669     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 670         return ('lzop', 'lzo');
 671     } elsif ($opt_compress eq 'gzip') {
 672         if ($opts->{pigz} > 0) {
 673             my $pigz_threads = $opts->{pigz};
 674             if ($pigz_threads == 1) {
 675                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 676                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 677             }
 678             return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
 679         } else {
 680             return ('gzip --rsyncable', 'gz');
 681         }
 682     } elsif ($opt_compress eq 'zstd') {
 683         my $zstd_threads = $opts->{zstd} // 1;
 684         if ($zstd_threads == 0) {
 685             my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 686             $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
 687         }
 688         return ("zstd --rsyncable --threads=${zstd_threads}", 'zst');
 689     } else {
 690         die "internal error - unknown compression option '$opt_compress'";
 691     }
 692 }
 693
 694 sub get_unprotected_backup_file_list {
 695     my ($dir, $bkname) = @_;
 696
 697     my $bklist = [];
 698     foreach my $fn (<$dir/${bkname}-*>) {
 699         my $archive_info = eval { PVE::Storage::archive_info($fn) } // {};
 700         if ($archive_info->{is_std_name}) {
 701             my $path = "$dir/$archive_info->{filename}";
 702             next if -e PVE::Storage::protection_file_path($path);
 703             my $backup = {
 704                 'path' => $path,
 705                 'ctime' => $archive_info->{ctime},
 706             };
 707             push @{$bklist}, $backup;
 708         }
 709     }
 710
 711     return $bklist;
 712 }
 713
 714 sub exec_backup_task {
 715     my ($self, $task) = @_;
 716
 717     my $opts = $self->{opts};
 718
 719     my $cfg = PVE::Storage::config();
 720     my $vmid = $task->{vmid};
 721     my $plugin = $task->{plugin};
 722
 723     $task->{backup_time} = time();
 724
 725     my $pbs_group_name;
 726     my $pbs_snapshot_name;
 727
 728     my $vmstarttime = time ();
 729
 730     my $logfd;
 731
 732     my $cleanup = {};
 733
 734     my $log_vm_online_again = sub {
 735         return if !defined($task->{vmstoptime});
 736         $task->{vmconttime} //= time();
 737         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 738         $delay = '<1' if $delay < 1;
 739         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 740     };
 741
 742     eval {
 743         die "unable to find VM '$vmid'\n" if !$plugin;
 744
 745         my $vmtype = $plugin->type();
 746
 747         if ($self->{opts}->{pbs}) {
 748             if ($vmtype eq 'lxc') {
 749                 $pbs_group_name = "ct/$vmid";
 750             } elsif  ($vmtype eq 'qemu') {
 751                 $pbs_group_name = "vm/$vmid";
 752             } else {
 753                 die "pbs backup not implemented for plugin type '$vmtype'\n";
 754             }
 755             my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
 756             $pbs_snapshot_name = "$pbs_group_name/$btime";
 757         }
 758
 759         # for now we deny backups of a running ha managed service in *stop* mode
 760         # as it interferes with the HA stack (started services should not stop).
 761         if ($opts->{mode} eq 'stop' &&
 762             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 763         {
 764             die "Cannot execute a backup with stop mode on a HA managed and".
 765                 " enabled Service. Use snapshot mode or disable the Service.\n";
 766         }
 767
 768         my $tmplog = "$logdir/$vmtype-$vmid.log";
 769
 770         my $bkname = "vzdump-$vmtype-$vmid";
 771         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
 772
 773         my $prune_options = $opts->{'prune-backups'};
 774
 775         my $backup_limit = 0;
 776         if (!$prune_options->{'keep-all'}) {
 777             foreach my $keep (values %{$prune_options}) {
 778                 $backup_limit += $keep;
 779             }
 780         }
 781
 782         if ($backup_limit && !$opts->{remove}) {
 783             my $count;
 784             if (my $storeid = $opts->{storage}) {
 785                 my $backups = PVE::Storage::volume_list($cfg, $storeid, $vmid, 'backup');
 786                 $count = grep {
 787                     !$_->{protected} && (!$_->{subtype} || $_->{subtype} eq $vmtype)
 788                 } $backups->@*;
 789             } else {
 790                 $count = scalar(get_unprotected_backup_file_list($opts->{dumpdir}, $bkname)->@*);
 791             }
 792
 793             die "There is a max backup limit of $backup_limit enforced by the".
 794                 " target storage or the vzdump parameters.".
 795                 " Either increase the limit or delete old backup(s).\n"
 796                 if $count >= $backup_limit;
 797         }
 798
 799         if (!$self->{opts}->{pbs}) {
 800             $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 801         }
 802
 803         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 804         my ($comp, $comp_ext) = compressor_info($opts);
 805         if ($comp && $comp_ext) {
 806             $ext .= ".${comp_ext}";
 807         }
 808
 809         if ($self->{opts}->{pbs}) {
 810             die "unable to pipe backup to stdout\n" if $opts->{stdout};
 811             $task->{target} = $pbs_snapshot_name;
 812         } else {
 813             if ($opts->{stdout}) {
 814                 $task->{target} = '-';
 815             } else {
 816                 $task->{target} = $task->{tmptar} = "$opts->{dumpdir}/$basename$ext";
 817                 $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 818                 unlink $task->{tmptar};
 819             }
 820         }
 821
 822         $task->{vmtype} = $vmtype;
 823
 824         my $pid = $$;
 825         if ($opts->{tmpdir}) {
 826             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp${pid}_$vmid/";
 827         } elsif ($self->{opts}->{pbs}) {
 828             $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 829         } else {
 830             # dumpdir is posix? then use it as temporary dir
 831             my $info = get_mount_info($opts->{dumpdir});
 832             if ($vmtype eq 'qemu' ||
 833                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 834                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 835             } else {
 836                 $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 837                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 838                           "using $task->{tmpdir} for temporary files", $logfd);
 839             }
 840         }
 841
 842         rmtree $task->{tmpdir};
 843         mkdir $task->{tmpdir};
 844         -d $task->{tmpdir} ||
 845             die "unable to create temporary directory '$task->{tmpdir}'";
 846
 847         $logfd = IO::File->new (">$tmplog") ||
 848             die "unable to create log file '$tmplog'";
 849
 850         $task->{dumpdir} = $opts->{dumpdir};
 851         $task->{storeid} = $opts->{storage};
 852         $task->{scfg} = $opts->{scfg};
 853         $task->{tmplog} = $tmplog;
 854
 855         unlink $task->{logfile} if defined($task->{logfile});
 856
 857         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 858         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
 859
 860         $plugin->set_logfd ($logfd);
 861
 862         # test is VM is running
 863         my ($running, $status_text) = $plugin->vm_status ($vmid);
 864
 865         debugmsg ('info', "status = ${status_text}", $logfd);
 866
 867         # lock VM (prevent config changes)
 868         $plugin->lock_vm ($vmid);
 869
 870         $cleanup->{unlock} = 1;
 871
 872         # prepare
 873
 874         my $mode = $running ? $task->{mode} : 'stop';
 875
 876         if ($mode eq 'snapshot') {
 877             my %saved_task = %$task;
 878             eval { $plugin->prepare ($task, $vmid, $mode); };
 879             if (my $err = $@) {
 880                 die $err if $err !~ m/^mode failure/;
 881                 debugmsg ('info',  $err, $logfd);
 882                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 883                 $mode = 'suspend'; # so prepare is called again below
 884                 %$task = %saved_task;
 885             }
 886         }
 887
 888         $cleanup->{prepared} = 1;
 889
 890         $task->{mode} = $mode;
 891
 892         debugmsg ('info', "backup mode: $mode", $logfd);
 893         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)  if $opts->{bwlimit};
 894         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 895
 896         if ($mode eq 'stop') {
 897             $plugin->prepare ($task, $vmid, $mode);
 898
 899             $self->run_hook_script ('backup-start', $task, $logfd);
 900
 901             if ($running) {
 902                 debugmsg ('info', "stopping virtual guest", $logfd);
 903                 $task->{vmstoptime} = time();
 904                 $self->run_hook_script ('pre-stop', $task, $logfd);
 905                 $plugin->stop_vm ($task, $vmid);
 906                 $cleanup->{restart} = 1;
 907             }
 908
 909
 910         } elsif ($mode eq 'suspend') {
 911             $plugin->prepare ($task, $vmid, $mode);
 912
 913             $self->run_hook_script ('backup-start', $task, $logfd);
 914
 915             if ($vmtype eq 'lxc') {
 916                 # pre-suspend rsync
 917                 $plugin->copy_data_phase1($task, $vmid);
 918             }
 919
 920             debugmsg ('info', "suspending guest", $logfd);
 921             $task->{vmstoptime} = time ();
 922             $self->run_hook_script ('pre-stop', $task, $logfd);
 923             $plugin->suspend_vm ($task, $vmid);
 924             $cleanup->{resume} = 1;
 925
 926             if ($vmtype eq 'lxc') {
 927                 # post-suspend rsync
 928                 $plugin->copy_data_phase2($task, $vmid);
 929
 930                 debugmsg ('info', "resuming guest", $logfd);
 931                 $cleanup->{resume} = 0;
 932                 $self->run_hook_script('pre-restart', $task, $logfd);
 933                 $plugin->resume_vm($task, $vmid);
 934                 $self->run_hook_script('post-restart', $task, $logfd);
 935                 $log_vm_online_again->();
 936             }
 937
 938         } elsif ($mode eq 'snapshot') {
 939             $self->run_hook_script ('backup-start', $task, $logfd);
 940
 941             my $snapshot_count = $task->{snapshot_count} || 0;
 942
 943             $self->run_hook_script ('pre-stop', $task, $logfd);
 944
 945             if ($snapshot_count > 1) {
 946                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 947                 $task->{vmstoptime} = time ();
 948                 $plugin->suspend_vm ($task, $vmid);
 949                 $cleanup->{resume} = 1;
 950             }
 951
 952             $plugin->snapshot ($task, $vmid);
 953
 954             $self->run_hook_script ('pre-restart', $task, $logfd);
 955
 956             if ($snapshot_count > 1) {
 957                 debugmsg ('info', "resume vm", $logfd);
 958                 $cleanup->{resume} = 0;
 959                 $plugin->resume_vm ($task, $vmid);
 960                 $log_vm_online_again->();
 961             }
 962
 963             $self->run_hook_script ('post-restart', $task, $logfd);
 964
 965         } else {
 966             die "internal error - unknown mode '$mode'\n";
 967         }
 968
 969         # assemble archive image
 970         $plugin->assemble ($task, $vmid);
 971
 972         # produce archive
 973
 974         if ($opts->{stdout}) {
 975             debugmsg ('info', "sending archive to stdout", $logfd);
 976             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 977             $self->run_hook_script ('backup-end', $task, $logfd);
 978             return;
 979         }
 980
 981         my $archive_txt = $self->{opts}->{pbs} ? 'Proxmox Backup Server' : 'vzdump';
 982         debugmsg('info', "creating $archive_txt archive '$task->{target}'", $logfd);
 983         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 984
 985         if ($self->{opts}->{pbs}) {
 986             # size is added to task struct in guest vzdump plugins
 987         } else {
 988             rename ($task->{tmptar}, $task->{target}) ||
 989                 die "unable to rename '$task->{tmptar}' to '$task->{target}'\n";
 990
 991             # determine size
 992             $task->{size} = (-s $task->{target}) || 0;
 993             my $cs = format_size ($task->{size});
 994             debugmsg ('info', "archive file size: $cs", $logfd);
 995         }
 996
 997         if ($opts->{remove}) {
 998             my $keepstr = join(', ', map { "$_=$prune_options->{$_}" } sort keys %$prune_options);
 999             debugmsg ('info', "prune older backups with retention: $keepstr", $logfd);
1000             my $pruned = 0;
1001             if (!defined($opts->{storage})) {
1002                 my $bklist = get_unprotected_backup_file_list($opts->{dumpdir}, $bkname);
1003
1004                 PVE::Storage::prune_mark_backup_group($bklist, $prune_options);
1005
1006                 foreach my $prune_entry (@{$bklist}) {
1007                     next if $prune_entry->{mark} ne 'remove';
1008                     $pruned++;
1009                     my $archive_path = $prune_entry->{path};
1010                     debugmsg ('info', "delete old backup '$archive_path'", $logfd);
1011                     PVE::Storage::archive_remove($archive_path);
1012                 }
1013             } else {
1014                 my $pruned_list = PVE::Storage::prune_backups(
1015                     $cfg,
1016                     $opts->{storage},
1017                     $prune_options,
1018                     $vmid,
1019                     $vmtype,
1020                     0,
1021                     sub { debugmsg($_[0], $_[1], $logfd) },
1022                 );
1023                 $pruned = scalar(grep { $_->{mark} eq 'remove' } $pruned_list->@*);
1024             }
1025             my $log_pruned_extra = $pruned > 0 ? " not covered by keep-retention policy" : "";
1026             debugmsg ('info', "pruned $pruned backup(s)${log_pruned_extra}", $logfd);
1027         }
1028
1029         $self->run_hook_script ('backup-end', $task, $logfd);
1030     };
1031     my $err = $@;
1032
1033     if ($plugin) {
1034         # clean-up
1035
1036         if ($cleanup->{unlock}) {
1037             eval { $plugin->unlock_vm ($vmid); };
1038             warn $@ if $@;
1039         }
1040
1041         if ($cleanup->{prepared}) {
1042             # only call cleanup when necessary (when prepare was executed)
1043             eval { $plugin->cleanup ($task, $vmid) };
1044             warn $@ if $@;
1045         }
1046
1047         eval { $plugin->set_logfd (undef); };
1048         warn $@ if $@;
1049
1050         if ($cleanup->{resume} || $cleanup->{restart}) {
1051             eval {
1052                 $self->run_hook_script ('pre-restart', $task, $logfd);
1053                 if ($cleanup->{resume}) {
1054                     debugmsg ('info', "resume vm", $logfd);
1055                     $plugin->resume_vm ($task, $vmid);
1056                 } else {
1057                     my $running = $plugin->vm_status($vmid);
1058                     if (!$running) {
1059                         debugmsg ('info', "restarting vm", $logfd);
1060                         $plugin->start_vm ($task, $vmid);
1061                     }
1062                 }
1063                 $self->run_hook_script ('post-restart', $task, $logfd);
1064             };
1065             my $err = $@;
1066             if ($err) {
1067                 warn $err;
1068             } else {
1069                 $log_vm_online_again->();
1070             }
1071         }
1072     }
1073
1074     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
1075     warn $@ if $@;
1076
1077     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
1078     warn $@ if $@;
1079
1080     my $delay = $task->{backuptime} = time () - $vmstarttime;
1081
1082     if ($err) {
1083         $task->{state} = 'err';
1084         $task->{msg} = $err;
1085         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1086         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
1087
1088         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1089
1090     } else {
1091         $task->{state} = 'ok';
1092         my $tstr = format_time ($delay);
1093         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1094         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1095     }
1096
1097     close ($logfd) if $logfd;
1098
1099     if ($task->{tmplog}) {
1100         if ($self->{opts}->{pbs}) {
1101             if ($task->{state} eq 'ok') {
1102                 eval {
1103                     PVE::Storage::PBSPlugin::run_raw_client_cmd(
1104                         $opts->{scfg},
1105                         $opts->{storage},
1106                         'upload-log',
1107                         [ $pbs_snapshot_name, $task->{tmplog} ],
1108                         errmsg => "uploading backup task log failed",
1109                         outfunc => sub {},
1110                     );
1111                 };
1112                 debugmsg('warn', "$@") if $@; # $@ contains already error prefix
1113             }
1114         } elsif ($task->{logfile}) {
1115             system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1116         }
1117     }
1118
1119     eval { $self->run_hook_script ('log-end', $task); };
1120
1121     die $err if $err && $err =~ m/^interrupted by signal$/;
1122 }
1123
1124 sub exec_backup {
1125     my ($self, $rpcenv, $authuser) = @_;
1126
1127     my $opts = $self->{opts};
1128
1129     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1130
1131     if (scalar(@{$self->{skiplist}})) {
1132         my $skip_string = join(', ', sort { $a <=> $b } @{$self->{skiplist}});
1133         debugmsg ('info', "skip external VMs: $skip_string");
1134     }
1135
1136     my $tasklist = [];
1137     my $vzdump_plugins =  {};
1138     foreach my $plugin (@{$self->{plugins}}) {
1139         my $type = $plugin->type();
1140         next if exists $vzdump_plugins->{$type};
1141         $vzdump_plugins->{$type} = $plugin;
1142     }
1143
1144     my $vmlist = PVE::Cluster::get_vmlist();
1145     my $vmids = [ sort { $a <=> $b } @{$opts->{vmids}} ];
1146     foreach my $vmid (@{$vmids}) {
1147         my $plugin;
1148         if (defined($vmlist->{ids}->{$vmid})) {
1149             my $guest_type = $vmlist->{ids}->{$vmid}->{type};
1150             $plugin = $vzdump_plugins->{$guest_type};
1151             next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all});
1152         }
1153         push @$tasklist, {
1154             mode => $opts->{mode},
1155             plugin => $plugin,
1156             state => 'todo',
1157             vmid => $vmid,
1158         };
1159     }
1160
1161     # Use in-memory files for the outer hook logs to pass them to sendmail.
1162     my $job_start_log = '';
1163     my $job_end_log = '';
1164     open my $job_start_fd, '>', \$job_start_log;
1165     open my $job_end_fd, '>', \$job_end_log;
1166
1167     my $starttime = time();
1168     my $errcount = 0;
1169     eval {
1170
1171         $self->run_hook_script ('job-start', undef, $job_start_fd);
1172
1173         foreach my $task (@$tasklist) {
1174             $self->exec_backup_task ($task);
1175             $errcount += 1 if $task->{state} ne 'ok';
1176         }
1177
1178         $self->run_hook_script ('job-end', undef, $job_end_fd);
1179     };
1180     my $err = $@;
1181
1182     $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1183
1184     if ($err) {
1185         debugmsg ('err', "Backup job failed - $err", undef, 1);
1186     } else {
1187         if ($errcount) {
1188             debugmsg ('info', "Backup job finished with errors", undef, 1);
1189         } else {
1190             debugmsg ('info', "Backup job finished successfully", undef, 1);
1191         }
1192     }
1193
1194     close $job_start_fd;
1195     close $job_end_fd;
1196
1197     my $totaltime = time() - $starttime;
1198
1199     eval {
1200         $self->sendmail(
1201             $tasklist,
1202             $totaltime,
1203             undef,
1204             $self->{job_init_log} . $job_start_log,
1205             $job_end_log,
1206         );
1207     };
1208     debugmsg ('err', $@) if $@;
1209
1210     die $err if $err;
1211
1212     die "job errors\n" if $errcount;
1213
1214     unlink $pidfile;
1215 }
1216
1217
1218 sub option_exists {
1219     my $key = shift;
1220     return defined($confdesc->{$key});
1221 }
1222
1223 # NOTE it might make sense to merge this and verify_vzdump_parameters(), but one
1224 # needs to adapt command_line() in guest-common's PVE/VZDump/Common.pm and detect
1225 # a second parsing attempt, because verify_vzdump_parameters() is called twice
1226 # during the update_job API call.
1227 sub parse_mailto_exclude_path {
1228     my ($param) = @_;
1229
1230     # exclude-path list need to be 0 separated
1231     if (defined($param->{'exclude-path'})) {
1232         my @expaths = split(/\0/, $param->{'exclude-path'} || '');
1233         $param->{'exclude-path'} = [ @expaths ];
1234     }
1235
1236     if (defined($param->{mailto})) {
1237         my @mailto = PVE::Tools::split_list(extract_param($param, 'mailto'));
1238         $param->{mailto} = [ @mailto ];
1239     }
1240
1241     return;
1242 }
1243
1244 sub verify_vzdump_parameters {
1245     my ($param, $check_missing) = @_;
1246
1247     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1248         if $param->{all} && $param->{vmid};
1249
1250     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1251         if $param->{exclude} && $param->{vmid};
1252
1253     raise_param_exc({ pool => "option conflicts with option 'vmid'"})
1254         if $param->{pool} && $param->{vmid};
1255
1256     raise_param_exc({ 'prune-backups' => "option conflicts with option 'maxfiles'"})
1257         if defined($param->{'prune-backups'}) && defined($param->{maxfiles});
1258
1259     $parse_prune_backups_maxfiles->($param, 'CLI parameters');
1260
1261     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
1262
1263     return if !$check_missing;
1264
1265     raise_param_exc({ vmid => "property is missing"})
1266         if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
1267
1268 }
1269
1270 sub stop_running_backups {
1271     my($self) = @_;
1272
1273     my $upid = PVE::Tools::file_read_firstline($pidfile);
1274     return if !$upid;
1275
1276     my $task = PVE::Tools::upid_decode($upid);
1277
1278     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1279         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1280         kill(15, $task->{pid});
1281         # wait max 15 seconds to shut down (else, do nothing for now)
1282         my $i;
1283         for ($i = 15; $i > 0; $i--) {
1284             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1285             sleep (1);
1286         }
1287         die "stopping backup process $task->{pid} failed\n" if $i == 0;
1288     }
1289 }
1290
1291 sub get_included_guests {
1292     my ($job) = @_;
1293
1294     my $vmids = [];
1295     my $vmids_per_node = {};
1296
1297     my $vmlist = PVE::Cluster::get_vmlist();
1298
1299     if ($job->{pool}) {
1300         $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool});
1301     } elsif ($job->{vmid}) {
1302         $vmids = [ split_list($job->{vmid}) ];
1303     } elsif ($job->{all}) {
1304         # all or exclude
1305         my $exclude = check_vmids(split_list($job->{exclude}));
1306         my $excludehash = { map { $_ => 1 } @$exclude };
1307
1308         for my $id (keys %{$vmlist->{ids}}) {
1309             next if $excludehash->{$id};
1310             push @$vmids, $id;
1311         }
1312     } else {
1313         return $vmids_per_node;
1314     }
1315     $vmids = check_vmids(@$vmids);
1316
1317     for my $vmid (@$vmids) {
1318         if (defined($vmlist->{ids}->{$vmid})) {
1319             my $node = $vmlist->{ids}->{$vmid}->{node};
1320             next if (defined $job->{node} && $job->{node} ne $node);
1321
1322             push @{$vmids_per_node->{$node}}, $vmid;
1323         } else {
1324             push @{$vmids_per_node->{''}}, $vmid;
1325         }
1326     }
1327
1328     return $vmids_per_node;
1329 }
1330
1331 1;