PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Path;
   8 use IO::File;
   9 use IO::Select;
  10 use IPC::Open3;
  11 use POSIX qw(strftime);
  12 use Time::Local;
  13
  14 use PVE::Cluster qw(cfs_read_file);
  15 use PVE::DataCenterConfig;
  16 use PVE::Exception qw(raise_param_exc);
  17 use PVE::HA::Config;
  18 use PVE::HA::Env::PVE2;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::RPCEnvironment;
  21 use PVE::Storage;
  22 use PVE::VZDump::Common;
  23 use PVE::VZDump::Plugin;
  24 use PVE::Tools qw(extract_param split_list);
  25 use PVE::API2Tools;
  26
  27 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  28
  29 my $lockfile = '/var/run/vzdump.lock';
  30 my $pidfile = '/var/run/vzdump.pid';
  31 my $logdir = '/var/log/vzdump';
  32
  33 my @plugins = qw();
  34
  35 my $confdesc = PVE::VZDump::Common::get_confdesc();
  36
  37 # Load available plugins
  38 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
  39 foreach my $plug (@pve_vzdump_classes) {
  40     my $filename = "/usr/share/perl5/$plug.pm";
  41     $filename =~ s!::!/!g;
  42     if (-f $filename) {
  43         eval { require $filename; };
  44         if (!$@) {
  45             $plug->import ();
  46             push @plugins, $plug;
  47         } else {
  48             die $@;
  49         }
  50     }
  51 }
  52
  53 # helper functions
  54
  55 sub debugmsg {
  56     my ($mtype, $msg, $logfd, $syslog) = @_;
  57
  58     PVE::VZDump::Plugin::debugmsg(@_);
  59 }
  60
  61 sub run_command {
  62     my ($logfd, $cmdstr, %param) = @_;
  63
  64     my $logfunc = sub {
  65         my $line = shift;
  66         debugmsg ('info', $line, $logfd);
  67     };
  68
  69     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
  70 }
  71
  72 sub storage_info {
  73     my $storage = shift;
  74
  75     my $cfg = PVE::Storage::config();
  76     my $scfg = PVE::Storage::storage_config($cfg, $storage);
  77     my $type = $scfg->{type};
  78
  79     die "can't use storage type '$type' for backup\n"
  80         if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'
  81               || $type eq 'cifs' || $type eq 'cephfs' || $type eq 'pbs'));
  82     die "can't use storage '$storage' for backups - wrong content type\n"
  83         if (!$scfg->{content}->{backup});
  84
  85     PVE::Storage::activate_storage($cfg, $storage);
  86
  87     if ($type eq 'pbs') {
  88         return {
  89             scfg => $scfg,
  90             maxfiles => $scfg->{maxfiles},
  91             pbs => 1,
  92         };
  93     } else {
  94         return {
  95             scfg => $scfg,
  96             dumpdir => PVE::Storage::get_backup_dir($cfg, $storage),
  97             maxfiles => $scfg->{maxfiles},
  98         };
  99     }
 100 }
 101
 102 sub format_size {
 103     my $size = shift;
 104
 105     my $kb = $size / 1024;
 106
 107     if ($kb < 1024) {
 108         return int ($kb) . "KB";
 109     }
 110
 111     my $mb = $size / (1024*1024);
 112     if ($mb < 1024) {
 113         return int ($mb) . "MB";
 114     }
 115     my $gb = $mb / 1024;
 116     if ($gb < 1024) {
 117         return sprintf ("%.2fGB", $gb);
 118     }
 119     my $tb = $gb / 1024;
 120     return sprintf ("%.2fTB", $tb);
 121 }
 122
 123 sub format_time {
 124     my $seconds = shift;
 125
 126     my $hours = int ($seconds/3600);
 127     $seconds = $seconds - $hours*3600;
 128     my $min = int ($seconds/60);
 129     $seconds = $seconds - $min*60;
 130
 131     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 132 }
 133
 134 sub encode8bit {
 135     my ($str) = @_;
 136
 137     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 138
 139     return $str;
 140 }
 141
 142 sub escape_html {
 143     my ($str) = @_;
 144
 145     $str =~ s/&/&amp;/g;
 146     $str =~ s/</&lt;/g;
 147     $str =~ s/>/&gt;/g;
 148
 149     return $str;
 150 }
 151
 152 sub check_bin {
 153     my ($bin)  = @_;
 154
 155     foreach my $p (split (/:/, $ENV{PATH})) {
 156         my $fn = "$p/$bin";
 157         if (-x $fn) {
 158             return $fn;
 159         }
 160     }
 161
 162     die "unable to find command '$bin'\n";
 163 }
 164
 165 sub check_vmids {
 166     my (@vmids) = @_;
 167
 168     my $res = [];
 169     for my $vmid (sort {$a <=> $b} @vmids) {
 170         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 171         $vmid = int ($vmid); # remove leading zeros
 172         next if !$vmid;
 173         push @$res, $vmid;
 174     }
 175
 176     return $res;
 177 }
 178
 179
 180 sub read_vzdump_defaults {
 181
 182     my $fn = "/etc/vzdump.conf";
 183
 184     my $defaults = {
 185         map {
 186             my $default = $confdesc->{$_}->{default};
 187              defined($default) ? ($_ => $default) : ()
 188         } keys %$confdesc
 189     };
 190
 191     my $raw;
 192     eval { $raw = PVE::Tools::file_get_contents($fn); };
 193     return $defaults if $@;
 194
 195     my $conf_schema = { type => 'object', properties => $confdesc, };
 196     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 197     if (my $excludes = $res->{'exclude-path'}) {
 198         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 199     }
 200     if (defined($res->{mailto})) {
 201         my @mailto = split_list($res->{mailto});
 202         $res->{mailto} = [ @mailto ];
 203     }
 204
 205     foreach my $key (keys %$defaults) {
 206         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 207     }
 208
 209     return $res;
 210 }
 211
 212 use constant MAX_MAIL_SIZE => 1024*1024;
 213 sub sendmail {
 214     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 215
 216     my $opts = $self->{opts};
 217
 218     my $mailto = $opts->{mailto};
 219
 220     return if !($mailto && scalar(@$mailto));
 221
 222     my $cmdline = $self->{cmdline};
 223
 224     my $ecount = 0;
 225     foreach my $task (@$tasklist) {
 226         $ecount++ if $task->{state} ne 'ok';
 227         chomp $task->{msg} if $task->{msg};
 228         $task->{backuptime} = 0 if !$task->{backuptime};
 229         $task->{size} = 0 if !$task->{size};
 230         $task->{tarfile} = 'unknown' if !$task->{tarfile};
 231         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 232
 233         if ($task->{state} eq 'todo') {
 234             $task->{msg} = 'aborted';
 235         }
 236     }
 237
 238     my $notify = $opts->{mailnotification} || 'always';
 239     return if (!$ecount && !$err && ($notify eq 'failure'));
 240
 241     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 242     if ($err) {
 243         if ($err =~ /\n/) {
 244             $stat .= ": multiple problems";
 245         } else {
 246             $stat .= ": $err";
 247             $err = undef;
 248         }
 249     }
 250
 251     my $hostname = `hostname -f` || PVE::INotify::nodename();
 252     chomp $hostname;
 253
 254     # text part
 255     my $text = $err ? "$err\n\n" : '';
 256     $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", qw(VMID STATUS TIME SIZE FILENAME));
 257     foreach my $task (@$tasklist) {
 258         my $vmid = $task->{vmid};
 259         if  ($task->{state} eq 'ok') {
 260
 261             $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", $vmid,
 262                                 $task->{state},
 263                                 format_time($task->{backuptime}),
 264                                 format_size ($task->{size}),
 265                                 $task->{tarfile});
 266         } else {
 267             $text .= sprintf ("%-10s %-6s %10s %8.2fMB  %s\n", $vmid,
 268                                 $task->{state},
 269                                 format_time($task->{backuptime}),
 270                                 0, '-');
 271         }
 272     }
 273
 274     my $text_log_part;
 275     $text_log_part .= "\nDetailed backup logs:\n\n";
 276     $text_log_part .= "$cmdline\n\n";
 277
 278     $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
 279     foreach my $task (@$tasklist) {
 280         my $vmid = $task->{vmid};
 281         my $log = $task->{tmplog};
 282         if (!$log) {
 283             $text_log_part .= "$vmid: no log available\n\n";
 284             next;
 285         }
 286         if (open (TMP, "$log")) {
 287             while (my $line = <TMP>) {
 288                 next if $line =~ /^status: \d+/; # not useful in mails
 289                 $text_log_part .= encode8bit ("$vmid: $line");
 290             }
 291         } else {
 292             $text_log_part .= "$vmid: Could not open log file\n\n";
 293         }
 294         close (TMP);
 295         $text_log_part .= "\n";
 296     }
 297     $text_log_part .= $detail_post if defined($detail_post);
 298
 299     # html part
 300     my $html = "<html><body>\n";
 301     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 302     $html .= "<table border=1 cellpadding=3>\n";
 303     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 304
 305     my $ssize = 0;
 306
 307     foreach my $task (@$tasklist) {
 308         my $vmid = $task->{vmid};
 309         my $name = $task->{hostname};
 310
 311         if  ($task->{state} eq 'ok') {
 312
 313             $ssize += $task->{size};
 314
 315             $html .= sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 316                                 $vmid, $name,
 317                                 format_time($task->{backuptime}),
 318                                 format_size ($task->{size}),
 319                                 escape_html ($task->{tarfile}));
 320         } else {
 321             $html .= sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 322                                 $vmid, $name, format_time($task->{backuptime}),
 323                                 escape_html ($task->{msg}));
 324         }
 325     }
 326
 327     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 328  format_time ($totaltime), format_size ($ssize));
 329
 330     $html .= "\n</table><br><br>\n";
 331     my $html_log_part;
 332     $html_log_part .= "Detailed backup logs:<br /><br />\n";
 333     $html_log_part .= "<pre>\n";
 334     $html_log_part .= escape_html($cmdline) . "\n\n";
 335
 336     $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 337     foreach my $task (@$tasklist) {
 338         my $vmid = $task->{vmid};
 339         my $log = $task->{tmplog};
 340         if (!$log) {
 341             $html_log_part .= "$vmid: no log available\n\n";
 342             next;
 343         }
 344         if (open (TMP, "$log")) {
 345             while (my $line = <TMP>) {
 346                 next if $line =~ /^status: \d+/; # not useful in mails
 347                 if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 348                     $html_log_part .= encode8bit ("$vmid: <font color=red>".
 349                         escape_html ($line) . "</font>");
 350                 } else {
 351                     $html_log_part .= encode8bit ("$vmid: " . escape_html ($line));
 352                 }
 353             }
 354         } else {
 355             $html_log_part .= "$vmid: Could not open log file\n\n";
 356         }
 357         close (TMP);
 358         $html_log_part .= "\n";
 359     }
 360     $html_log_part .= escape_html($detail_post) if defined($detail_post);
 361     $html_log_part .= "</pre>";
 362     my $html_end .= "\n</body></html>\n";
 363     # end html part
 364
 365     if (length($text) + length($text_log_part) +
 366         length($html) + length($html_log_part) < MAX_MAIL_SIZE)
 367     {
 368         $html .= $html_log_part;
 369         $text .= $text_log_part;
 370     } else {
 371         my $msg = "Log output was too long to be sent by mail. ".
 372             "See Task History for details!\n";
 373         $text .= $msg;
 374         $html .= "<p>$msg</p>";
 375         $html .= $html_end;
 376     }
 377
 378     my $subject = "vzdump backup status ($hostname) : $stat";
 379
 380     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 381     my $mailfrom = $dcconf->{email_from} || "root";
 382
 383     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 384 };
 385
 386 sub new {
 387     my ($class, $cmdline, $opts, $skiplist) = @_;
 388
 389     mkpath $logdir;
 390
 391     check_bin ('cp');
 392     check_bin ('df');
 393     check_bin ('sendmail');
 394     check_bin ('rsync');
 395     check_bin ('tar');
 396     check_bin ('mount');
 397     check_bin ('umount');
 398     check_bin ('cstream');
 399     check_bin ('ionice');
 400
 401     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 402         check_bin ('lvcreate');
 403         check_bin ('lvs');
 404         check_bin ('lvremove');
 405     }
 406
 407     my $defaults = read_vzdump_defaults();
 408
 409     my $maxfiles = $opts->{maxfiles}; # save here, because we overwrite with default
 410
 411     $opts->{remove} = 1 if !defined($opts->{remove});
 412
 413     foreach my $k (keys %$defaults) {
 414         next if $k eq 'exclude-path'; # dealt with separately
 415         if ($k eq 'dumpdir' || $k eq 'storage') {
 416             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 417                 !defined ($opts->{storage});
 418         } else {
 419             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 420         }
 421     }
 422
 423     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 424     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 425
 426     $skiplist = [] if !$skiplist;
 427     my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist };
 428
 429     my $findexcl = $self->{findexcl} = [];
 430     if ($defaults->{'exclude-path'}) {
 431         push @$findexcl, @{$defaults->{'exclude-path'}};
 432     }
 433
 434     if ($opts->{'exclude-path'}) {
 435         push @$findexcl, @{$opts->{'exclude-path'}};
 436     }
 437
 438     if ($opts->{stdexcludes}) {
 439         push @$findexcl, '/tmp/?*',
 440                          '/var/tmp/?*',
 441                          '/var/run/?*.pid';
 442     }
 443
 444     foreach my $p (@plugins) {
 445
 446         my $pd = $p->new ($self);
 447
 448         push @{$self->{plugins}}, $pd;
 449     }
 450
 451     if (defined($opts->{storage}) && $opts->{stdout}) {
 452         die "cannot use options 'storage' and 'stdout' at the same time\n";
 453     } elsif (defined($opts->{storage}) && defined($opts->{dumpdir})) {
 454         die "cannot use options 'storage' and 'dumpdir' at the same time\n";
 455     }
 456
 457     if (!$opts->{dumpdir} && !$opts->{storage}) {
 458         $opts->{storage} = 'local';
 459     }
 460
 461     my $errors = '';
 462
 463     if ($opts->{storage}) {
 464         my $info = eval { storage_info ($opts->{storage}) };
 465         $errors .= "could not get storage information for '$opts->{storage}': $@"
 466             if ($@);
 467         $opts->{dumpdir} = $info->{dumpdir};
 468         $opts->{scfg} = $info->{scfg};
 469         $opts->{pbs} = $info->{pbs};
 470         $maxfiles //= $info->{maxfiles};
 471     } elsif ($opts->{dumpdir}) {
 472         $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
 473             if ! -d $opts->{dumpdir};
 474     } else {
 475         die "internal error";
 476     }
 477
 478     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 479         $errors .= "\n" if $errors;
 480         $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
 481     }
 482
 483     if ($errors) {
 484         eval { $self->sendmail([], 0, $errors); };
 485         debugmsg ('err', $@) if $@;
 486         die "$errors\n";
 487     }
 488
 489     $opts->{maxfiles} = $maxfiles if defined($maxfiles);
 490
 491     return $self;
 492
 493 }
 494
 495 sub get_mount_info {
 496     my ($dir) = @_;
 497
 498     # Note: df 'available' can be negative, and percentage set to '-'
 499
 500     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 501
 502     my $res;
 503
 504     my $parser = sub {
 505         my $line = shift;
 506         if (my ($fsid, $fstype, undef, $mp) = $line =~
 507             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 508             $res = {
 509                 device => $fsid,
 510                 fstype => $fstype,
 511                 mountpoint => $mp,
 512             };
 513         }
 514     };
 515
 516     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 517     warn $@ if $@;
 518
 519     return $res;
 520 }
 521
 522 sub getlock {
 523     my ($self, $upid) = @_;
 524
 525     my $fh;
 526
 527     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 528
 529     die "missimg UPID" if !$upid; # should not happen
 530
 531     if (!open (SERVER_FLCK, ">>$lockfile")) {
 532         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 533         die "can't open lock on file '$lockfile' - $!";
 534     }
 535
 536     if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 537
 538         if (!$maxwait) {
 539             debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
 540             die "can't acquire lock '$lockfile' (wait = 0)";
 541         }
 542
 543         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 544
 545         eval {
 546             alarm ($maxwait * 60);
 547
 548             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 549
 550             if (!flock (SERVER_FLCK, LOCK_EX)) {
 551                 my $err = $!;
 552                 close (SERVER_FLCK);
 553                 alarm (0);
 554                 die "$err\n";
 555             }
 556             alarm (0);
 557         };
 558         alarm (0);
 559
 560         my $err = $@;
 561
 562         if ($err) {
 563             debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
 564             die "can't acquire lock '$lockfile' - $err";
 565         }
 566
 567         debugmsg('info', "got global lock", undef, 1);
 568     }
 569
 570     PVE::Tools::file_set_contents($pidfile, $upid);
 571 }
 572
 573 sub run_hook_script {
 574     my ($self, $phase, $task, $logfd) = @_;
 575
 576     my $opts = $self->{opts};
 577
 578     my $script = $opts->{script};
 579     return if !$script;
 580
 581     if (!-x $script) {
 582         die "The hook script '$script' is not executable.\n";
 583     }
 584
 585     my $cmd = "$script $phase";
 586
 587     $cmd .= " $task->{mode} $task->{vmid}" if ($task);
 588
 589     local %ENV;
 590     # set immutable opts directly (so they are available in all phases)
 591     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 592     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 593
 594     foreach my $ek (qw(vmtype hostname tarfile logfile)) {
 595         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 596     }
 597
 598     run_command ($logfd, $cmd);
 599 }
 600
 601 sub compressor_info {
 602     my ($opts) = @_;
 603     my $opt_compress = $opts->{compress};
 604
 605     if (!$opt_compress || $opt_compress eq '0') {
 606         return undef;
 607     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 608         return ('lzop', 'lzo');
 609     } elsif ($opt_compress eq 'gzip') {
 610         if ($opts->{pigz} > 0) {
 611             my $pigz_threads = $opts->{pigz};
 612             if ($pigz_threads == 1) {
 613                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 614                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 615             }
 616             return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
 617         } else {
 618             return ('gzip --rsyncable', 'gz');
 619         }
 620     } elsif ($opt_compress eq 'zstd') {
 621         my $zstd_threads = $opts->{zstd} // 1;
 622         if ($zstd_threads == 0) {
 623             my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 624             $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
 625         }
 626         return ("zstd --rsyncable --threads=${zstd_threads}", 'zst');
 627     } else {
 628         die "internal error - unknown compression option '$opt_compress'";
 629     }
 630 }
 631
 632 sub get_backup_file_list {
 633     my ($dir, $bkname, $exclude_fn) = @_;
 634
 635     my $bklist = [];
 636     foreach my $fn (<$dir/${bkname}-*>) {
 637         next if $exclude_fn && $fn eq $exclude_fn;
 638         if ($fn =~ m!/(${bkname}-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?)))$!) {
 639             $fn = "$dir/$1"; # untaint
 640             my $t = timelocal ($7, $6, $5, $4, $3 - 1, $2);
 641             push @$bklist, [$fn, $t];
 642         }
 643     }
 644
 645     return $bklist;
 646 }
 647
 648 sub exec_backup_task {
 649     my ($self, $task) = @_;
 650
 651     my $opts = $self->{opts};
 652
 653     my $vmid = $task->{vmid};
 654     my $plugin = $task->{plugin};
 655     my $vmtype = $plugin->type();
 656
 657     $task->{backup_time} = time();
 658
 659     my $pbs_group_name;
 660     my $pbs_snapshot_name;
 661
 662     if ($self->{opts}->{pbs}) {
 663         if ($vmtype eq 'lxc') {
 664             $pbs_group_name = "ct/$vmid";
 665         } elsif  ($vmtype eq 'qemu') {
 666             $pbs_group_name = "vm/$vmid";
 667         } else {
 668             die "pbs backup not implemented for plugin type '$vmtype'\n";
 669         }
 670         my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
 671         $pbs_snapshot_name = "$pbs_group_name/$btime";
 672     }
 673
 674     my $vmstarttime = time ();
 675
 676     my $logfd;
 677
 678     my $cleanup = {};
 679
 680     my $log_vm_online_again = sub {
 681         return if !defined($task->{vmstoptime});
 682         $task->{vmconttime} //= time();
 683         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 684         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 685     };
 686
 687     eval {
 688         die "unable to find VM '$vmid'\n" if !$plugin;
 689
 690         # for now we deny backups of a running ha managed service in *stop* mode
 691         # as it interferes with the HA stack (started services should not stop).
 692         if ($opts->{mode} eq 'stop' &&
 693             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 694         {
 695             die "Cannot execute a backup with stop mode on a HA managed and".
 696                 " enabled Service. Use snapshot mode or disable the Service.\n";
 697         }
 698
 699         my $tmplog = "$logdir/$vmtype-$vmid.log";
 700
 701         my $bkname = "vzdump-$vmtype-$vmid";
 702         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
 703
 704         my $maxfiles = $opts->{maxfiles};
 705
 706         if ($maxfiles && !$opts->{remove}) {
 707             my $count;
 708             if ($self->{opts}->{pbs}) {
 709                 my $res = PVE::Storage::PBSPlugin::run_client_cmd($opts->{scfg}, $opts->{storage}, 'snapshots', $pbs_group_name);
 710                 $count = scalar(@$res);
 711             } else {
 712                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 713                 $count = scalar(@$bklist);
 714             }
 715             die "There is a max backup limit of ($maxfiles) enforced by the".
 716                 " target storage or the vzdump parameters.".
 717                 " Either increase the limit or delete old backup(s).\n"
 718                 if $count >= $maxfiles;
 719         }
 720
 721         if (!$self->{opts}->{pbs}) {
 722             $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 723         }
 724
 725         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 726         my ($comp, $comp_ext) = compressor_info($opts);
 727         if ($comp && $comp_ext) {
 728             $ext .= ".${comp_ext}";
 729         }
 730
 731         if ($self->{opts}->{pbs}) {
 732             die "unable to pipe backup to stdout\n" if $opts->{stdout};
 733         } else {
 734             if ($opts->{stdout}) {
 735                 $task->{tarfile} = '-';
 736             } else {
 737                 my $tarfile = $task->{tarfile} = "$opts->{dumpdir}/$basename$ext";
 738                 $task->{tmptar} = $task->{tarfile};
 739                 $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 740                 unlink $task->{tmptar};
 741             }
 742         }
 743
 744         $task->{vmtype} = $vmtype;
 745
 746         if ($self->{opts}->{pbs}) {
 747             $task->{tmpdir} = "/var/tmp/vzdumptmp$$"; #fixme
 748         } elsif ($opts->{tmpdir}) {
 749             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp$$";
 750         } else {
 751             # dumpdir is posix? then use it as temporary dir
 752             my $info = get_mount_info($opts->{dumpdir});
 753             if ($vmtype eq 'qemu' ||
 754                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 755                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 756             } else {
 757                 $task->{tmpdir} = "/var/tmp/vzdumptmp$$";
 758                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 759                           "using $task->{tmpdir} for temporary files", $logfd);
 760             }
 761         }
 762
 763         rmtree $task->{tmpdir};
 764         mkdir $task->{tmpdir};
 765         -d $task->{tmpdir} ||
 766             die "unable to create temporary directory '$task->{tmpdir}'";
 767
 768         $logfd = IO::File->new (">$tmplog") ||
 769             die "unable to create log file '$tmplog'";
 770
 771         $task->{dumpdir} = $opts->{dumpdir};
 772         $task->{storeid} = $opts->{storage};
 773         $task->{scfg} = $opts->{scfg};
 774         $task->{tmplog} = $tmplog;
 775
 776         unlink $task->{logfile} if defined($task->{logfile});
 777
 778         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 779         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
 780
 781         $plugin->set_logfd ($logfd);
 782
 783         # test is VM is running
 784         my ($running, $status_text) = $plugin->vm_status ($vmid);
 785
 786         debugmsg ('info', "status = ${status_text}", $logfd);
 787
 788         # lock VM (prevent config changes)
 789         $plugin->lock_vm ($vmid);
 790
 791         $cleanup->{unlock} = 1;
 792
 793         # prepare
 794
 795         my $mode = $running ? $task->{mode} : 'stop';
 796
 797         if ($mode eq 'snapshot') {
 798             my %saved_task = %$task;
 799             eval { $plugin->prepare ($task, $vmid, $mode); };
 800             if (my $err = $@) {
 801                 die $err if $err !~ m/^mode failure/;
 802                 debugmsg ('info',  $err, $logfd);
 803                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 804                 $mode = 'suspend'; # so prepare is called again below
 805                 %$task = %saved_task;
 806             }
 807         }
 808
 809         $cleanup->{prepared} = 1;
 810
 811         $task->{mode} = $mode;
 812
 813         debugmsg ('info', "backup mode: $mode", $logfd);
 814
 815         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
 816             if $opts->{bwlimit};
 817
 818         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 819
 820         if ($mode eq 'stop') {
 821
 822             $plugin->prepare ($task, $vmid, $mode);
 823
 824             $self->run_hook_script ('backup-start', $task, $logfd);
 825
 826             if ($running) {
 827                 debugmsg ('info', "stopping vm", $logfd);
 828                 $task->{vmstoptime} = time();
 829                 $self->run_hook_script ('pre-stop', $task, $logfd);
 830                 $plugin->stop_vm ($task, $vmid);
 831                 $cleanup->{restart} = 1;
 832             }
 833
 834
 835         } elsif ($mode eq 'suspend') {
 836
 837             $plugin->prepare ($task, $vmid, $mode);
 838
 839             $self->run_hook_script ('backup-start', $task, $logfd);
 840
 841             if ($vmtype eq 'lxc') {
 842                 # pre-suspend rsync
 843                 $plugin->copy_data_phase1($task, $vmid);
 844             }
 845
 846             debugmsg ('info', "suspend vm", $logfd);
 847             $task->{vmstoptime} = time ();
 848             $self->run_hook_script ('pre-stop', $task, $logfd);
 849             $plugin->suspend_vm ($task, $vmid);
 850             $cleanup->{resume} = 1;
 851
 852             if ($vmtype eq 'lxc') {
 853                 # post-suspend rsync
 854                 $plugin->copy_data_phase2($task, $vmid);
 855
 856                 debugmsg ('info', "resume vm", $logfd);
 857                 $cleanup->{resume} = 0;
 858                 $self->run_hook_script('pre-restart', $task, $logfd);
 859                 $plugin->resume_vm($task, $vmid);
 860                 $self->run_hook_script('post-restart', $task, $logfd);
 861                 $log_vm_online_again->();
 862             }
 863
 864         } elsif ($mode eq 'snapshot') {
 865
 866             $self->run_hook_script ('backup-start', $task, $logfd);
 867
 868             my $snapshot_count = $task->{snapshot_count} || 0;
 869
 870             $self->run_hook_script ('pre-stop', $task, $logfd);
 871
 872             if ($snapshot_count > 1) {
 873                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 874                 $task->{vmstoptime} = time ();
 875                 $plugin->suspend_vm ($task, $vmid);
 876                 $cleanup->{resume} = 1;
 877             }
 878
 879             $plugin->snapshot ($task, $vmid);
 880
 881             $self->run_hook_script ('pre-restart', $task, $logfd);
 882
 883             if ($snapshot_count > 1) {
 884                 debugmsg ('info', "resume vm", $logfd);
 885                 $cleanup->{resume} = 0;
 886                 $plugin->resume_vm ($task, $vmid);
 887                 $log_vm_online_again->();
 888             }
 889
 890             $self->run_hook_script ('post-restart', $task, $logfd);
 891
 892         } else {
 893             die "internal error - unknown mode '$mode'\n";
 894         }
 895
 896         # assemble archive image
 897         $plugin->assemble ($task, $vmid);
 898
 899         # produce archive
 900
 901         if ($opts->{stdout}) {
 902             debugmsg ('info', "sending archive to stdout", $logfd);
 903             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 904             $self->run_hook_script ('backup-end', $task, $logfd);
 905             return;
 906         }
 907
 908         # fixme: ??
 909         if ($self->{opts}->{pbs}) {
 910             debugmsg ('info', "creating pbs archive on storage '$opts->{storage}'", $logfd);
 911         } else {
 912             debugmsg ('info', "creating archive '$task->{tarfile}'", $logfd);
 913         }
 914         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 915
 916         if ($self->{opts}->{pbs}) {
 917             # fixme: log size ?
 918             debugmsg ('info', "pbs upload finished", $logfd);
 919         } else {
 920             rename ($task->{tmptar}, $task->{tarfile}) ||
 921                 die "unable to rename '$task->{tmptar}' to '$task->{tarfile}'\n";
 922
 923             # determine size
 924             $task->{size} = (-s $task->{tarfile}) || 0;
 925             my $cs = format_size ($task->{size});
 926             debugmsg ('info', "archive file size: $cs", $logfd);
 927         }
 928
 929         # purge older backup
 930         if ($maxfiles && $opts->{remove}) {
 931
 932             if ($self->{opts}->{pbs}) {
 933                 my $args = [$pbs_group_name, '--quiet', '1', '--keep-last', $maxfiles];
 934                 my $logfunc = sub { my $line = shift; debugmsg ('info', $line, $logfd); };
 935                 PVE::Storage::PBSPlugin::run_raw_client_cmd(
 936                     $opts->{scfg}, $opts->{storage}, 'prune', $args, logfunc => $logfunc);
 937             } else {
 938                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{tarfile});
 939                 $bklist = [ sort { $b->[1] <=> $a->[1] } @$bklist ];
 940
 941                 while (scalar (@$bklist) >= $maxfiles) {
 942                     my $d = pop @$bklist;
 943                     debugmsg ('info', "delete old backup '$d->[0]'", $logfd);
 944                     unlink $d->[0];
 945                     my $logfn = $d->[0];
 946                     $logfn =~ s/\.(tgz|((tar|vma)(\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?))$/\.log/;
 947                     unlink $logfn;
 948                 }
 949             }
 950         }
 951
 952         $self->run_hook_script ('backup-end', $task, $logfd);
 953     };
 954     my $err = $@;
 955
 956     if ($plugin) {
 957         # clean-up
 958
 959         if ($cleanup->{unlock}) {
 960             eval { $plugin->unlock_vm ($vmid); };
 961             warn $@ if $@;
 962         }
 963
 964         if ($cleanup->{prepared}) {
 965             # only call cleanup when necessary (when prepare was executed)
 966             eval { $plugin->cleanup ($task, $vmid) };
 967             warn $@ if $@;
 968         }
 969
 970         eval { $plugin->set_logfd (undef); };
 971         warn $@ if $@;
 972
 973         if ($cleanup->{resume} || $cleanup->{restart}) {
 974             eval {
 975                 $self->run_hook_script ('pre-restart', $task, $logfd);
 976                 if ($cleanup->{resume}) {
 977                     debugmsg ('info', "resume vm", $logfd);
 978                     $plugin->resume_vm ($task, $vmid);
 979                 } else {
 980                     my $running = $plugin->vm_status($vmid);
 981                     if (!$running) {
 982                         debugmsg ('info', "restarting vm", $logfd);
 983                         $plugin->start_vm ($task, $vmid);
 984                     }
 985                 }
 986                 $self->run_hook_script ('post-restart', $task, $logfd);
 987             };
 988             my $err = $@;
 989             if ($err) {
 990                 warn $err;
 991             } else {
 992                 $log_vm_online_again->();
 993             }
 994         }
 995     }
 996
 997     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
 998     warn $@ if $@;
 999
1000     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
1001     warn $@ if $@;
1002
1003     my $delay = $task->{backuptime} = time () - $vmstarttime;
1004
1005     if ($err) {
1006         $task->{state} = 'err';
1007         $task->{msg} = $err;
1008         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1009         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
1010
1011         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1012
1013     } else {
1014         $task->{state} = 'ok';
1015         my $tstr = format_time ($delay);
1016         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1017         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1018     }
1019
1020     close ($logfd) if $logfd;
1021
1022     if ($task->{tmplog}) {
1023         if ($self->{opts}->{pbs}) {
1024             if ($task->{state} eq 'ok') {
1025                 my $param = [$pbs_snapshot_name, $task->{tmplog}];
1026                 PVE::Storage::PBSPlugin::run_raw_client_cmd(
1027                     $opts->{scfg}, $opts->{storage}, 'upload-log', $param, errmsg => "upload log failed");
1028             }
1029         } elsif ($task->{logfile}) {
1030             system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1031         }
1032     }
1033
1034     eval { $self->run_hook_script ('log-end', $task); };
1035
1036     die $err if $err && $err =~ m/^interrupted by signal$/;
1037 }
1038
1039 sub exec_backup {
1040     my ($self, $rpcenv, $authuser) = @_;
1041
1042     my $opts = $self->{opts};
1043
1044     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1045     debugmsg ('info', "skip external VMs: " . join(', ', @{$self->{skiplist}}))
1046         if scalar(@{$self->{skiplist}});
1047
1048     my $tasklist = [];
1049     my $vzdump_plugins =  {};
1050     foreach my $plugin (@{$self->{plugins}}) {
1051         my $type = $plugin->type();
1052         next if exists $vzdump_plugins->{$type};
1053         $vzdump_plugins->{$type} = $plugin;
1054     }
1055
1056     my $vmlist = PVE::Cluster::get_vmlist();
1057     foreach my $vmid (sort @{$opts->{vmids}}) {
1058         my $guest_type = $vmlist->{ids}->{$vmid}->{type};
1059         my $plugin = $vzdump_plugins->{$guest_type};
1060         next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all});
1061         push @$tasklist, {
1062             mode => $opts->{mode},
1063             plugin => $plugin,
1064             state => 'todo',
1065             vmid => $vmid,
1066         };
1067     }
1068
1069     # Use in-memory files for the outer hook logs to pass them to sendmail.
1070     my $job_start_log = '';
1071     my $job_end_log = '';
1072     open my $job_start_fd, '>', \$job_start_log;
1073     open my $job_end_fd, '>', \$job_end_log;
1074
1075     my $starttime = time();
1076     my $errcount = 0;
1077     eval {
1078
1079         $self->run_hook_script ('job-start', undef, $job_start_fd);
1080
1081         foreach my $task (@$tasklist) {
1082             $self->exec_backup_task ($task);
1083             $errcount += 1 if $task->{state} ne 'ok';
1084         }
1085
1086         $self->run_hook_script ('job-end', undef, $job_end_fd);
1087     };
1088     my $err = $@;
1089
1090     $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1091
1092     if ($err) {
1093         debugmsg ('err', "Backup job failed - $err", undef, 1);
1094     } else {
1095         if ($errcount) {
1096             debugmsg ('info', "Backup job finished with errors", undef, 1);
1097         } else {
1098             debugmsg ('info', "Backup job finished successfully", undef, 1);
1099         }
1100     }
1101
1102     close $job_start_fd;
1103     close $job_end_fd;
1104
1105     my $totaltime = time() - $starttime;
1106
1107     eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); };
1108     debugmsg ('err', $@) if $@;
1109
1110     die $err if $err;
1111
1112     die "job errors\n" if $errcount;
1113
1114     unlink $pidfile;
1115 }
1116
1117
1118 sub option_exists {
1119     my $key = shift;
1120     return defined($confdesc->{$key});
1121 }
1122
1123 sub verify_vzdump_parameters {
1124     my ($param, $check_missing) = @_;
1125
1126     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1127         if $param->{all} && $param->{vmid};
1128
1129     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1130         if $param->{exclude} && $param->{vmid};
1131
1132     raise_param_exc({ pool => "option conflicts with option 'vmid'"})
1133         if $param->{pool} && $param->{vmid};
1134
1135     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
1136
1137     warn "option 'size' is deprecated and will be removed in a future " .
1138          "release, please update your script/configuration!\n"
1139         if defined($param->{size});
1140
1141     return if !$check_missing;
1142
1143     raise_param_exc({ vmid => "property is missing"})
1144         if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
1145
1146 }
1147
1148 sub stop_running_backups {
1149     my($self) = @_;
1150
1151     my $upid = PVE::Tools::file_read_firstline($pidfile);
1152     return if !$upid;
1153
1154     my $task = PVE::Tools::upid_decode($upid);
1155
1156     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1157         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1158         kill(15, $task->{pid});
1159         # wait max 15 seconds to shut down (else, do nothing for now)
1160         my $i;
1161         for ($i = 15; $i > 0; $i--) {
1162             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1163             sleep (1);
1164         }
1165         die "stopping backup process $task->{pid} failed\n" if $i == 0;
1166     }
1167 }
1168
1169 sub get_included_guests {
1170     my ($job) = @_;
1171
1172     my $nodename = PVE::INotify::nodename();
1173     my $vmids = [];
1174     my $vmids_per_node = {};
1175
1176     my $vmlist = PVE::Cluster::get_vmlist();
1177
1178     if ($job->{pool}) {
1179         $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool});
1180     } elsif ($job->{vmid}) {
1181         $vmids = [ split_list($job->{vmid}) ];
1182     } else {
1183         # all or exclude
1184         my $exclude = check_vmids(split_list($job->{exclude}));
1185         my $excludehash = { map { $_ => 1 } @$exclude };
1186
1187         for my $id (keys %{$vmlist->{ids}}) {
1188             next if $excludehash->{$id};
1189             push @$vmids, $id;
1190         }
1191     }
1192     $vmids = check_vmids(@$vmids);
1193
1194     for my $vmid (@$vmids) {
1195         my $node = $vmlist->{ids}->{$vmid}->{node};
1196         next if (defined $job->{node} && $job->{node} ne $node);
1197
1198         push @{$vmids_per_node->{$node}}, $vmid;
1199     }
1200
1201     return $vmids_per_node;
1202 }
1203
1204 1;