PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Path;
   8 use IO::File;
   9 use IO::Select;
  10 use IPC::Open3;
  11 use POSIX qw(strftime);
  12 use Time::Local;
  13
  14 use PVE::Cluster qw(cfs_read_file);
  15 use PVE::DataCenterConfig;
  16 use PVE::Exception qw(raise_param_exc);
  17 use PVE::HA::Config;
  18 use PVE::HA::Env::PVE2;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::RPCEnvironment;
  21 use PVE::Storage;
  22 use PVE::VZDump::Common;
  23 use PVE::VZDump::Plugin;
  24 use PVE::Tools qw(extract_param split_list);
  25 use PVE::API2Tools;
  26
  27 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  28
  29 my $lockfile = '/var/run/vzdump.lock';
  30 my $pidfile = '/var/run/vzdump.pid';
  31 my $logdir = '/var/log/vzdump';
  32
  33 my @plugins = qw();
  34
  35 my $confdesc = PVE::VZDump::Common::get_confdesc();
  36
  37 # Load available plugins
  38 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
  39 foreach my $plug (@pve_vzdump_classes) {
  40     my $filename = "/usr/share/perl5/$plug.pm";
  41     $filename =~ s!::!/!g;
  42     if (-f $filename) {
  43         eval { require $filename; };
  44         if (!$@) {
  45             $plug->import ();
  46             push @plugins, $plug;
  47         } else {
  48             die $@;
  49         }
  50     }
  51 }
  52
  53 # helper functions
  54
  55 sub debugmsg {
  56     my ($mtype, $msg, $logfd, $syslog) = @_;
  57
  58     PVE::VZDump::Plugin::debugmsg(@_);
  59 }
  60
  61 sub run_command {
  62     my ($logfd, $cmdstr, %param) = @_;
  63
  64     my $logfunc = sub {
  65         my $line = shift;
  66         debugmsg ('info', $line, $logfd);
  67     };
  68
  69     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
  70 }
  71
  72 sub storage_info {
  73     my $storage = shift;
  74
  75     my $cfg = PVE::Storage::config();
  76     my $scfg = PVE::Storage::storage_config($cfg, $storage);
  77     my $type = $scfg->{type};
  78
  79     die "can't use storage type '$type' for backup\n"
  80         if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'
  81               || $type eq 'cifs' || $type eq 'cephfs' || $type eq 'pbs'));
  82     die "can't use storage '$storage' for backups - wrong content type\n"
  83         if (!$scfg->{content}->{backup});
  84
  85     PVE::Storage::activate_storage($cfg, $storage);
  86
  87     my $info = {
  88         scfg => $scfg,
  89         maxfiles => $scfg->{maxfiles},
  90     };
  91
  92     if ($type eq 'pbs') {
  93         $info->{pbs} = 1;
  94     } else {
  95         $info->{dumpdir} = PVE::Storage::get_backup_dir($cfg, $storage);
  96     }
  97
  98     return $info;
  99 }
 100
 101 sub format_size {
 102     my $size = shift;
 103
 104     my $kb = $size / 1024;
 105
 106     if ($kb < 1024) {
 107         return int ($kb) . "KB";
 108     }
 109
 110     my $mb = $size / (1024*1024);
 111     if ($mb < 1024) {
 112         return int ($mb) . "MB";
 113     }
 114     my $gb = $mb / 1024;
 115     if ($gb < 1024) {
 116         return sprintf ("%.2fGB", $gb);
 117     }
 118     my $tb = $gb / 1024;
 119     return sprintf ("%.2fTB", $tb);
 120 }
 121
 122 sub format_time {
 123     my $seconds = shift;
 124
 125     my $hours = int ($seconds/3600);
 126     $seconds = $seconds - $hours*3600;
 127     my $min = int ($seconds/60);
 128     $seconds = $seconds - $min*60;
 129
 130     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 131 }
 132
 133 sub encode8bit {
 134     my ($str) = @_;
 135
 136     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 137
 138     return $str;
 139 }
 140
 141 sub escape_html {
 142     my ($str) = @_;
 143
 144     $str =~ s/&/&amp;/g;
 145     $str =~ s/</&lt;/g;
 146     $str =~ s/>/&gt;/g;
 147
 148     return $str;
 149 }
 150
 151 sub check_bin {
 152     my ($bin)  = @_;
 153
 154     foreach my $p (split (/:/, $ENV{PATH})) {
 155         my $fn = "$p/$bin";
 156         if (-x $fn) {
 157             return $fn;
 158         }
 159     }
 160
 161     die "unable to find command '$bin'\n";
 162 }
 163
 164 sub check_vmids {
 165     my (@vmids) = @_;
 166
 167     my $res = [];
 168     for my $vmid (sort {$a <=> $b} @vmids) {
 169         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 170         $vmid = int ($vmid); # remove leading zeros
 171         next if !$vmid;
 172         push @$res, $vmid;
 173     }
 174
 175     return $res;
 176 }
 177
 178
 179 sub read_vzdump_defaults {
 180
 181     my $fn = "/etc/vzdump.conf";
 182
 183     my $defaults = {
 184         map {
 185             my $default = $confdesc->{$_}->{default};
 186              defined($default) ? ($_ => $default) : ()
 187         } keys %$confdesc
 188     };
 189
 190     my $raw;
 191     eval { $raw = PVE::Tools::file_get_contents($fn); };
 192     return $defaults if $@;
 193
 194     my $conf_schema = { type => 'object', properties => $confdesc, };
 195     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 196     if (my $excludes = $res->{'exclude-path'}) {
 197         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 198     }
 199     if (defined($res->{mailto})) {
 200         my @mailto = split_list($res->{mailto});
 201         $res->{mailto} = [ @mailto ];
 202     }
 203
 204     foreach my $key (keys %$defaults) {
 205         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 206     }
 207
 208     return $res;
 209 }
 210
 211 use constant MAX_MAIL_SIZE => 1024*1024;
 212 sub sendmail {
 213     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 214
 215     my $opts = $self->{opts};
 216
 217     my $mailto = $opts->{mailto};
 218
 219     return if !($mailto && scalar(@$mailto));
 220
 221     my $cmdline = $self->{cmdline};
 222
 223     my $ecount = 0;
 224     foreach my $task (@$tasklist) {
 225         $ecount++ if $task->{state} ne 'ok';
 226         chomp $task->{msg} if $task->{msg};
 227         $task->{backuptime} = 0 if !$task->{backuptime};
 228         $task->{size} = 0 if !$task->{size};
 229         $task->{target} = 'unknown' if !$task->{target};
 230         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 231
 232         if ($task->{state} eq 'todo') {
 233             $task->{msg} = 'aborted';
 234         }
 235     }
 236
 237     my $notify = $opts->{mailnotification} || 'always';
 238     return if (!$ecount && !$err && ($notify eq 'failure'));
 239
 240     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 241     if ($err) {
 242         if ($err =~ /\n/) {
 243             $stat .= ": multiple problems";
 244         } else {
 245             $stat .= ": $err";
 246             $err = undef;
 247         }
 248     }
 249
 250     my $hostname = `hostname -f` || PVE::INotify::nodename();
 251     chomp $hostname;
 252
 253     # text part
 254     my $text = $err ? "$err\n\n" : '';
 255     $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", qw(VMID STATUS TIME SIZE FILENAME));
 256     foreach my $task (@$tasklist) {
 257         my $vmid = $task->{vmid};
 258         if  ($task->{state} eq 'ok') {
 259
 260             $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", $vmid,
 261                                 $task->{state},
 262                                 format_time($task->{backuptime}),
 263                                 format_size ($task->{size}),
 264                                 $task->{target});
 265         } else {
 266             $text .= sprintf ("%-10s %-6s %10s %8.2fMB  %s\n", $vmid,
 267                                 $task->{state},
 268                                 format_time($task->{backuptime}),
 269                                 0, '-');
 270         }
 271     }
 272
 273     my $text_log_part;
 274     $text_log_part .= "\nDetailed backup logs:\n\n";
 275     $text_log_part .= "$cmdline\n\n";
 276
 277     $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
 278     foreach my $task (@$tasklist) {
 279         my $vmid = $task->{vmid};
 280         my $log = $task->{tmplog};
 281         if (!$log) {
 282             $text_log_part .= "$vmid: no log available\n\n";
 283             next;
 284         }
 285         if (open (TMP, "$log")) {
 286             while (my $line = <TMP>) {
 287                 next if $line =~ /^status: \d+/; # not useful in mails
 288                 $text_log_part .= encode8bit ("$vmid: $line");
 289             }
 290         } else {
 291             $text_log_part .= "$vmid: Could not open log file\n\n";
 292         }
 293         close (TMP);
 294         $text_log_part .= "\n";
 295     }
 296     $text_log_part .= $detail_post if defined($detail_post);
 297
 298     # html part
 299     my $html = "<html><body>\n";
 300     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 301     $html .= "<table border=1 cellpadding=3>\n";
 302     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 303
 304     my $ssize = 0;
 305
 306     foreach my $task (@$tasklist) {
 307         my $vmid = $task->{vmid};
 308         my $name = $task->{hostname};
 309
 310         if  ($task->{state} eq 'ok') {
 311
 312             $ssize += $task->{size};
 313
 314             $html .= sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 315                                 $vmid, $name,
 316                                 format_time($task->{backuptime}),
 317                                 format_size ($task->{size}),
 318                                 escape_html ($task->{target}));
 319         } else {
 320             $html .= sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 321                                 $vmid, $name, format_time($task->{backuptime}),
 322                                 escape_html ($task->{msg}));
 323         }
 324     }
 325
 326     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 327  format_time ($totaltime), format_size ($ssize));
 328
 329     $html .= "\n</table><br><br>\n";
 330     my $html_log_part;
 331     $html_log_part .= "Detailed backup logs:<br /><br />\n";
 332     $html_log_part .= "<pre>\n";
 333     $html_log_part .= escape_html($cmdline) . "\n\n";
 334
 335     $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 336     foreach my $task (@$tasklist) {
 337         my $vmid = $task->{vmid};
 338         my $log = $task->{tmplog};
 339         if (!$log) {
 340             $html_log_part .= "$vmid: no log available\n\n";
 341             next;
 342         }
 343         if (open (TMP, "$log")) {
 344             while (my $line = <TMP>) {
 345                 next if $line =~ /^status: \d+/; # not useful in mails
 346                 if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 347                     $html_log_part .= encode8bit ("$vmid: <font color=red>".
 348                         escape_html ($line) . "</font>");
 349                 } else {
 350                     $html_log_part .= encode8bit ("$vmid: " . escape_html ($line));
 351                 }
 352             }
 353         } else {
 354             $html_log_part .= "$vmid: Could not open log file\n\n";
 355         }
 356         close (TMP);
 357         $html_log_part .= "\n";
 358     }
 359     $html_log_part .= escape_html($detail_post) if defined($detail_post);
 360     $html_log_part .= "</pre>";
 361     my $html_end .= "\n</body></html>\n";
 362     # end html part
 363
 364     if (length($text) + length($text_log_part) +
 365         length($html) + length($html_log_part) < MAX_MAIL_SIZE)
 366     {
 367         $html .= $html_log_part;
 368         $text .= $text_log_part;
 369     } else {
 370         my $msg = "Log output was too long to be sent by mail. ".
 371             "See Task History for details!\n";
 372         $text .= $msg;
 373         $html .= "<p>$msg</p>";
 374         $html .= $html_end;
 375     }
 376
 377     my $subject = "vzdump backup status ($hostname) : $stat";
 378
 379     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 380     my $mailfrom = $dcconf->{email_from} || "root";
 381
 382     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 383 };
 384
 385 sub new {
 386     my ($class, $cmdline, $opts, $skiplist) = @_;
 387
 388     mkpath $logdir;
 389
 390     check_bin ('cp');
 391     check_bin ('df');
 392     check_bin ('sendmail');
 393     check_bin ('rsync');
 394     check_bin ('tar');
 395     check_bin ('mount');
 396     check_bin ('umount');
 397     check_bin ('cstream');
 398     check_bin ('ionice');
 399
 400     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 401         check_bin ('lvcreate');
 402         check_bin ('lvs');
 403         check_bin ('lvremove');
 404     }
 405
 406     my $defaults = read_vzdump_defaults();
 407
 408     $opts->{remove} = 1 if !defined($opts->{remove});
 409
 410     foreach my $k (keys %$defaults) {
 411         next if $k eq 'exclude-path' || $k eq 'maxfiles'; # dealt with separately
 412         if ($k eq 'dumpdir' || $k eq 'storage') {
 413             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 414                 !defined ($opts->{storage});
 415         } else {
 416             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 417         }
 418     }
 419
 420     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 421     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 422
 423     $skiplist = [] if !$skiplist;
 424     my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist };
 425
 426     my $findexcl = $self->{findexcl} = [];
 427     if ($defaults->{'exclude-path'}) {
 428         push @$findexcl, @{$defaults->{'exclude-path'}};
 429     }
 430
 431     if ($opts->{'exclude-path'}) {
 432         push @$findexcl, @{$opts->{'exclude-path'}};
 433     }
 434
 435     if ($opts->{stdexcludes}) {
 436         push @$findexcl, '/tmp/?*',
 437                          '/var/tmp/?*',
 438                          '/var/run/?*.pid';
 439     }
 440
 441     foreach my $p (@plugins) {
 442
 443         my $pd = $p->new ($self);
 444
 445         push @{$self->{plugins}}, $pd;
 446     }
 447
 448     if (defined($opts->{storage}) && $opts->{stdout}) {
 449         die "cannot use options 'storage' and 'stdout' at the same time\n";
 450     } elsif (defined($opts->{storage}) && defined($opts->{dumpdir})) {
 451         die "cannot use options 'storage' and 'dumpdir' at the same time\n";
 452     }
 453
 454     if (!$opts->{dumpdir} && !$opts->{storage}) {
 455         $opts->{storage} = 'local';
 456     }
 457
 458     my $errors = '';
 459
 460     if ($opts->{storage}) {
 461         my $info = eval { storage_info ($opts->{storage}) };
 462         $errors .= "could not get storage information for '$opts->{storage}': $@"
 463             if ($@);
 464         $opts->{dumpdir} = $info->{dumpdir};
 465         $opts->{scfg} = $info->{scfg};
 466         $opts->{pbs} = $info->{pbs};
 467         $opts->{maxfiles} //= $info->{maxfiles};
 468     } elsif ($opts->{dumpdir}) {
 469         $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
 470             if ! -d $opts->{dumpdir};
 471     } else {
 472         die "internal error";
 473     }
 474
 475     $opts->{maxfiles} //= $defaults->{maxfiles};
 476
 477     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 478         $errors .= "\n" if $errors;
 479         $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
 480     }
 481
 482     if ($errors) {
 483         eval { $self->sendmail([], 0, $errors); };
 484         debugmsg ('err', $@) if $@;
 485         die "$errors\n";
 486     }
 487
 488     return $self;
 489 }
 490
 491 sub get_mount_info {
 492     my ($dir) = @_;
 493
 494     # Note: df 'available' can be negative, and percentage set to '-'
 495
 496     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 497
 498     my $res;
 499
 500     my $parser = sub {
 501         my $line = shift;
 502         if (my ($fsid, $fstype, undef, $mp) = $line =~
 503             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 504             $res = {
 505                 device => $fsid,
 506                 fstype => $fstype,
 507                 mountpoint => $mp,
 508             };
 509         }
 510     };
 511
 512     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 513     warn $@ if $@;
 514
 515     return $res;
 516 }
 517
 518 sub getlock {
 519     my ($self, $upid) = @_;
 520
 521     my $fh;
 522
 523     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 524
 525     die "missimg UPID" if !$upid; # should not happen
 526
 527     if (!open (SERVER_FLCK, ">>$lockfile")) {
 528         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 529         die "can't open lock on file '$lockfile' - $!";
 530     }
 531
 532     if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 533
 534         if (!$maxwait) {
 535             debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
 536             die "can't acquire lock '$lockfile' (wait = 0)";
 537         }
 538
 539         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 540
 541         eval {
 542             alarm ($maxwait * 60);
 543
 544             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 545
 546             if (!flock (SERVER_FLCK, LOCK_EX)) {
 547                 my $err = $!;
 548                 close (SERVER_FLCK);
 549                 alarm (0);
 550                 die "$err\n";
 551             }
 552             alarm (0);
 553         };
 554         alarm (0);
 555
 556         my $err = $@;
 557
 558         if ($err) {
 559             debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
 560             die "can't acquire lock '$lockfile' - $err";
 561         }
 562
 563         debugmsg('info', "got global lock", undef, 1);
 564     }
 565
 566     PVE::Tools::file_set_contents($pidfile, $upid);
 567 }
 568
 569 sub run_hook_script {
 570     my ($self, $phase, $task, $logfd) = @_;
 571
 572     my $opts = $self->{opts};
 573
 574     my $script = $opts->{script};
 575     return if !$script;
 576
 577     if (!-x $script) {
 578         die "The hook script '$script' is not executable.\n";
 579     }
 580
 581     my $cmd = "$script $phase";
 582
 583     $cmd .= " $task->{mode} $task->{vmid}" if ($task);
 584
 585     local %ENV;
 586     # set immutable opts directly (so they are available in all phases)
 587     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 588     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 589
 590     foreach my $ek (qw(vmtype hostname target logfile)) {
 591         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 592     }
 593
 594     run_command ($logfd, $cmd);
 595 }
 596
 597 sub compressor_info {
 598     my ($opts) = @_;
 599     my $opt_compress = $opts->{compress};
 600
 601     if (!$opt_compress || $opt_compress eq '0') {
 602         return undef;
 603     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 604         return ('lzop', 'lzo');
 605     } elsif ($opt_compress eq 'gzip') {
 606         if ($opts->{pigz} > 0) {
 607             my $pigz_threads = $opts->{pigz};
 608             if ($pigz_threads == 1) {
 609                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 610                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 611             }
 612             return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
 613         } else {
 614             return ('gzip --rsyncable', 'gz');
 615         }
 616     } elsif ($opt_compress eq 'zstd') {
 617         my $zstd_threads = $opts->{zstd} // 1;
 618         if ($zstd_threads == 0) {
 619             my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 620             $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
 621         }
 622         return ("zstd --rsyncable --threads=${zstd_threads}", 'zst');
 623     } else {
 624         die "internal error - unknown compression option '$opt_compress'";
 625     }
 626 }
 627
 628 sub get_backup_file_list {
 629     my ($dir, $bkname, $exclude_fn) = @_;
 630
 631     my $bklist = [];
 632     foreach my $fn (<$dir/${bkname}-*>) {
 633         next if $exclude_fn && $fn eq $exclude_fn;
 634         if ($fn =~ m!/(${bkname}-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?)))$!) {
 635             $fn = "$dir/$1"; # untaint
 636             my $t = timelocal ($7, $6, $5, $4, $3 - 1, $2);
 637             push @$bklist, [$fn, $t];
 638         }
 639     }
 640
 641     return $bklist;
 642 }
 643
 644 sub exec_backup_task {
 645     my ($self, $task) = @_;
 646
 647     my $opts = $self->{opts};
 648
 649     my $vmid = $task->{vmid};
 650     my $plugin = $task->{plugin};
 651     my $vmtype = $plugin->type();
 652
 653     $task->{backup_time} = time();
 654
 655     my $pbs_group_name;
 656     my $pbs_snapshot_name;
 657
 658     if ($self->{opts}->{pbs}) {
 659         if ($vmtype eq 'lxc') {
 660             $pbs_group_name = "ct/$vmid";
 661         } elsif  ($vmtype eq 'qemu') {
 662             $pbs_group_name = "vm/$vmid";
 663         } else {
 664             die "pbs backup not implemented for plugin type '$vmtype'\n";
 665         }
 666         my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
 667         $pbs_snapshot_name = "$pbs_group_name/$btime";
 668     }
 669
 670     my $vmstarttime = time ();
 671
 672     my $logfd;
 673
 674     my $cleanup = {};
 675
 676     my $log_vm_online_again = sub {
 677         return if !defined($task->{vmstoptime});
 678         $task->{vmconttime} //= time();
 679         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 680         $delay = '<1' if $delay < 1;
 681         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 682     };
 683
 684     eval {
 685         die "unable to find VM '$vmid'\n" if !$plugin;
 686
 687         # for now we deny backups of a running ha managed service in *stop* mode
 688         # as it interferes with the HA stack (started services should not stop).
 689         if ($opts->{mode} eq 'stop' &&
 690             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 691         {
 692             die "Cannot execute a backup with stop mode on a HA managed and".
 693                 " enabled Service. Use snapshot mode or disable the Service.\n";
 694         }
 695
 696         my $tmplog = "$logdir/$vmtype-$vmid.log";
 697
 698         my $bkname = "vzdump-$vmtype-$vmid";
 699         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
 700
 701         my $maxfiles = $opts->{maxfiles};
 702
 703         if ($maxfiles && !$opts->{remove}) {
 704             my $count;
 705             if ($self->{opts}->{pbs}) {
 706                 my $res = PVE::Storage::PBSPlugin::run_client_cmd($opts->{scfg}, $opts->{storage}, 'snapshots', $pbs_group_name);
 707                 $count = scalar(@$res);
 708             } else {
 709                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 710                 $count = scalar(@$bklist);
 711             }
 712             die "There is a max backup limit of ($maxfiles) enforced by the".
 713                 " target storage or the vzdump parameters.".
 714                 " Either increase the limit or delete old backup(s).\n"
 715                 if $count >= $maxfiles;
 716         }
 717
 718         if (!$self->{opts}->{pbs}) {
 719             $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 720         }
 721
 722         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 723         my ($comp, $comp_ext) = compressor_info($opts);
 724         if ($comp && $comp_ext) {
 725             $ext .= ".${comp_ext}";
 726         }
 727
 728         if ($self->{opts}->{pbs}) {
 729             die "unable to pipe backup to stdout\n" if $opts->{stdout};
 730             $task->{target} = $pbs_snapshot_name;
 731         } else {
 732             if ($opts->{stdout}) {
 733                 $task->{target} = '-';
 734             } else {
 735                 $task->{target} = $task->{tmptar} = "$opts->{dumpdir}/$basename$ext";
 736                 $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 737                 unlink $task->{tmptar};
 738             }
 739         }
 740
 741         $task->{vmtype} = $vmtype;
 742
 743         if ($self->{opts}->{pbs}) {
 744             $task->{tmpdir} = "/var/tmp/vzdumptmp$$"; #fixme
 745         } elsif ($opts->{tmpdir}) {
 746             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp$$";
 747         } else {
 748             # dumpdir is posix? then use it as temporary dir
 749             my $info = get_mount_info($opts->{dumpdir});
 750             if ($vmtype eq 'qemu' ||
 751                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 752                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 753             } else {
 754                 $task->{tmpdir} = "/var/tmp/vzdumptmp$$";
 755                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 756                           "using $task->{tmpdir} for temporary files", $logfd);
 757             }
 758         }
 759
 760         rmtree $task->{tmpdir};
 761         mkdir $task->{tmpdir};
 762         -d $task->{tmpdir} ||
 763             die "unable to create temporary directory '$task->{tmpdir}'";
 764
 765         $logfd = IO::File->new (">$tmplog") ||
 766             die "unable to create log file '$tmplog'";
 767
 768         $task->{dumpdir} = $opts->{dumpdir};
 769         $task->{storeid} = $opts->{storage};
 770         $task->{scfg} = $opts->{scfg};
 771         $task->{tmplog} = $tmplog;
 772
 773         unlink $task->{logfile} if defined($task->{logfile});
 774
 775         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 776         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
 777
 778         $plugin->set_logfd ($logfd);
 779
 780         # test is VM is running
 781         my ($running, $status_text) = $plugin->vm_status ($vmid);
 782
 783         debugmsg ('info', "status = ${status_text}", $logfd);
 784
 785         # lock VM (prevent config changes)
 786         $plugin->lock_vm ($vmid);
 787
 788         $cleanup->{unlock} = 1;
 789
 790         # prepare
 791
 792         my $mode = $running ? $task->{mode} : 'stop';
 793
 794         if ($mode eq 'snapshot') {
 795             my %saved_task = %$task;
 796             eval { $plugin->prepare ($task, $vmid, $mode); };
 797             if (my $err = $@) {
 798                 die $err if $err !~ m/^mode failure/;
 799                 debugmsg ('info',  $err, $logfd);
 800                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 801                 $mode = 'suspend'; # so prepare is called again below
 802                 %$task = %saved_task;
 803             }
 804         }
 805
 806         $cleanup->{prepared} = 1;
 807
 808         $task->{mode} = $mode;
 809
 810         debugmsg ('info', "backup mode: $mode", $logfd);
 811
 812         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
 813             if $opts->{bwlimit};
 814
 815         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 816
 817         if ($mode eq 'stop') {
 818
 819             $plugin->prepare ($task, $vmid, $mode);
 820
 821             $self->run_hook_script ('backup-start', $task, $logfd);
 822
 823             if ($running) {
 824                 debugmsg ('info', "stopping vm", $logfd);
 825                 $task->{vmstoptime} = time();
 826                 $self->run_hook_script ('pre-stop', $task, $logfd);
 827                 $plugin->stop_vm ($task, $vmid);
 828                 $cleanup->{restart} = 1;
 829             }
 830
 831
 832         } elsif ($mode eq 'suspend') {
 833
 834             $plugin->prepare ($task, $vmid, $mode);
 835
 836             $self->run_hook_script ('backup-start', $task, $logfd);
 837
 838             if ($vmtype eq 'lxc') {
 839                 # pre-suspend rsync
 840                 $plugin->copy_data_phase1($task, $vmid);
 841             }
 842
 843             debugmsg ('info', "suspending guest", $logfd);
 844             $task->{vmstoptime} = time ();
 845             $self->run_hook_script ('pre-stop', $task, $logfd);
 846             $plugin->suspend_vm ($task, $vmid);
 847             $cleanup->{resume} = 1;
 848
 849             if ($vmtype eq 'lxc') {
 850                 # post-suspend rsync
 851                 $plugin->copy_data_phase2($task, $vmid);
 852
 853                 debugmsg ('info', "resuming guest", $logfd);
 854                 $cleanup->{resume} = 0;
 855                 $self->run_hook_script('pre-restart', $task, $logfd);
 856                 $plugin->resume_vm($task, $vmid);
 857                 $self->run_hook_script('post-restart', $task, $logfd);
 858                 $log_vm_online_again->();
 859             }
 860
 861         } elsif ($mode eq 'snapshot') {
 862
 863             $self->run_hook_script ('backup-start', $task, $logfd);
 864
 865             my $snapshot_count = $task->{snapshot_count} || 0;
 866
 867             $self->run_hook_script ('pre-stop', $task, $logfd);
 868
 869             if ($snapshot_count > 1) {
 870                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 871                 $task->{vmstoptime} = time ();
 872                 $plugin->suspend_vm ($task, $vmid);
 873                 $cleanup->{resume} = 1;
 874             }
 875
 876             $plugin->snapshot ($task, $vmid);
 877
 878             $self->run_hook_script ('pre-restart', $task, $logfd);
 879
 880             if ($snapshot_count > 1) {
 881                 debugmsg ('info', "resume vm", $logfd);
 882                 $cleanup->{resume} = 0;
 883                 $plugin->resume_vm ($task, $vmid);
 884                 $log_vm_online_again->();
 885             }
 886
 887             $self->run_hook_script ('post-restart', $task, $logfd);
 888
 889         } else {
 890             die "internal error - unknown mode '$mode'\n";
 891         }
 892
 893         # assemble archive image
 894         $plugin->assemble ($task, $vmid);
 895
 896         # produce archive
 897
 898         if ($opts->{stdout}) {
 899             debugmsg ('info', "sending archive to stdout", $logfd);
 900             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 901             $self->run_hook_script ('backup-end', $task, $logfd);
 902             return;
 903         }
 904
 905         my $archive_txt = $self->{opts}->{pbs} ? 'Proxmox Backup Server' : 'vzdump';
 906         debugmsg('info', "creating $archive_txt archive '$task->{target}'", $logfd);
 907         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 908
 909         if ($self->{opts}->{pbs}) {
 910             # size is added to task struct in guest vzdump plugins
 911         } else {
 912             rename ($task->{tmptar}, $task->{target}) ||
 913                 die "unable to rename '$task->{tmptar}' to '$task->{target}'\n";
 914
 915             # determine size
 916             $task->{size} = (-s $task->{target}) || 0;
 917             my $cs = format_size ($task->{size});
 918             debugmsg ('info', "archive file size: $cs", $logfd);
 919         }
 920
 921         # purge older backup
 922         if ($maxfiles && $opts->{remove}) {
 923
 924             if ($self->{opts}->{pbs}) {
 925                 my $args = [$pbs_group_name, '--quiet', '1', '--keep-last', $maxfiles];
 926                 my $logfunc = sub { my $line = shift; debugmsg ('info', $line, $logfd); };
 927                 PVE::Storage::PBSPlugin::run_raw_client_cmd(
 928                     $opts->{scfg}, $opts->{storage}, 'prune', $args, logfunc => $logfunc);
 929             } else {
 930                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{target});
 931                 $bklist = [ sort { $b->[1] <=> $a->[1] } @$bklist ];
 932
 933                 while (scalar (@$bklist) >= $maxfiles) {
 934                     my $d = pop @$bklist;
 935                     debugmsg ('info', "delete old backup '$d->[0]'", $logfd);
 936                     unlink $d->[0];
 937                     my $logfn = $d->[0];
 938                     $logfn =~ s/\.(tgz|((tar|vma)(\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?))$/\.log/;
 939                     unlink $logfn;
 940                 }
 941             }
 942         }
 943
 944         $self->run_hook_script ('backup-end', $task, $logfd);
 945     };
 946     my $err = $@;
 947
 948     if ($plugin) {
 949         # clean-up
 950
 951         if ($cleanup->{unlock}) {
 952             eval { $plugin->unlock_vm ($vmid); };
 953             warn $@ if $@;
 954         }
 955
 956         if ($cleanup->{prepared}) {
 957             # only call cleanup when necessary (when prepare was executed)
 958             eval { $plugin->cleanup ($task, $vmid) };
 959             warn $@ if $@;
 960         }
 961
 962         eval { $plugin->set_logfd (undef); };
 963         warn $@ if $@;
 964
 965         if ($cleanup->{resume} || $cleanup->{restart}) {
 966             eval {
 967                 $self->run_hook_script ('pre-restart', $task, $logfd);
 968                 if ($cleanup->{resume}) {
 969                     debugmsg ('info', "resume vm", $logfd);
 970                     $plugin->resume_vm ($task, $vmid);
 971                 } else {
 972                     my $running = $plugin->vm_status($vmid);
 973                     if (!$running) {
 974                         debugmsg ('info', "restarting vm", $logfd);
 975                         $plugin->start_vm ($task, $vmid);
 976                     }
 977                 }
 978                 $self->run_hook_script ('post-restart', $task, $logfd);
 979             };
 980             my $err = $@;
 981             if ($err) {
 982                 warn $err;
 983             } else {
 984                 $log_vm_online_again->();
 985             }
 986         }
 987     }
 988
 989     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
 990     warn $@ if $@;
 991
 992     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
 993     warn $@ if $@;
 994
 995     my $delay = $task->{backuptime} = time () - $vmstarttime;
 996
 997     if ($err) {
 998         $task->{state} = 'err';
 999         $task->{msg} = $err;
1000         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1001         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
1002
1003         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1004
1005     } else {
1006         $task->{state} = 'ok';
1007         my $tstr = format_time ($delay);
1008         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1009         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1010     }
1011
1012     close ($logfd) if $logfd;
1013
1014     if ($task->{tmplog}) {
1015         if ($self->{opts}->{pbs}) {
1016             if ($task->{state} eq 'ok') {
1017                 my $param = [$pbs_snapshot_name, $task->{tmplog}];
1018                 PVE::Storage::PBSPlugin::run_raw_client_cmd(
1019                     $opts->{scfg}, $opts->{storage}, 'upload-log', $param, errmsg => "upload log failed");
1020             }
1021         } elsif ($task->{logfile}) {
1022             system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1023         }
1024     }
1025
1026     eval { $self->run_hook_script ('log-end', $task); };
1027
1028     die $err if $err && $err =~ m/^interrupted by signal$/;
1029 }
1030
1031 sub exec_backup {
1032     my ($self, $rpcenv, $authuser) = @_;
1033
1034     my $opts = $self->{opts};
1035
1036     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1037     debugmsg ('info', "skip external VMs: " . join(', ', @{$self->{skiplist}}))
1038         if scalar(@{$self->{skiplist}});
1039
1040     my $tasklist = [];
1041     my $vzdump_plugins =  {};
1042     foreach my $plugin (@{$self->{plugins}}) {
1043         my $type = $plugin->type();
1044         next if exists $vzdump_plugins->{$type};
1045         $vzdump_plugins->{$type} = $plugin;
1046     }
1047
1048     my $vmlist = PVE::Cluster::get_vmlist();
1049     foreach my $vmid (sort @{$opts->{vmids}}) {
1050         my $guest_type = $vmlist->{ids}->{$vmid}->{type};
1051         my $plugin = $vzdump_plugins->{$guest_type};
1052         next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all});
1053         push @$tasklist, {
1054             mode => $opts->{mode},
1055             plugin => $plugin,
1056             state => 'todo',
1057             vmid => $vmid,
1058         };
1059     }
1060
1061     # Use in-memory files for the outer hook logs to pass them to sendmail.
1062     my $job_start_log = '';
1063     my $job_end_log = '';
1064     open my $job_start_fd, '>', \$job_start_log;
1065     open my $job_end_fd, '>', \$job_end_log;
1066
1067     my $starttime = time();
1068     my $errcount = 0;
1069     eval {
1070
1071         $self->run_hook_script ('job-start', undef, $job_start_fd);
1072
1073         foreach my $task (@$tasklist) {
1074             $self->exec_backup_task ($task);
1075             $errcount += 1 if $task->{state} ne 'ok';
1076         }
1077
1078         $self->run_hook_script ('job-end', undef, $job_end_fd);
1079     };
1080     my $err = $@;
1081
1082     $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1083
1084     if ($err) {
1085         debugmsg ('err', "Backup job failed - $err", undef, 1);
1086     } else {
1087         if ($errcount) {
1088             debugmsg ('info', "Backup job finished with errors", undef, 1);
1089         } else {
1090             debugmsg ('info', "Backup job finished successfully", undef, 1);
1091         }
1092     }
1093
1094     close $job_start_fd;
1095     close $job_end_fd;
1096
1097     my $totaltime = time() - $starttime;
1098
1099     eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); };
1100     debugmsg ('err', $@) if $@;
1101
1102     die $err if $err;
1103
1104     die "job errors\n" if $errcount;
1105
1106     unlink $pidfile;
1107 }
1108
1109
1110 sub option_exists {
1111     my $key = shift;
1112     return defined($confdesc->{$key});
1113 }
1114
1115 sub verify_vzdump_parameters {
1116     my ($param, $check_missing) = @_;
1117
1118     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1119         if $param->{all} && $param->{vmid};
1120
1121     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1122         if $param->{exclude} && $param->{vmid};
1123
1124     raise_param_exc({ pool => "option conflicts with option 'vmid'"})
1125         if $param->{pool} && $param->{vmid};
1126
1127     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
1128
1129     warn "option 'size' is deprecated and will be removed in a future " .
1130          "release, please update your script/configuration!\n"
1131         if defined($param->{size});
1132
1133     return if !$check_missing;
1134
1135     raise_param_exc({ vmid => "property is missing"})
1136         if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
1137
1138 }
1139
1140 sub stop_running_backups {
1141     my($self) = @_;
1142
1143     my $upid = PVE::Tools::file_read_firstline($pidfile);
1144     return if !$upid;
1145
1146     my $task = PVE::Tools::upid_decode($upid);
1147
1148     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1149         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1150         kill(15, $task->{pid});
1151         # wait max 15 seconds to shut down (else, do nothing for now)
1152         my $i;
1153         for ($i = 15; $i > 0; $i--) {
1154             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1155             sleep (1);
1156         }
1157         die "stopping backup process $task->{pid} failed\n" if $i == 0;
1158     }
1159 }
1160
1161 sub get_included_guests {
1162     my ($job) = @_;
1163
1164     my $nodename = PVE::INotify::nodename();
1165     my $vmids = [];
1166     my $vmids_per_node = {};
1167
1168     my $vmlist = PVE::Cluster::get_vmlist();
1169
1170     if ($job->{pool}) {
1171         $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool});
1172     } elsif ($job->{vmid}) {
1173         $vmids = [ split_list($job->{vmid}) ];
1174     } elsif ($job->{all}) {
1175         # all or exclude
1176         my $exclude = check_vmids(split_list($job->{exclude}));
1177         my $excludehash = { map { $_ => 1 } @$exclude };
1178
1179         for my $id (keys %{$vmlist->{ids}}) {
1180             next if $excludehash->{$id};
1181             push @$vmids, $id;
1182         }
1183     } else {
1184         return $vmids_per_node;
1185     }
1186     $vmids = check_vmids(@$vmids);
1187
1188     for my $vmid (@$vmids) {
1189         my $node = $vmlist->{ids}->{$vmid}->{node};
1190         next if (defined $job->{node} && $job->{node} ne $node);
1191
1192         push @{$vmids_per_node->{$node}}, $vmid;
1193     }
1194
1195     return $vmids_per_node;
1196 }
1197
1198 1;