PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Path;
   8 use IO::File;
   9 use IO::Select;
  10 use IPC::Open3;
  11 use POSIX qw(strftime);
  12 use Time::Local;
  13
  14 use PVE::Cluster qw(cfs_read_file);
  15 use PVE::DataCenterConfig;
  16 use PVE::Exception qw(raise_param_exc);
  17 use PVE::HA::Config;
  18 use PVE::HA::Env::PVE2;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::RPCEnvironment;
  21 use PVE::Storage;
  22 use PVE::VZDump::Common;
  23 use PVE::VZDump::Plugin;
  24 use PVE::Tools qw(extract_param split_list);
  25 use PVE::API2Tools;
  26
  27 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  28
  29 my $lockfile = '/var/run/vzdump.lock';
  30 my $pidfile = '/var/run/vzdump.pid';
  31 my $logdir = '/var/log/vzdump';
  32
  33 my @plugins = qw();
  34
  35 my $confdesc = PVE::VZDump::Common::get_confdesc();
  36
  37 # Load available plugins
  38 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
  39 foreach my $plug (@pve_vzdump_classes) {
  40     my $filename = "/usr/share/perl5/$plug.pm";
  41     $filename =~ s!::!/!g;
  42     if (-f $filename) {
  43         eval { require $filename; };
  44         if (!$@) {
  45             $plug->import ();
  46             push @plugins, $plug;
  47         } else {
  48             die $@;
  49         }
  50     }
  51 }
  52
  53 # helper functions
  54
  55 sub debugmsg {
  56     my ($mtype, $msg, $logfd, $syslog) = @_;
  57
  58     PVE::VZDump::Plugin::debugmsg(@_);
  59 }
  60
  61 sub run_command {
  62     my ($logfd, $cmdstr, %param) = @_;
  63
  64     my $logfunc = sub {
  65         my $line = shift;
  66         debugmsg ('info', $line, $logfd);
  67     };
  68
  69     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
  70 }
  71
  72 sub storage_info {
  73     my $storage = shift;
  74
  75     my $cfg = PVE::Storage::config();
  76     my $scfg = PVE::Storage::storage_config($cfg, $storage);
  77     my $type = $scfg->{type};
  78
  79     die "can't use storage type '$type' for backup\n"
  80         if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'
  81               || $type eq 'cifs' || $type eq 'cephfs' || $type eq 'pbs'));
  82     die "can't use storage '$storage' for backups - wrong content type\n"
  83         if (!$scfg->{content}->{backup});
  84
  85     PVE::Storage::activate_storage($cfg, $storage);
  86
  87     my $info = {
  88         scfg => $scfg,
  89         maxfiles => $scfg->{maxfiles},
  90     };
  91
  92     $info->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'})
  93         if defined($scfg->{'prune-backups'});
  94
  95     die "cannot have 'maxfiles' and 'prune-backups' configured at the same time\n"
  96         if defined($info->{'prune-backups'}) && defined($info->{maxfiles});
  97
  98     if ($type eq 'pbs') {
  99         $info->{pbs} = 1;
 100     } else {
 101         $info->{dumpdir} = PVE::Storage::get_backup_dir($cfg, $storage);
 102     }
 103
 104     return $info;
 105 }
 106
 107 sub format_size {
 108     my $size = shift;
 109
 110     my $kb = $size / 1024;
 111
 112     if ($kb < 1024) {
 113         return int ($kb) . "KB";
 114     }
 115
 116     my $mb = $size / (1024*1024);
 117     if ($mb < 1024) {
 118         return int ($mb) . "MB";
 119     }
 120     my $gb = $mb / 1024;
 121     if ($gb < 1024) {
 122         return sprintf ("%.2fGB", $gb);
 123     }
 124     my $tb = $gb / 1024;
 125     return sprintf ("%.2fTB", $tb);
 126 }
 127
 128 sub format_time {
 129     my $seconds = shift;
 130
 131     my $hours = int ($seconds/3600);
 132     $seconds = $seconds - $hours*3600;
 133     my $min = int ($seconds/60);
 134     $seconds = $seconds - $min*60;
 135
 136     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 137 }
 138
 139 sub encode8bit {
 140     my ($str) = @_;
 141
 142     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 143
 144     return $str;
 145 }
 146
 147 sub escape_html {
 148     my ($str) = @_;
 149
 150     $str =~ s/&/&amp;/g;
 151     $str =~ s/</&lt;/g;
 152     $str =~ s/>/&gt;/g;
 153
 154     return $str;
 155 }
 156
 157 sub check_bin {
 158     my ($bin)  = @_;
 159
 160     foreach my $p (split (/:/, $ENV{PATH})) {
 161         my $fn = "$p/$bin";
 162         if (-x $fn) {
 163             return $fn;
 164         }
 165     }
 166
 167     die "unable to find command '$bin'\n";
 168 }
 169
 170 sub check_vmids {
 171     my (@vmids) = @_;
 172
 173     my $res = [];
 174     for my $vmid (sort {$a <=> $b} @vmids) {
 175         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 176         $vmid = int ($vmid); # remove leading zeros
 177         next if !$vmid;
 178         push @$res, $vmid;
 179     }
 180
 181     return $res;
 182 }
 183
 184
 185 sub read_vzdump_defaults {
 186
 187     my $fn = "/etc/vzdump.conf";
 188
 189     my $defaults = {
 190         map {
 191             my $default = $confdesc->{$_}->{default};
 192              defined($default) ? ($_ => $default) : ()
 193         } keys %$confdesc
 194     };
 195
 196     my $raw;
 197     eval { $raw = PVE::Tools::file_get_contents($fn); };
 198     return $defaults if $@;
 199
 200     my $conf_schema = { type => 'object', properties => $confdesc, };
 201     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 202     if (my $excludes = $res->{'exclude-path'}) {
 203         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 204     }
 205     if (defined($res->{mailto})) {
 206         my @mailto = split_list($res->{mailto});
 207         $res->{mailto} = [ @mailto ];
 208     }
 209
 210     foreach my $key (keys %$defaults) {
 211         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 212     }
 213
 214     return $res;
 215 }
 216
 217 use constant MAX_MAIL_SIZE => 1024*1024;
 218 sub sendmail {
 219     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 220
 221     my $opts = $self->{opts};
 222
 223     my $mailto = $opts->{mailto};
 224
 225     return if !($mailto && scalar(@$mailto));
 226
 227     my $cmdline = $self->{cmdline};
 228
 229     my $ecount = 0;
 230     foreach my $task (@$tasklist) {
 231         $ecount++ if $task->{state} ne 'ok';
 232         chomp $task->{msg} if $task->{msg};
 233         $task->{backuptime} = 0 if !$task->{backuptime};
 234         $task->{size} = 0 if !$task->{size};
 235         $task->{target} = 'unknown' if !$task->{target};
 236         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 237
 238         if ($task->{state} eq 'todo') {
 239             $task->{msg} = 'aborted';
 240         }
 241     }
 242
 243     my $notify = $opts->{mailnotification} || 'always';
 244     return if (!$ecount && !$err && ($notify eq 'failure'));
 245
 246     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 247     if ($err) {
 248         if ($err =~ /\n/) {
 249             $stat .= ": multiple problems";
 250         } else {
 251             $stat .= ": $err";
 252             $err = undef;
 253         }
 254     }
 255
 256     my $hostname = `hostname -f` || PVE::INotify::nodename();
 257     chomp $hostname;
 258
 259     # text part
 260     my $text = $err ? "$err\n\n" : '';
 261     my $namelength = 20;
 262     $text .= sprintf (
 263         "%-10s %-${namelength}s %-6s %10s %10s  %s\n",
 264         qw(VMID NAME STATUS TIME SIZE FILENAME)
 265     );
 266     foreach my $task (@$tasklist) {
 267         my $name = substr($task->{hostname}, 0, $namelength);
 268         my $successful = $task->{state} eq 'ok';
 269         my $size = $successful ? format_size ($task->{size}) : 0;
 270         my $filename = $successful ? $task->{target} : '-';
 271         my $size_fmt = $successful ? "%10s": "%8.2fMB";
 272         $text .= sprintf(
 273             "%-10s %-${namelength}s %-6s %10s $size_fmt  %s\n",
 274             $task->{vmid},
 275             $name,
 276             $task->{state},
 277             format_time($task->{backuptime}),
 278             $size,
 279             $filename,
 280         );
 281     }
 282
 283     my $text_log_part;
 284     $text_log_part .= "\nDetailed backup logs:\n\n";
 285     $text_log_part .= "$cmdline\n\n";
 286
 287     $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
 288     foreach my $task (@$tasklist) {
 289         my $vmid = $task->{vmid};
 290         my $log = $task->{tmplog};
 291         if (!$log) {
 292             $text_log_part .= "$vmid: no log available\n\n";
 293             next;
 294         }
 295         if (open (TMP, "$log")) {
 296             while (my $line = <TMP>) {
 297                 next if $line =~ /^status: \d+/; # not useful in mails
 298                 $text_log_part .= encode8bit ("$vmid: $line");
 299             }
 300         } else {
 301             $text_log_part .= "$vmid: Could not open log file\n\n";
 302         }
 303         close (TMP);
 304         $text_log_part .= "\n";
 305     }
 306     $text_log_part .= $detail_post if defined($detail_post);
 307
 308     # html part
 309     my $html = "<html><body>\n";
 310     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 311     $html .= "<table border=1 cellpadding=3>\n";
 312     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 313
 314     my $ssize = 0;
 315
 316     foreach my $task (@$tasklist) {
 317         my $vmid = $task->{vmid};
 318         my $name = $task->{hostname};
 319
 320         if  ($task->{state} eq 'ok') {
 321
 322             $ssize += $task->{size};
 323
 324             $html .= sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 325                                 $vmid, $name,
 326                                 format_time($task->{backuptime}),
 327                                 format_size ($task->{size}),
 328                                 escape_html ($task->{target}));
 329         } else {
 330             $html .= sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 331                                 $vmid, $name, format_time($task->{backuptime}),
 332                                 escape_html ($task->{msg}));
 333         }
 334     }
 335
 336     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 337  format_time ($totaltime), format_size ($ssize));
 338
 339     $html .= "\n</table><br><br>\n";
 340     my $html_log_part;
 341     $html_log_part .= "Detailed backup logs:<br /><br />\n";
 342     $html_log_part .= "<pre>\n";
 343     $html_log_part .= escape_html($cmdline) . "\n\n";
 344
 345     $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 346     foreach my $task (@$tasklist) {
 347         my $vmid = $task->{vmid};
 348         my $log = $task->{tmplog};
 349         if (!$log) {
 350             $html_log_part .= "$vmid: no log available\n\n";
 351             next;
 352         }
 353         if (open (TMP, "$log")) {
 354             while (my $line = <TMP>) {
 355                 next if $line =~ /^status: \d+/; # not useful in mails
 356                 if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 357                     $html_log_part .= encode8bit ("$vmid: <font color=red>".
 358                         escape_html ($line) . "</font>");
 359                 } else {
 360                     $html_log_part .= encode8bit ("$vmid: " . escape_html ($line));
 361                 }
 362             }
 363         } else {
 364             $html_log_part .= "$vmid: Could not open log file\n\n";
 365         }
 366         close (TMP);
 367         $html_log_part .= "\n";
 368     }
 369     $html_log_part .= escape_html($detail_post) if defined($detail_post);
 370     $html_log_part .= "</pre>";
 371     my $html_end .= "\n</body></html>\n";
 372     # end html part
 373
 374     if (length($text) + length($text_log_part) +
 375         length($html) + length($html_log_part) < MAX_MAIL_SIZE)
 376     {
 377         $html .= $html_log_part;
 378         $text .= $text_log_part;
 379     } else {
 380         my $msg = "Log output was too long to be sent by mail. ".
 381             "See Task History for details!\n";
 382         $text .= $msg;
 383         $html .= "<p>$msg</p>";
 384         $html .= $html_end;
 385     }
 386
 387     my $subject = "vzdump backup status ($hostname) : $stat";
 388
 389     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 390     my $mailfrom = $dcconf->{email_from} || "root";
 391
 392     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 393 };
 394
 395 sub new {
 396     my ($class, $cmdline, $opts, $skiplist) = @_;
 397
 398     mkpath $logdir;
 399
 400     check_bin ('cp');
 401     check_bin ('df');
 402     check_bin ('sendmail');
 403     check_bin ('rsync');
 404     check_bin ('tar');
 405     check_bin ('mount');
 406     check_bin ('umount');
 407     check_bin ('cstream');
 408     check_bin ('ionice');
 409
 410     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 411         check_bin ('lvcreate');
 412         check_bin ('lvs');
 413         check_bin ('lvremove');
 414     }
 415
 416     my $defaults = read_vzdump_defaults();
 417
 418     $opts->{remove} = 1 if !defined($opts->{remove});
 419
 420     foreach my $k (keys %$defaults) {
 421         next if $k eq 'exclude-path' || $k eq 'maxfiles'; # dealt with separately
 422         if ($k eq 'dumpdir' || $k eq 'storage') {
 423             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 424                 !defined ($opts->{storage});
 425         } else {
 426             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 427         }
 428     }
 429
 430     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 431     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 432
 433     $skiplist = [] if !$skiplist;
 434     my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist };
 435
 436     my $findexcl = $self->{findexcl} = [];
 437     if ($defaults->{'exclude-path'}) {
 438         push @$findexcl, @{$defaults->{'exclude-path'}};
 439     }
 440
 441     if ($opts->{'exclude-path'}) {
 442         push @$findexcl, @{$opts->{'exclude-path'}};
 443     }
 444
 445     if ($opts->{stdexcludes}) {
 446         push @$findexcl, '/tmp/?*',
 447                          '/var/tmp/?*',
 448                          '/var/run/?*.pid';
 449     }
 450
 451     foreach my $p (@plugins) {
 452
 453         my $pd = $p->new ($self);
 454
 455         push @{$self->{plugins}}, $pd;
 456     }
 457
 458     if (defined($opts->{storage}) && $opts->{stdout}) {
 459         die "cannot use options 'storage' and 'stdout' at the same time\n";
 460     } elsif (defined($opts->{storage}) && defined($opts->{dumpdir})) {
 461         die "cannot use options 'storage' and 'dumpdir' at the same time\n";
 462     }
 463
 464     if (!$opts->{dumpdir} && !$opts->{storage}) {
 465         $opts->{storage} = 'local';
 466     }
 467
 468     my $errors = '';
 469
 470     if ($opts->{storage}) {
 471         my $info = eval { storage_info ($opts->{storage}) };
 472         if (my $err = $@) {
 473             $errors .= "could not get storage information for '$opts->{storage}': $err";
 474         } else {
 475             $opts->{dumpdir} = $info->{dumpdir};
 476             $opts->{scfg} = $info->{scfg};
 477             $opts->{pbs} = $info->{pbs};
 478
 479             if (!defined($opts->{'prune-backups'}) && !defined($opts->{maxfiles})) {
 480                 $opts->{'prune-backups'} = $info->{'prune-backups'};
 481                 $opts->{maxfiles} = $info->{maxfiles};
 482             }
 483         }
 484     } elsif ($opts->{dumpdir}) {
 485         $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
 486             if ! -d $opts->{dumpdir};
 487     } else {
 488         die "internal error";
 489     }
 490
 491     if (!defined($opts->{'prune-backups'})) {
 492         my $maxfiles = delete $opts->{maxfiles} // $defaults->{maxfiles};
 493         $maxfiles = int($maxfiles); # shouldn't be necessary, but be safe
 494         if ($maxfiles) {
 495             $opts->{'prune-backups'} = { 'keep-last' => $maxfiles };
 496         } else {
 497             # maxfiles being zero means keep all, so avoid triggering any remove code path to be safe
 498             $opts->{remove} = 0;
 499         }
 500     }
 501
 502     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 503         $errors .= "\n" if $errors;
 504         $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
 505     }
 506
 507     if ($errors) {
 508         eval { $self->sendmail([], 0, $errors); };
 509         debugmsg ('err', $@) if $@;
 510         die "$errors\n";
 511     }
 512
 513     return $self;
 514 }
 515
 516 sub get_mount_info {
 517     my ($dir) = @_;
 518
 519     # Note: df 'available' can be negative, and percentage set to '-'
 520
 521     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 522
 523     my $res;
 524
 525     my $parser = sub {
 526         my $line = shift;
 527         if (my ($fsid, $fstype, undef, $mp) = $line =~
 528             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 529             $res = {
 530                 device => $fsid,
 531                 fstype => $fstype,
 532                 mountpoint => $mp,
 533             };
 534         }
 535     };
 536
 537     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 538     warn $@ if $@;
 539
 540     return $res;
 541 }
 542
 543 sub getlock {
 544     my ($self, $upid) = @_;
 545
 546     my $fh;
 547
 548     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 549
 550     die "missing UPID" if !$upid; # should not happen
 551
 552     if (!open (SERVER_FLCK, ">>$lockfile")) {
 553         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 554         die "can't open lock on file '$lockfile' - $!";
 555     }
 556
 557     if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 558
 559         if (!$maxwait) {
 560             debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
 561             die "can't acquire lock '$lockfile' (wait = 0)";
 562         }
 563
 564         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 565
 566         eval {
 567             alarm ($maxwait * 60);
 568
 569             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 570
 571             if (!flock (SERVER_FLCK, LOCK_EX)) {
 572                 my $err = $!;
 573                 close (SERVER_FLCK);
 574                 alarm (0);
 575                 die "$err\n";
 576             }
 577             alarm (0);
 578         };
 579         alarm (0);
 580
 581         my $err = $@;
 582
 583         if ($err) {
 584             debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
 585             die "can't acquire lock '$lockfile' - $err";
 586         }
 587
 588         debugmsg('info', "got global lock", undef, 1);
 589     }
 590
 591     PVE::Tools::file_set_contents($pidfile, $upid);
 592 }
 593
 594 sub run_hook_script {
 595     my ($self, $phase, $task, $logfd) = @_;
 596
 597     my $opts = $self->{opts};
 598
 599     my $script = $opts->{script};
 600     return if !$script;
 601
 602     if (!-x $script) {
 603         die "The hook script '$script' is not executable.\n";
 604     }
 605
 606     my $cmd = "$script $phase";
 607
 608     $cmd .= " $task->{mode} $task->{vmid}" if ($task);
 609
 610     local %ENV;
 611     # set immutable opts directly (so they are available in all phases)
 612     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 613     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 614
 615     foreach my $ek (qw(vmtype hostname target logfile)) {
 616         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 617     }
 618     # FIXME: for backwards compatibility - drop with PVE 7.0
 619     $ENV{TARFILE} = $task->{target} if $task->{target};
 620
 621     run_command ($logfd, $cmd);
 622 }
 623
 624 sub compressor_info {
 625     my ($opts) = @_;
 626     my $opt_compress = $opts->{compress};
 627
 628     if (!$opt_compress || $opt_compress eq '0') {
 629         return undef;
 630     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 631         return ('lzop', 'lzo');
 632     } elsif ($opt_compress eq 'gzip') {
 633         if ($opts->{pigz} > 0) {
 634             my $pigz_threads = $opts->{pigz};
 635             if ($pigz_threads == 1) {
 636                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 637                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 638             }
 639             return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
 640         } else {
 641             return ('gzip --rsyncable', 'gz');
 642         }
 643     } elsif ($opt_compress eq 'zstd') {
 644         my $zstd_threads = $opts->{zstd} // 1;
 645         if ($zstd_threads == 0) {
 646             my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 647             $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
 648         }
 649         return ("zstd --rsyncable --threads=${zstd_threads}", 'zst');
 650     } else {
 651         die "internal error - unknown compression option '$opt_compress'";
 652     }
 653 }
 654
 655 sub get_backup_file_list {
 656     my ($dir, $bkname, $exclude_fn) = @_;
 657
 658     my $bklist = [];
 659     foreach my $fn (<$dir/${bkname}-*>) {
 660         next if $exclude_fn && $fn eq $exclude_fn;
 661
 662         my $archive_info = eval { PVE::Storage::archive_info($fn) } // {};
 663         if ($archive_info->{is_std_name}) {
 664             my $filename = $archive_info->{filename};
 665             my $backup = {
 666                 'path' => "$dir/$filename",
 667                 'ctime' => $archive_info->{ctime},
 668             };
 669             push @{$bklist}, $backup;
 670         }
 671     }
 672
 673     return $bklist;
 674 }
 675
 676 sub exec_backup_task {
 677     my ($self, $task) = @_;
 678
 679     my $opts = $self->{opts};
 680
 681     my $cfg = PVE::Storage::config();
 682     my $vmid = $task->{vmid};
 683     my $plugin = $task->{plugin};
 684
 685     $task->{backup_time} = time();
 686
 687     my $pbs_group_name;
 688     my $pbs_snapshot_name;
 689
 690     my $vmstarttime = time ();
 691
 692     my $logfd;
 693
 694     my $cleanup = {};
 695
 696     my $log_vm_online_again = sub {
 697         return if !defined($task->{vmstoptime});
 698         $task->{vmconttime} //= time();
 699         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 700         $delay = '<1' if $delay < 1;
 701         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 702     };
 703
 704     eval {
 705         die "unable to find VM '$vmid'\n" if !$plugin;
 706
 707         my $vmtype = $plugin->type();
 708
 709         if ($self->{opts}->{pbs}) {
 710             if ($vmtype eq 'lxc') {
 711                 $pbs_group_name = "ct/$vmid";
 712             } elsif  ($vmtype eq 'qemu') {
 713                 $pbs_group_name = "vm/$vmid";
 714             } else {
 715                 die "pbs backup not implemented for plugin type '$vmtype'\n";
 716             }
 717             my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
 718             $pbs_snapshot_name = "$pbs_group_name/$btime";
 719         }
 720
 721         # for now we deny backups of a running ha managed service in *stop* mode
 722         # as it interferes with the HA stack (started services should not stop).
 723         if ($opts->{mode} eq 'stop' &&
 724             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 725         {
 726             die "Cannot execute a backup with stop mode on a HA managed and".
 727                 " enabled Service. Use snapshot mode or disable the Service.\n";
 728         }
 729
 730         my $tmplog = "$logdir/$vmtype-$vmid.log";
 731
 732         my $bkname = "vzdump-$vmtype-$vmid";
 733         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
 734
 735         my $prune_options = $opts->{'prune-backups'};
 736
 737         my $backup_limit = 0;
 738         foreach my $keep (values %{$prune_options}) {
 739             $backup_limit += $keep;
 740         }
 741
 742         if ($backup_limit && !$opts->{remove}) {
 743             my $count;
 744             if ($self->{opts}->{pbs}) {
 745                 my $res = PVE::Storage::PBSPlugin::run_client_cmd($opts->{scfg}, $opts->{storage}, 'snapshots', $pbs_group_name);
 746                 $count = scalar(@$res);
 747             } else {
 748                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 749                 $count = scalar(@$bklist);
 750             }
 751             die "There is a max backup limit of $backup_limit enforced by the".
 752                 " target storage or the vzdump parameters.".
 753                 " Either increase the limit or delete old backup(s).\n"
 754                 if $count >= $backup_limit;
 755         }
 756
 757         if (!$self->{opts}->{pbs}) {
 758             $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 759         }
 760
 761         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 762         my ($comp, $comp_ext) = compressor_info($opts);
 763         if ($comp && $comp_ext) {
 764             $ext .= ".${comp_ext}";
 765         }
 766
 767         if ($self->{opts}->{pbs}) {
 768             die "unable to pipe backup to stdout\n" if $opts->{stdout};
 769             $task->{target} = $pbs_snapshot_name;
 770         } else {
 771             if ($opts->{stdout}) {
 772                 $task->{target} = '-';
 773             } else {
 774                 $task->{target} = $task->{tmptar} = "$opts->{dumpdir}/$basename$ext";
 775                 $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 776                 unlink $task->{tmptar};
 777             }
 778         }
 779
 780         $task->{vmtype} = $vmtype;
 781
 782         my $pid = $$;
 783         if ($opts->{tmpdir}) {
 784             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp${pid}_$vmid/";
 785         } elsif ($self->{opts}->{pbs}) {
 786             $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 787         } else {
 788             # dumpdir is posix? then use it as temporary dir
 789             my $info = get_mount_info($opts->{dumpdir});
 790             if ($vmtype eq 'qemu' ||
 791                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 792                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 793             } else {
 794                 $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 795                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 796                           "using $task->{tmpdir} for temporary files", $logfd);
 797             }
 798         }
 799
 800         rmtree $task->{tmpdir};
 801         mkdir $task->{tmpdir};
 802         -d $task->{tmpdir} ||
 803             die "unable to create temporary directory '$task->{tmpdir}'";
 804
 805         $logfd = IO::File->new (">$tmplog") ||
 806             die "unable to create log file '$tmplog'";
 807
 808         $task->{dumpdir} = $opts->{dumpdir};
 809         $task->{storeid} = $opts->{storage};
 810         $task->{scfg} = $opts->{scfg};
 811         $task->{tmplog} = $tmplog;
 812
 813         unlink $task->{logfile} if defined($task->{logfile});
 814
 815         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 816         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
 817
 818         $plugin->set_logfd ($logfd);
 819
 820         # test is VM is running
 821         my ($running, $status_text) = $plugin->vm_status ($vmid);
 822
 823         debugmsg ('info', "status = ${status_text}", $logfd);
 824
 825         # lock VM (prevent config changes)
 826         $plugin->lock_vm ($vmid);
 827
 828         $cleanup->{unlock} = 1;
 829
 830         # prepare
 831
 832         my $mode = $running ? $task->{mode} : 'stop';
 833
 834         if ($mode eq 'snapshot') {
 835             my %saved_task = %$task;
 836             eval { $plugin->prepare ($task, $vmid, $mode); };
 837             if (my $err = $@) {
 838                 die $err if $err !~ m/^mode failure/;
 839                 debugmsg ('info',  $err, $logfd);
 840                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 841                 $mode = 'suspend'; # so prepare is called again below
 842                 %$task = %saved_task;
 843             }
 844         }
 845
 846         $cleanup->{prepared} = 1;
 847
 848         $task->{mode} = $mode;
 849
 850         debugmsg ('info', "backup mode: $mode", $logfd);
 851
 852         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
 853             if $opts->{bwlimit};
 854
 855         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 856
 857         if ($mode eq 'stop') {
 858
 859             $plugin->prepare ($task, $vmid, $mode);
 860
 861             $self->run_hook_script ('backup-start', $task, $logfd);
 862
 863             if ($running) {
 864                 debugmsg ('info', "stopping vm", $logfd);
 865                 $task->{vmstoptime} = time();
 866                 $self->run_hook_script ('pre-stop', $task, $logfd);
 867                 $plugin->stop_vm ($task, $vmid);
 868                 $cleanup->{restart} = 1;
 869             }
 870
 871
 872         } elsif ($mode eq 'suspend') {
 873
 874             $plugin->prepare ($task, $vmid, $mode);
 875
 876             $self->run_hook_script ('backup-start', $task, $logfd);
 877
 878             if ($vmtype eq 'lxc') {
 879                 # pre-suspend rsync
 880                 $plugin->copy_data_phase1($task, $vmid);
 881             }
 882
 883             debugmsg ('info', "suspending guest", $logfd);
 884             $task->{vmstoptime} = time ();
 885             $self->run_hook_script ('pre-stop', $task, $logfd);
 886             $plugin->suspend_vm ($task, $vmid);
 887             $cleanup->{resume} = 1;
 888
 889             if ($vmtype eq 'lxc') {
 890                 # post-suspend rsync
 891                 $plugin->copy_data_phase2($task, $vmid);
 892
 893                 debugmsg ('info', "resuming guest", $logfd);
 894                 $cleanup->{resume} = 0;
 895                 $self->run_hook_script('pre-restart', $task, $logfd);
 896                 $plugin->resume_vm($task, $vmid);
 897                 $self->run_hook_script('post-restart', $task, $logfd);
 898                 $log_vm_online_again->();
 899             }
 900
 901         } elsif ($mode eq 'snapshot') {
 902
 903             $self->run_hook_script ('backup-start', $task, $logfd);
 904
 905             my $snapshot_count = $task->{snapshot_count} || 0;
 906
 907             $self->run_hook_script ('pre-stop', $task, $logfd);
 908
 909             if ($snapshot_count > 1) {
 910                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 911                 $task->{vmstoptime} = time ();
 912                 $plugin->suspend_vm ($task, $vmid);
 913                 $cleanup->{resume} = 1;
 914             }
 915
 916             $plugin->snapshot ($task, $vmid);
 917
 918             $self->run_hook_script ('pre-restart', $task, $logfd);
 919
 920             if ($snapshot_count > 1) {
 921                 debugmsg ('info', "resume vm", $logfd);
 922                 $cleanup->{resume} = 0;
 923                 $plugin->resume_vm ($task, $vmid);
 924                 $log_vm_online_again->();
 925             }
 926
 927             $self->run_hook_script ('post-restart', $task, $logfd);
 928
 929         } else {
 930             die "internal error - unknown mode '$mode'\n";
 931         }
 932
 933         # assemble archive image
 934         $plugin->assemble ($task, $vmid);
 935
 936         # produce archive
 937
 938         if ($opts->{stdout}) {
 939             debugmsg ('info', "sending archive to stdout", $logfd);
 940             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 941             $self->run_hook_script ('backup-end', $task, $logfd);
 942             return;
 943         }
 944
 945         my $archive_txt = $self->{opts}->{pbs} ? 'Proxmox Backup Server' : 'vzdump';
 946         debugmsg('info', "creating $archive_txt archive '$task->{target}'", $logfd);
 947         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 948
 949         if ($self->{opts}->{pbs}) {
 950             # size is added to task struct in guest vzdump plugins
 951         } else {
 952             rename ($task->{tmptar}, $task->{target}) ||
 953                 die "unable to rename '$task->{tmptar}' to '$task->{target}'\n";
 954
 955             # determine size
 956             $task->{size} = (-s $task->{target}) || 0;
 957             my $cs = format_size ($task->{size});
 958             debugmsg ('info', "archive file size: $cs", $logfd);
 959         }
 960
 961         # purge older backup
 962         if ($opts->{remove}) {
 963             if (!defined($opts->{storage})) {
 964                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{target});
 965                 PVE::Storage::prune_mark_backup_group($bklist, $prune_options);
 966
 967                 foreach my $prune_entry (@{$bklist}) {
 968                     next if $prune_entry->{mark} ne 'remove';
 969
 970                     my $archive_path = $prune_entry->{path};
 971                     debugmsg ('info', "delete old backup '$archive_path'", $logfd);
 972                     PVE::Storage::archive_remove($archive_path);
 973                 }
 974             } else {
 975                 my $logfunc = sub { debugmsg($_[0], $_[1], $logfd) };
 976                 PVE::Storage::prune_backups($cfg, $opts->{storage}, $prune_options, $vmid, $vmtype, 0, $logfunc);
 977             }
 978         }
 979
 980         $self->run_hook_script ('backup-end', $task, $logfd);
 981     };
 982     my $err = $@;
 983
 984     if ($plugin) {
 985         # clean-up
 986
 987         if ($cleanup->{unlock}) {
 988             eval { $plugin->unlock_vm ($vmid); };
 989             warn $@ if $@;
 990         }
 991
 992         if ($cleanup->{prepared}) {
 993             # only call cleanup when necessary (when prepare was executed)
 994             eval { $plugin->cleanup ($task, $vmid) };
 995             warn $@ if $@;
 996         }
 997
 998         eval { $plugin->set_logfd (undef); };
 999         warn $@ if $@;
1000
1001         if ($cleanup->{resume} || $cleanup->{restart}) {
1002             eval {
1003                 $self->run_hook_script ('pre-restart', $task, $logfd);
1004                 if ($cleanup->{resume}) {
1005                     debugmsg ('info', "resume vm", $logfd);
1006                     $plugin->resume_vm ($task, $vmid);
1007                 } else {
1008                     my $running = $plugin->vm_status($vmid);
1009                     if (!$running) {
1010                         debugmsg ('info', "restarting vm", $logfd);
1011                         $plugin->start_vm ($task, $vmid);
1012                     }
1013                 }
1014                 $self->run_hook_script ('post-restart', $task, $logfd);
1015             };
1016             my $err = $@;
1017             if ($err) {
1018                 warn $err;
1019             } else {
1020                 $log_vm_online_again->();
1021             }
1022         }
1023     }
1024
1025     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
1026     warn $@ if $@;
1027
1028     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
1029     warn $@ if $@;
1030
1031     my $delay = $task->{backuptime} = time () - $vmstarttime;
1032
1033     if ($err) {
1034         $task->{state} = 'err';
1035         $task->{msg} = $err;
1036         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1037         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
1038
1039         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1040
1041     } else {
1042         $task->{state} = 'ok';
1043         my $tstr = format_time ($delay);
1044         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1045         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1046     }
1047
1048     close ($logfd) if $logfd;
1049
1050     if ($task->{tmplog}) {
1051         if ($self->{opts}->{pbs}) {
1052             if ($task->{state} eq 'ok') {
1053                 my $param = [$pbs_snapshot_name, $task->{tmplog}];
1054                 PVE::Storage::PBSPlugin::run_raw_client_cmd(
1055                     $opts->{scfg}, $opts->{storage}, 'upload-log', $param, errmsg => "upload log failed");
1056             }
1057         } elsif ($task->{logfile}) {
1058             system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1059         }
1060     }
1061
1062     eval { $self->run_hook_script ('log-end', $task); };
1063
1064     die $err if $err && $err =~ m/^interrupted by signal$/;
1065 }
1066
1067 sub exec_backup {
1068     my ($self, $rpcenv, $authuser) = @_;
1069
1070     my $opts = $self->{opts};
1071
1072     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1073
1074     if (scalar(@{$self->{skiplist}})) {
1075         my $skip_string = join(', ', sort { $a <=> $b } @{$self->{skiplist}});
1076         debugmsg ('info', "skip external VMs: $skip_string");
1077     }
1078
1079     my $tasklist = [];
1080     my $vzdump_plugins =  {};
1081     foreach my $plugin (@{$self->{plugins}}) {
1082         my $type = $plugin->type();
1083         next if exists $vzdump_plugins->{$type};
1084         $vzdump_plugins->{$type} = $plugin;
1085     }
1086
1087     my $vmlist = PVE::Cluster::get_vmlist();
1088     my $vmids = [ sort { $a <=> $b } @{$opts->{vmids}} ];
1089     foreach my $vmid (@{$vmids}) {
1090         my $plugin;
1091         if (defined($vmlist->{ids}->{$vmid})) {
1092             my $guest_type = $vmlist->{ids}->{$vmid}->{type};
1093             $plugin = $vzdump_plugins->{$guest_type};
1094             next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all});
1095         }
1096         push @$tasklist, {
1097             mode => $opts->{mode},
1098             plugin => $plugin,
1099             state => 'todo',
1100             vmid => $vmid,
1101         };
1102     }
1103
1104     # Use in-memory files for the outer hook logs to pass them to sendmail.
1105     my $job_start_log = '';
1106     my $job_end_log = '';
1107     open my $job_start_fd, '>', \$job_start_log;
1108     open my $job_end_fd, '>', \$job_end_log;
1109
1110     my $starttime = time();
1111     my $errcount = 0;
1112     eval {
1113
1114         $self->run_hook_script ('job-start', undef, $job_start_fd);
1115
1116         foreach my $task (@$tasklist) {
1117             $self->exec_backup_task ($task);
1118             $errcount += 1 if $task->{state} ne 'ok';
1119         }
1120
1121         $self->run_hook_script ('job-end', undef, $job_end_fd);
1122     };
1123     my $err = $@;
1124
1125     $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1126
1127     if ($err) {
1128         debugmsg ('err', "Backup job failed - $err", undef, 1);
1129     } else {
1130         if ($errcount) {
1131             debugmsg ('info', "Backup job finished with errors", undef, 1);
1132         } else {
1133             debugmsg ('info', "Backup job finished successfully", undef, 1);
1134         }
1135     }
1136
1137     close $job_start_fd;
1138     close $job_end_fd;
1139
1140     my $totaltime = time() - $starttime;
1141
1142     eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); };
1143     debugmsg ('err', $@) if $@;
1144
1145     die $err if $err;
1146
1147     die "job errors\n" if $errcount;
1148
1149     unlink $pidfile;
1150 }
1151
1152
1153 sub option_exists {
1154     my $key = shift;
1155     return defined($confdesc->{$key});
1156 }
1157
1158 sub verify_vzdump_parameters {
1159     my ($param, $check_missing) = @_;
1160
1161     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1162         if $param->{all} && $param->{vmid};
1163
1164     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1165         if $param->{exclude} && $param->{vmid};
1166
1167     raise_param_exc({ pool => "option conflicts with option 'vmid'"})
1168         if $param->{pool} && $param->{vmid};
1169
1170     raise_param_exc({ 'prune-backups' => "option conflicts with option 'maxfiles'"})
1171         if defined($param->{'prune-backups'}) && defined($param->{maxfiles});
1172
1173     $param->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $param->{'prune-backups'})
1174         if defined($param->{'prune-backups'});
1175
1176     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
1177
1178     warn "option 'size' is deprecated and will be removed in a future " .
1179          "release, please update your script/configuration!\n"
1180         if defined($param->{size});
1181
1182     return if !$check_missing;
1183
1184     raise_param_exc({ vmid => "property is missing"})
1185         if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
1186
1187 }
1188
1189 sub stop_running_backups {
1190     my($self) = @_;
1191
1192     my $upid = PVE::Tools::file_read_firstline($pidfile);
1193     return if !$upid;
1194
1195     my $task = PVE::Tools::upid_decode($upid);
1196
1197     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1198         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1199         kill(15, $task->{pid});
1200         # wait max 15 seconds to shut down (else, do nothing for now)
1201         my $i;
1202         for ($i = 15; $i > 0; $i--) {
1203             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1204             sleep (1);
1205         }
1206         die "stopping backup process $task->{pid} failed\n" if $i == 0;
1207     }
1208 }
1209
1210 sub get_included_guests {
1211     my ($job) = @_;
1212
1213     my $vmids = [];
1214     my $vmids_per_node = {};
1215
1216     my $vmlist = PVE::Cluster::get_vmlist();
1217
1218     if ($job->{pool}) {
1219         $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool});
1220     } elsif ($job->{vmid}) {
1221         $vmids = [ split_list($job->{vmid}) ];
1222     } elsif ($job->{all}) {
1223         # all or exclude
1224         my $exclude = check_vmids(split_list($job->{exclude}));
1225         my $excludehash = { map { $_ => 1 } @$exclude };
1226
1227         for my $id (keys %{$vmlist->{ids}}) {
1228             next if $excludehash->{$id};
1229             push @$vmids, $id;
1230         }
1231     } else {
1232         return $vmids_per_node;
1233     }
1234     $vmids = check_vmids(@$vmids);
1235
1236     for my $vmid (@$vmids) {
1237         if (defined($vmlist->{ids}->{$vmid})) {
1238             my $node = $vmlist->{ids}->{$vmid}->{node};
1239             next if (defined $job->{node} && $job->{node} ne $node);
1240
1241             push @{$vmids_per_node->{$node}}, $vmid;
1242         } else {
1243             push @{$vmids_per_node->{''}}, $vmid;
1244         }
1245     }
1246
1247     return $vmids_per_node;
1248 }
1249
1250 1;