PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Path;
   8 use IO::File;
   9 use IO::Select;
  10 use IPC::Open3;
  11 use POSIX qw(strftime);
  12 use Time::Local;
  13
  14 use PVE::Cluster qw(cfs_read_file);
  15 use PVE::DataCenterConfig;
  16 use PVE::Exception qw(raise_param_exc);
  17 use PVE::HA::Config;
  18 use PVE::HA::Env::PVE2;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::RPCEnvironment;
  21 use PVE::Storage;
  22 use PVE::VZDump::Common;
  23 use PVE::VZDump::Plugin;
  24 use PVE::Tools qw(extract_param split_list);
  25 use PVE::API2Tools;
  26
  27 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  28
  29 my $lockfile = '/var/run/vzdump.lock';
  30 my $pidfile = '/var/run/vzdump.pid';
  31 my $logdir = '/var/log/vzdump';
  32
  33 my @plugins = qw();
  34
  35 my $confdesc = PVE::VZDump::Common::get_confdesc();
  36
  37 # Load available plugins
  38 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
  39 foreach my $plug (@pve_vzdump_classes) {
  40     my $filename = "/usr/share/perl5/$plug.pm";
  41     $filename =~ s!::!/!g;
  42     if (-f $filename) {
  43         eval { require $filename; };
  44         if (!$@) {
  45             $plug->import ();
  46             push @plugins, $plug;
  47         } else {
  48             die $@;
  49         }
  50     }
  51 }
  52
  53 # helper functions
  54
  55 sub debugmsg {
  56     my ($mtype, $msg, $logfd, $syslog) = @_;
  57
  58     PVE::VZDump::Plugin::debugmsg(@_);
  59 }
  60
  61 sub run_command {
  62     my ($logfd, $cmdstr, %param) = @_;
  63
  64     my $logfunc = sub {
  65         my $line = shift;
  66         debugmsg ('info', $line, $logfd);
  67     };
  68
  69     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
  70 }
  71
  72 my $parse_prune_backups_maxfiles = sub {
  73     my ($param, $kind) = @_;
  74
  75     my $maxfiles = delete $param->{maxfiles};
  76     my $prune_backups = $param->{'prune-backups'};
  77
  78     debugmsg('warn', "both 'maxfiles' and 'prune-backups' defined as ${kind} - ignoring 'maxfiles'")
  79         if defined($maxfiles) && defined($prune_backups);
  80
  81     if (defined($prune_backups)) {
  82         return if ref($prune_backups) eq 'HASH'; # already parsed
  83         $param->{'prune-backups'} = PVE::JSONSchema::parse_property_string(
  84             'prune-backups',
  85             $prune_backups
  86         );
  87     } elsif (defined($maxfiles)) {
  88         if ($maxfiles) {
  89             $param->{'prune-backups'} = { 'keep-last' => $maxfiles };
  90         } else {
  91             $param->{'prune-backups'} = { 'keep-all' => 1 };
  92         }
  93     }
  94 };
  95
  96 sub storage_info {
  97     my $storage = shift;
  98
  99     my $cfg = PVE::Storage::config();
 100     my $scfg = PVE::Storage::storage_config($cfg, $storage);
 101     my $type = $scfg->{type};
 102
 103     die "can't use storage type '$type' for backup\n"
 104         if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'
 105               || $type eq 'cifs' || $type eq 'cephfs' || $type eq 'pbs'));
 106     die "can't use storage '$storage' for backups - wrong content type\n"
 107         if (!$scfg->{content}->{backup});
 108
 109     PVE::Storage::activate_storage($cfg, $storage);
 110
 111     my $info = {
 112         scfg => $scfg,
 113     };
 114
 115     $info->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'})
 116         if defined($scfg->{'prune-backups'});
 117
 118     if ($type eq 'pbs') {
 119         $info->{pbs} = 1;
 120     } else {
 121         $info->{dumpdir} = PVE::Storage::get_backup_dir($cfg, $storage);
 122     }
 123
 124     return $info;
 125 }
 126
 127 sub format_size {
 128     my $size = shift;
 129
 130     my $kb = $size / 1024;
 131
 132     if ($kb < 1024) {
 133         return int ($kb) . "KB";
 134     }
 135
 136     my $mb = $size / (1024*1024);
 137     if ($mb < 1024) {
 138         return int ($mb) . "MB";
 139     }
 140     my $gb = $mb / 1024;
 141     if ($gb < 1024) {
 142         return sprintf ("%.2fGB", $gb);
 143     }
 144     my $tb = $gb / 1024;
 145     return sprintf ("%.2fTB", $tb);
 146 }
 147
 148 sub format_time {
 149     my $seconds = shift;
 150
 151     my $hours = int ($seconds/3600);
 152     $seconds = $seconds - $hours*3600;
 153     my $min = int ($seconds/60);
 154     $seconds = $seconds - $min*60;
 155
 156     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 157 }
 158
 159 sub encode8bit {
 160     my ($str) = @_;
 161
 162     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 163
 164     return $str;
 165 }
 166
 167 sub escape_html {
 168     my ($str) = @_;
 169
 170     $str =~ s/&/&amp;/g;
 171     $str =~ s/</&lt;/g;
 172     $str =~ s/>/&gt;/g;
 173
 174     return $str;
 175 }
 176
 177 sub check_bin {
 178     my ($bin)  = @_;
 179
 180     foreach my $p (split (/:/, $ENV{PATH})) {
 181         my $fn = "$p/$bin";
 182         if (-x $fn) {
 183             return $fn;
 184         }
 185     }
 186
 187     die "unable to find command '$bin'\n";
 188 }
 189
 190 sub check_vmids {
 191     my (@vmids) = @_;
 192
 193     my $res = [];
 194     for my $vmid (sort {$a <=> $b} @vmids) {
 195         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 196         $vmid = int ($vmid); # remove leading zeros
 197         next if !$vmid;
 198         push @$res, $vmid;
 199     }
 200
 201     return $res;
 202 }
 203
 204
 205 sub read_vzdump_defaults {
 206
 207     my $fn = "/etc/vzdump.conf";
 208
 209     my $defaults = {
 210         map {
 211             my $default = $confdesc->{$_}->{default};
 212              defined($default) ? ($_ => $default) : ()
 213         } keys %$confdesc
 214     };
 215     $parse_prune_backups_maxfiles->($defaults, "defaults in VZDump schema");
 216
 217     my $raw;
 218     eval { $raw = PVE::Tools::file_get_contents($fn); };
 219     return $defaults if $@;
 220
 221     my $conf_schema = { type => 'object', properties => $confdesc, };
 222     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 223     if (my $excludes = $res->{'exclude-path'}) {
 224         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 225     }
 226     if (defined($res->{mailto})) {
 227         my @mailto = split_list($res->{mailto});
 228         $res->{mailto} = [ @mailto ];
 229     }
 230     $parse_prune_backups_maxfiles->($res, "options in '$fn'");
 231
 232     foreach my $key (keys %$defaults) {
 233         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 234     }
 235
 236     if (defined($res->{storage}) && defined($res->{dumpdir})) {
 237         debugmsg('warn', "both 'storage' and 'dumpdir' defined in '$fn' - ignoring 'dumpdir'");
 238         delete $res->{dumpdir};
 239     }
 240
 241     return $res;
 242 }
 243
 244 use constant MAX_MAIL_SIZE => 1024*1024;
 245 sub sendmail {
 246     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 247
 248     my $opts = $self->{opts};
 249
 250     my $mailto = $opts->{mailto};
 251
 252     return if !($mailto && scalar(@$mailto));
 253
 254     my $cmdline = $self->{cmdline};
 255
 256     my $ecount = 0;
 257     foreach my $task (@$tasklist) {
 258         $ecount++ if $task->{state} ne 'ok';
 259         chomp $task->{msg} if $task->{msg};
 260         $task->{backuptime} = 0 if !$task->{backuptime};
 261         $task->{size} = 0 if !$task->{size};
 262         $task->{target} = 'unknown' if !$task->{target};
 263         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 264
 265         if ($task->{state} eq 'todo') {
 266             $task->{msg} = 'aborted';
 267         }
 268     }
 269
 270     my $notify = $opts->{mailnotification} || 'always';
 271     return if (!$ecount && !$err && ($notify eq 'failure'));
 272
 273     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 274     if ($err) {
 275         if ($err =~ /\n/) {
 276             $stat .= ": multiple problems";
 277         } else {
 278             $stat .= ": $err";
 279             $err = undef;
 280         }
 281     }
 282
 283     my $hostname = `hostname -f` || PVE::INotify::nodename();
 284     chomp $hostname;
 285
 286     # text part
 287     my $text = $err ? "$err\n\n" : '';
 288     my $namelength = 20;
 289     $text .= sprintf (
 290         "%-10s %-${namelength}s %-6s %10s %10s  %s\n",
 291         qw(VMID NAME STATUS TIME SIZE FILENAME)
 292     );
 293     foreach my $task (@$tasklist) {
 294         my $name = substr($task->{hostname}, 0, $namelength);
 295         my $successful = $task->{state} eq 'ok';
 296         my $size = $successful ? format_size ($task->{size}) : 0;
 297         my $filename = $successful ? $task->{target} : '-';
 298         my $size_fmt = $successful ? "%10s": "%8.2fMB";
 299         $text .= sprintf(
 300             "%-10s %-${namelength}s %-6s %10s $size_fmt  %s\n",
 301             $task->{vmid},
 302             $name,
 303             $task->{state},
 304             format_time($task->{backuptime}),
 305             $size,
 306             $filename,
 307         );
 308     }
 309
 310     my $text_log_part;
 311     $text_log_part .= "\nDetailed backup logs:\n\n";
 312     $text_log_part .= "$cmdline\n\n";
 313
 314     $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
 315     foreach my $task (@$tasklist) {
 316         my $vmid = $task->{vmid};
 317         my $log = $task->{tmplog};
 318         if (!$log) {
 319             $text_log_part .= "$vmid: no log available\n\n";
 320             next;
 321         }
 322         if (open (TMP, "$log")) {
 323             while (my $line = <TMP>) {
 324                 next if $line =~ /^status: \d+/; # not useful in mails
 325                 $text_log_part .= encode8bit ("$vmid: $line");
 326             }
 327         } else {
 328             $text_log_part .= "$vmid: Could not open log file\n\n";
 329         }
 330         close (TMP);
 331         $text_log_part .= "\n";
 332     }
 333     $text_log_part .= $detail_post if defined($detail_post);
 334
 335     # html part
 336     my $html = "<html><body>\n";
 337     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 338     $html .= "<table border=1 cellpadding=3>\n";
 339     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 340
 341     my $ssize = 0;
 342
 343     foreach my $task (@$tasklist) {
 344         my $vmid = $task->{vmid};
 345         my $name = $task->{hostname};
 346
 347         if  ($task->{state} eq 'ok') {
 348
 349             $ssize += $task->{size};
 350
 351             $html .= sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 352                                 $vmid, $name,
 353                                 format_time($task->{backuptime}),
 354                                 format_size ($task->{size}),
 355                                 escape_html ($task->{target}));
 356         } else {
 357             $html .= sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 358                                 $vmid, $name, format_time($task->{backuptime}),
 359                                 escape_html ($task->{msg}));
 360         }
 361     }
 362
 363     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 364  format_time ($totaltime), format_size ($ssize));
 365
 366     $html .= "\n</table><br><br>\n";
 367     my $html_log_part;
 368     $html_log_part .= "Detailed backup logs:<br /><br />\n";
 369     $html_log_part .= "<pre>\n";
 370     $html_log_part .= escape_html($cmdline) . "\n\n";
 371
 372     $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 373     foreach my $task (@$tasklist) {
 374         my $vmid = $task->{vmid};
 375         my $log = $task->{tmplog};
 376         if (!$log) {
 377             $html_log_part .= "$vmid: no log available\n\n";
 378             next;
 379         }
 380         if (open (TMP, "$log")) {
 381             while (my $line = <TMP>) {
 382                 next if $line =~ /^status: \d+/; # not useful in mails
 383                 if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 384                     $html_log_part .= encode8bit ("$vmid: <font color=red>".
 385                         escape_html ($line) . "</font>");
 386                 } else {
 387                     $html_log_part .= encode8bit ("$vmid: " . escape_html ($line));
 388                 }
 389             }
 390         } else {
 391             $html_log_part .= "$vmid: Could not open log file\n\n";
 392         }
 393         close (TMP);
 394         $html_log_part .= "\n";
 395     }
 396     $html_log_part .= escape_html($detail_post) if defined($detail_post);
 397     $html_log_part .= "</pre>";
 398     my $html_end = "\n</body></html>\n";
 399     # end html part
 400
 401     if (length($text) + length($text_log_part) +
 402         length($html) + length($html_log_part) +
 403         length($html_end) < MAX_MAIL_SIZE)
 404     {
 405         $html .= $html_log_part;
 406         $html .= $html_end;
 407         $text .= $text_log_part;
 408     } else {
 409         my $msg = "Log output was too long to be sent by mail. ".
 410             "See Task History for details!\n";
 411         $text .= $msg;
 412         $html .= "<p>$msg</p>";
 413         $html .= $html_end;
 414     }
 415
 416     my $subject = "vzdump backup status ($hostname) : $stat";
 417
 418     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 419     my $mailfrom = $dcconf->{email_from} || "root";
 420
 421     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 422 };
 423
 424 sub new {
 425     my ($class, $cmdline, $opts, $skiplist) = @_;
 426
 427     mkpath $logdir;
 428
 429     check_bin ('cp');
 430     check_bin ('df');
 431     check_bin ('sendmail');
 432     check_bin ('rsync');
 433     check_bin ('tar');
 434     check_bin ('mount');
 435     check_bin ('umount');
 436     check_bin ('cstream');
 437     check_bin ('ionice');
 438
 439     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 440         check_bin ('lvcreate');
 441         check_bin ('lvs');
 442         check_bin ('lvremove');
 443     }
 444
 445     my $defaults = read_vzdump_defaults();
 446
 447     $opts->{remove} = 1 if !defined($opts->{remove});
 448
 449     foreach my $k (keys %$defaults) {
 450         next if $k eq 'exclude-path' || $k eq 'prune-backups'; # dealt with separately
 451         if ($k eq 'dumpdir' || $k eq 'storage') {
 452             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 453                 !defined ($opts->{storage});
 454         } else {
 455             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 456         }
 457     }
 458
 459     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 460     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 461
 462     $skiplist = [] if !$skiplist;
 463     my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist };
 464
 465     my $findexcl = $self->{findexcl} = [];
 466     if ($defaults->{'exclude-path'}) {
 467         push @$findexcl, @{$defaults->{'exclude-path'}};
 468     }
 469
 470     if ($opts->{'exclude-path'}) {
 471         push @$findexcl, @{$opts->{'exclude-path'}};
 472     }
 473
 474     if ($opts->{stdexcludes}) {
 475         push @$findexcl, '/tmp/?*',
 476                          '/var/tmp/?*',
 477                          '/var/run/?*.pid';
 478     }
 479
 480     foreach my $p (@plugins) {
 481
 482         my $pd = $p->new ($self);
 483
 484         push @{$self->{plugins}}, $pd;
 485     }
 486
 487     if (defined($opts->{storage}) && $opts->{stdout}) {
 488         die "cannot use options 'storage' and 'stdout' at the same time\n";
 489     } elsif (defined($opts->{storage}) && defined($opts->{dumpdir})) {
 490         die "cannot use options 'storage' and 'dumpdir' at the same time\n";
 491     }
 492
 493     if (!$opts->{dumpdir} && !$opts->{storage}) {
 494         $opts->{storage} = 'local';
 495     }
 496
 497     my $errors = '';
 498
 499     if ($opts->{storage}) {
 500         my $info = eval { storage_info ($opts->{storage}) };
 501         if (my $err = $@) {
 502             chomp($err);
 503             $errors .= "could not get storage information for '$opts->{storage}': $err";
 504         } else {
 505             $opts->{dumpdir} = $info->{dumpdir};
 506             $opts->{scfg} = $info->{scfg};
 507             $opts->{pbs} = $info->{pbs};
 508             $opts->{'prune-backups'} //= $info->{'prune-backups'};
 509         }
 510     } elsif ($opts->{dumpdir}) {
 511         $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
 512             if ! -d $opts->{dumpdir};
 513     } else {
 514         die "internal error";
 515     }
 516
 517     $opts->{'prune-backups'} //= $defaults->{'prune-backups'};
 518
 519     # avoid triggering any remove code path if keep-all is set
 520     $opts->{remove} = 0 if $opts->{'prune-backups'}->{'keep-all'};
 521
 522     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 523         $errors .= "\n" if $errors;
 524         $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
 525     }
 526
 527     if ($errors) {
 528         eval { $self->sendmail([], 0, $errors); };
 529         debugmsg ('err', $@) if $@;
 530         die "$errors\n";
 531     }
 532
 533     return $self;
 534 }
 535
 536 sub get_mount_info {
 537     my ($dir) = @_;
 538
 539     # Note: df 'available' can be negative, and percentage set to '-'
 540
 541     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 542
 543     my $res;
 544
 545     my $parser = sub {
 546         my $line = shift;
 547         if (my ($fsid, $fstype, undef, $mp) = $line =~
 548             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 549             $res = {
 550                 device => $fsid,
 551                 fstype => $fstype,
 552                 mountpoint => $mp,
 553             };
 554         }
 555     };
 556
 557     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 558     warn $@ if $@;
 559
 560     return $res;
 561 }
 562
 563 sub getlock {
 564     my ($self, $upid) = @_;
 565
 566     my $fh;
 567
 568     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 569
 570     die "missing UPID" if !$upid; # should not happen
 571
 572     if (!open (SERVER_FLCK, ">>$lockfile")) {
 573         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 574         die "can't open lock on file '$lockfile' - $!";
 575     }
 576
 577     if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 578
 579         if (!$maxwait) {
 580             debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
 581             die "can't acquire lock '$lockfile' (wait = 0)";
 582         }
 583
 584         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 585
 586         eval {
 587             alarm ($maxwait * 60);
 588
 589             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 590
 591             if (!flock (SERVER_FLCK, LOCK_EX)) {
 592                 my $err = $!;
 593                 close (SERVER_FLCK);
 594                 alarm (0);
 595                 die "$err\n";
 596             }
 597             alarm (0);
 598         };
 599         alarm (0);
 600
 601         my $err = $@;
 602
 603         if ($err) {
 604             debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
 605             die "can't acquire lock '$lockfile' - $err";
 606         }
 607
 608         debugmsg('info', "got global lock", undef, 1);
 609     }
 610
 611     PVE::Tools::file_set_contents($pidfile, $upid);
 612 }
 613
 614 sub run_hook_script {
 615     my ($self, $phase, $task, $logfd) = @_;
 616
 617     my $opts = $self->{opts};
 618
 619     my $script = $opts->{script};
 620     return if !$script;
 621
 622     if (!-x $script) {
 623         die "The hook script '$script' is not executable.\n";
 624     }
 625
 626     my $cmd = "$script $phase";
 627
 628     $cmd .= " $task->{mode} $task->{vmid}" if ($task);
 629
 630     local %ENV;
 631     # set immutable opts directly (so they are available in all phases)
 632     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 633     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 634
 635     foreach my $ek (qw(vmtype hostname target logfile)) {
 636         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 637     }
 638     # FIXME: for backwards compatibility - drop with PVE 7.0
 639     $ENV{TARFILE} = $task->{target} if $task->{target};
 640
 641     run_command ($logfd, $cmd);
 642 }
 643
 644 sub compressor_info {
 645     my ($opts) = @_;
 646     my $opt_compress = $opts->{compress};
 647
 648     if (!$opt_compress || $opt_compress eq '0') {
 649         return undef;
 650     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 651         return ('lzop', 'lzo');
 652     } elsif ($opt_compress eq 'gzip') {
 653         if ($opts->{pigz} > 0) {
 654             my $pigz_threads = $opts->{pigz};
 655             if ($pigz_threads == 1) {
 656                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 657                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 658             }
 659             return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
 660         } else {
 661             return ('gzip --rsyncable', 'gz');
 662         }
 663     } elsif ($opt_compress eq 'zstd') {
 664         my $zstd_threads = $opts->{zstd} // 1;
 665         if ($zstd_threads == 0) {
 666             my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 667             $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
 668         }
 669         return ("zstd --rsyncable --threads=${zstd_threads}", 'zst');
 670     } else {
 671         die "internal error - unknown compression option '$opt_compress'";
 672     }
 673 }
 674
 675 sub get_backup_file_list {
 676     my ($dir, $bkname, $exclude_fn) = @_;
 677
 678     my $bklist = [];
 679     foreach my $fn (<$dir/${bkname}-*>) {
 680         next if $exclude_fn && $fn eq $exclude_fn;
 681
 682         my $archive_info = eval { PVE::Storage::archive_info($fn) } // {};
 683         if ($archive_info->{is_std_name}) {
 684             my $filename = $archive_info->{filename};
 685             my $backup = {
 686                 'path' => "$dir/$filename",
 687                 'ctime' => $archive_info->{ctime},
 688             };
 689             push @{$bklist}, $backup;
 690         }
 691     }
 692
 693     return $bklist;
 694 }
 695
 696 sub exec_backup_task {
 697     my ($self, $task) = @_;
 698
 699     my $opts = $self->{opts};
 700
 701     my $cfg = PVE::Storage::config();
 702     my $vmid = $task->{vmid};
 703     my $plugin = $task->{plugin};
 704
 705     $task->{backup_time} = time();
 706
 707     my $pbs_group_name;
 708     my $pbs_snapshot_name;
 709
 710     my $vmstarttime = time ();
 711
 712     my $logfd;
 713
 714     my $cleanup = {};
 715
 716     my $log_vm_online_again = sub {
 717         return if !defined($task->{vmstoptime});
 718         $task->{vmconttime} //= time();
 719         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 720         $delay = '<1' if $delay < 1;
 721         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 722     };
 723
 724     eval {
 725         die "unable to find VM '$vmid'\n" if !$plugin;
 726
 727         my $vmtype = $plugin->type();
 728
 729         if ($self->{opts}->{pbs}) {
 730             if ($vmtype eq 'lxc') {
 731                 $pbs_group_name = "ct/$vmid";
 732             } elsif  ($vmtype eq 'qemu') {
 733                 $pbs_group_name = "vm/$vmid";
 734             } else {
 735                 die "pbs backup not implemented for plugin type '$vmtype'\n";
 736             }
 737             my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
 738             $pbs_snapshot_name = "$pbs_group_name/$btime";
 739         }
 740
 741         # for now we deny backups of a running ha managed service in *stop* mode
 742         # as it interferes with the HA stack (started services should not stop).
 743         if ($opts->{mode} eq 'stop' &&
 744             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 745         {
 746             die "Cannot execute a backup with stop mode on a HA managed and".
 747                 " enabled Service. Use snapshot mode or disable the Service.\n";
 748         }
 749
 750         my $tmplog = "$logdir/$vmtype-$vmid.log";
 751
 752         my $bkname = "vzdump-$vmtype-$vmid";
 753         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
 754
 755         my $prune_options = $opts->{'prune-backups'};
 756
 757         my $backup_limit = 0;
 758         if (!$prune_options->{'keep-all'}) {
 759             foreach my $keep (values %{$prune_options}) {
 760                 $backup_limit += $keep;
 761             }
 762         }
 763
 764         if ($backup_limit && !$opts->{remove}) {
 765             my $count;
 766             if ($self->{opts}->{pbs}) {
 767                 my $res = PVE::Storage::PBSPlugin::run_client_cmd($opts->{scfg}, $opts->{storage}, 'snapshots', $pbs_group_name);
 768                 $count = scalar(@$res);
 769             } else {
 770                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 771                 $count = scalar(@$bklist);
 772             }
 773             die "There is a max backup limit of $backup_limit enforced by the".
 774                 " target storage or the vzdump parameters.".
 775                 " Either increase the limit or delete old backup(s).\n"
 776                 if $count >= $backup_limit;
 777         }
 778
 779         if (!$self->{opts}->{pbs}) {
 780             $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 781         }
 782
 783         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 784         my ($comp, $comp_ext) = compressor_info($opts);
 785         if ($comp && $comp_ext) {
 786             $ext .= ".${comp_ext}";
 787         }
 788
 789         if ($self->{opts}->{pbs}) {
 790             die "unable to pipe backup to stdout\n" if $opts->{stdout};
 791             $task->{target} = $pbs_snapshot_name;
 792         } else {
 793             if ($opts->{stdout}) {
 794                 $task->{target} = '-';
 795             } else {
 796                 $task->{target} = $task->{tmptar} = "$opts->{dumpdir}/$basename$ext";
 797                 $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 798                 unlink $task->{tmptar};
 799             }
 800         }
 801
 802         $task->{vmtype} = $vmtype;
 803
 804         my $pid = $$;
 805         if ($opts->{tmpdir}) {
 806             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp${pid}_$vmid/";
 807         } elsif ($self->{opts}->{pbs}) {
 808             $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 809         } else {
 810             # dumpdir is posix? then use it as temporary dir
 811             my $info = get_mount_info($opts->{dumpdir});
 812             if ($vmtype eq 'qemu' ||
 813                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 814                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 815             } else {
 816                 $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 817                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 818                           "using $task->{tmpdir} for temporary files", $logfd);
 819             }
 820         }
 821
 822         rmtree $task->{tmpdir};
 823         mkdir $task->{tmpdir};
 824         -d $task->{tmpdir} ||
 825             die "unable to create temporary directory '$task->{tmpdir}'";
 826
 827         $logfd = IO::File->new (">$tmplog") ||
 828             die "unable to create log file '$tmplog'";
 829
 830         $task->{dumpdir} = $opts->{dumpdir};
 831         $task->{storeid} = $opts->{storage};
 832         $task->{scfg} = $opts->{scfg};
 833         $task->{tmplog} = $tmplog;
 834
 835         unlink $task->{logfile} if defined($task->{logfile});
 836
 837         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 838         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
 839
 840         $plugin->set_logfd ($logfd);
 841
 842         # test is VM is running
 843         my ($running, $status_text) = $plugin->vm_status ($vmid);
 844
 845         debugmsg ('info', "status = ${status_text}", $logfd);
 846
 847         # lock VM (prevent config changes)
 848         $plugin->lock_vm ($vmid);
 849
 850         $cleanup->{unlock} = 1;
 851
 852         # prepare
 853
 854         my $mode = $running ? $task->{mode} : 'stop';
 855
 856         if ($mode eq 'snapshot') {
 857             my %saved_task = %$task;
 858             eval { $plugin->prepare ($task, $vmid, $mode); };
 859             if (my $err = $@) {
 860                 die $err if $err !~ m/^mode failure/;
 861                 debugmsg ('info',  $err, $logfd);
 862                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 863                 $mode = 'suspend'; # so prepare is called again below
 864                 %$task = %saved_task;
 865             }
 866         }
 867
 868         $cleanup->{prepared} = 1;
 869
 870         $task->{mode} = $mode;
 871
 872         debugmsg ('info', "backup mode: $mode", $logfd);
 873
 874         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
 875             if $opts->{bwlimit};
 876
 877         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 878
 879         if ($mode eq 'stop') {
 880
 881             $plugin->prepare ($task, $vmid, $mode);
 882
 883             $self->run_hook_script ('backup-start', $task, $logfd);
 884
 885             if ($running) {
 886                 debugmsg ('info', "stopping vm", $logfd);
 887                 $task->{vmstoptime} = time();
 888                 $self->run_hook_script ('pre-stop', $task, $logfd);
 889                 $plugin->stop_vm ($task, $vmid);
 890                 $cleanup->{restart} = 1;
 891             }
 892
 893
 894         } elsif ($mode eq 'suspend') {
 895
 896             $plugin->prepare ($task, $vmid, $mode);
 897
 898             $self->run_hook_script ('backup-start', $task, $logfd);
 899
 900             if ($vmtype eq 'lxc') {
 901                 # pre-suspend rsync
 902                 $plugin->copy_data_phase1($task, $vmid);
 903             }
 904
 905             debugmsg ('info', "suspending guest", $logfd);
 906             $task->{vmstoptime} = time ();
 907             $self->run_hook_script ('pre-stop', $task, $logfd);
 908             $plugin->suspend_vm ($task, $vmid);
 909             $cleanup->{resume} = 1;
 910
 911             if ($vmtype eq 'lxc') {
 912                 # post-suspend rsync
 913                 $plugin->copy_data_phase2($task, $vmid);
 914
 915                 debugmsg ('info', "resuming guest", $logfd);
 916                 $cleanup->{resume} = 0;
 917                 $self->run_hook_script('pre-restart', $task, $logfd);
 918                 $plugin->resume_vm($task, $vmid);
 919                 $self->run_hook_script('post-restart', $task, $logfd);
 920                 $log_vm_online_again->();
 921             }
 922
 923         } elsif ($mode eq 'snapshot') {
 924
 925             $self->run_hook_script ('backup-start', $task, $logfd);
 926
 927             my $snapshot_count = $task->{snapshot_count} || 0;
 928
 929             $self->run_hook_script ('pre-stop', $task, $logfd);
 930
 931             if ($snapshot_count > 1) {
 932                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 933                 $task->{vmstoptime} = time ();
 934                 $plugin->suspend_vm ($task, $vmid);
 935                 $cleanup->{resume} = 1;
 936             }
 937
 938             $plugin->snapshot ($task, $vmid);
 939
 940             $self->run_hook_script ('pre-restart', $task, $logfd);
 941
 942             if ($snapshot_count > 1) {
 943                 debugmsg ('info', "resume vm", $logfd);
 944                 $cleanup->{resume} = 0;
 945                 $plugin->resume_vm ($task, $vmid);
 946                 $log_vm_online_again->();
 947             }
 948
 949             $self->run_hook_script ('post-restart', $task, $logfd);
 950
 951         } else {
 952             die "internal error - unknown mode '$mode'\n";
 953         }
 954
 955         # assemble archive image
 956         $plugin->assemble ($task, $vmid);
 957
 958         # produce archive
 959
 960         if ($opts->{stdout}) {
 961             debugmsg ('info', "sending archive to stdout", $logfd);
 962             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 963             $self->run_hook_script ('backup-end', $task, $logfd);
 964             return;
 965         }
 966
 967         my $archive_txt = $self->{opts}->{pbs} ? 'Proxmox Backup Server' : 'vzdump';
 968         debugmsg('info', "creating $archive_txt archive '$task->{target}'", $logfd);
 969         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 970
 971         if ($self->{opts}->{pbs}) {
 972             # size is added to task struct in guest vzdump plugins
 973         } else {
 974             rename ($task->{tmptar}, $task->{target}) ||
 975                 die "unable to rename '$task->{tmptar}' to '$task->{target}'\n";
 976
 977             # determine size
 978             $task->{size} = (-s $task->{target}) || 0;
 979             my $cs = format_size ($task->{size});
 980             debugmsg ('info', "archive file size: $cs", $logfd);
 981         }
 982
 983         # purge older backup
 984         if ($opts->{remove}) {
 985             if (!defined($opts->{storage})) {
 986                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{target});
 987                 PVE::Storage::prune_mark_backup_group($bklist, $prune_options);
 988
 989                 foreach my $prune_entry (@{$bklist}) {
 990                     next if $prune_entry->{mark} ne 'remove';
 991
 992                     my $archive_path = $prune_entry->{path};
 993                     debugmsg ('info', "delete old backup '$archive_path'", $logfd);
 994                     PVE::Storage::archive_remove($archive_path);
 995                 }
 996             } else {
 997                 my $logfunc = sub { debugmsg($_[0], $_[1], $logfd) };
 998                 PVE::Storage::prune_backups($cfg, $opts->{storage}, $prune_options, $vmid, $vmtype, 0, $logfunc);
 999             }
1000         }
1001
1002         $self->run_hook_script ('backup-end', $task, $logfd);
1003     };
1004     my $err = $@;
1005
1006     if ($plugin) {
1007         # clean-up
1008
1009         if ($cleanup->{unlock}) {
1010             eval { $plugin->unlock_vm ($vmid); };
1011             warn $@ if $@;
1012         }
1013
1014         if ($cleanup->{prepared}) {
1015             # only call cleanup when necessary (when prepare was executed)
1016             eval { $plugin->cleanup ($task, $vmid) };
1017             warn $@ if $@;
1018         }
1019
1020         eval { $plugin->set_logfd (undef); };
1021         warn $@ if $@;
1022
1023         if ($cleanup->{resume} || $cleanup->{restart}) {
1024             eval {
1025                 $self->run_hook_script ('pre-restart', $task, $logfd);
1026                 if ($cleanup->{resume}) {
1027                     debugmsg ('info', "resume vm", $logfd);
1028                     $plugin->resume_vm ($task, $vmid);
1029                 } else {
1030                     my $running = $plugin->vm_status($vmid);
1031                     if (!$running) {
1032                         debugmsg ('info', "restarting vm", $logfd);
1033                         $plugin->start_vm ($task, $vmid);
1034                     }
1035                 }
1036                 $self->run_hook_script ('post-restart', $task, $logfd);
1037             };
1038             my $err = $@;
1039             if ($err) {
1040                 warn $err;
1041             } else {
1042                 $log_vm_online_again->();
1043             }
1044         }
1045     }
1046
1047     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
1048     warn $@ if $@;
1049
1050     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
1051     warn $@ if $@;
1052
1053     my $delay = $task->{backuptime} = time () - $vmstarttime;
1054
1055     if ($err) {
1056         $task->{state} = 'err';
1057         $task->{msg} = $err;
1058         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1059         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
1060
1061         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1062
1063     } else {
1064         $task->{state} = 'ok';
1065         my $tstr = format_time ($delay);
1066         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1067         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1068     }
1069
1070     close ($logfd) if $logfd;
1071
1072     if ($task->{tmplog}) {
1073         if ($self->{opts}->{pbs}) {
1074             if ($task->{state} eq 'ok') {
1075                 my $param = [$pbs_snapshot_name, $task->{tmplog}];
1076                 PVE::Storage::PBSPlugin::run_raw_client_cmd(
1077                     $opts->{scfg}, $opts->{storage}, 'upload-log', $param, errmsg => "upload log failed");
1078             }
1079         } elsif ($task->{logfile}) {
1080             system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1081         }
1082     }
1083
1084     eval { $self->run_hook_script ('log-end', $task); };
1085
1086     die $err if $err && $err =~ m/^interrupted by signal$/;
1087 }
1088
1089 sub exec_backup {
1090     my ($self, $rpcenv, $authuser) = @_;
1091
1092     my $opts = $self->{opts};
1093
1094     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1095
1096     if (scalar(@{$self->{skiplist}})) {
1097         my $skip_string = join(', ', sort { $a <=> $b } @{$self->{skiplist}});
1098         debugmsg ('info', "skip external VMs: $skip_string");
1099     }
1100
1101     my $tasklist = [];
1102     my $vzdump_plugins =  {};
1103     foreach my $plugin (@{$self->{plugins}}) {
1104         my $type = $plugin->type();
1105         next if exists $vzdump_plugins->{$type};
1106         $vzdump_plugins->{$type} = $plugin;
1107     }
1108
1109     my $vmlist = PVE::Cluster::get_vmlist();
1110     my $vmids = [ sort { $a <=> $b } @{$opts->{vmids}} ];
1111     foreach my $vmid (@{$vmids}) {
1112         my $plugin;
1113         if (defined($vmlist->{ids}->{$vmid})) {
1114             my $guest_type = $vmlist->{ids}->{$vmid}->{type};
1115             $plugin = $vzdump_plugins->{$guest_type};
1116             next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all});
1117         }
1118         push @$tasklist, {
1119             mode => $opts->{mode},
1120             plugin => $plugin,
1121             state => 'todo',
1122             vmid => $vmid,
1123         };
1124     }
1125
1126     # Use in-memory files for the outer hook logs to pass them to sendmail.
1127     my $job_start_log = '';
1128     my $job_end_log = '';
1129     open my $job_start_fd, '>', \$job_start_log;
1130     open my $job_end_fd, '>', \$job_end_log;
1131
1132     my $starttime = time();
1133     my $errcount = 0;
1134     eval {
1135
1136         $self->run_hook_script ('job-start', undef, $job_start_fd);
1137
1138         foreach my $task (@$tasklist) {
1139             $self->exec_backup_task ($task);
1140             $errcount += 1 if $task->{state} ne 'ok';
1141         }
1142
1143         $self->run_hook_script ('job-end', undef, $job_end_fd);
1144     };
1145     my $err = $@;
1146
1147     $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1148
1149     if ($err) {
1150         debugmsg ('err', "Backup job failed - $err", undef, 1);
1151     } else {
1152         if ($errcount) {
1153             debugmsg ('info', "Backup job finished with errors", undef, 1);
1154         } else {
1155             debugmsg ('info', "Backup job finished successfully", undef, 1);
1156         }
1157     }
1158
1159     close $job_start_fd;
1160     close $job_end_fd;
1161
1162     my $totaltime = time() - $starttime;
1163
1164     eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); };
1165     debugmsg ('err', $@) if $@;
1166
1167     die $err if $err;
1168
1169     die "job errors\n" if $errcount;
1170
1171     unlink $pidfile;
1172 }
1173
1174
1175 sub option_exists {
1176     my $key = shift;
1177     return defined($confdesc->{$key});
1178 }
1179
1180 sub verify_vzdump_parameters {
1181     my ($param, $check_missing) = @_;
1182
1183     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1184         if $param->{all} && $param->{vmid};
1185
1186     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1187         if $param->{exclude} && $param->{vmid};
1188
1189     raise_param_exc({ pool => "option conflicts with option 'vmid'"})
1190         if $param->{pool} && $param->{vmid};
1191
1192     raise_param_exc({ 'prune-backups' => "option conflicts with option 'maxfiles'"})
1193         if defined($param->{'prune-backups'}) && defined($param->{maxfiles});
1194
1195     $parse_prune_backups_maxfiles->($param, 'CLI parameters');
1196
1197     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
1198
1199     warn "option 'size' is deprecated and will be removed in a future " .
1200          "release, please update your script/configuration!\n"
1201         if defined($param->{size});
1202
1203     return if !$check_missing;
1204
1205     raise_param_exc({ vmid => "property is missing"})
1206         if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
1207
1208 }
1209
1210 sub stop_running_backups {
1211     my($self) = @_;
1212
1213     my $upid = PVE::Tools::file_read_firstline($pidfile);
1214     return if !$upid;
1215
1216     my $task = PVE::Tools::upid_decode($upid);
1217
1218     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1219         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1220         kill(15, $task->{pid});
1221         # wait max 15 seconds to shut down (else, do nothing for now)
1222         my $i;
1223         for ($i = 15; $i > 0; $i--) {
1224             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1225             sleep (1);
1226         }
1227         die "stopping backup process $task->{pid} failed\n" if $i == 0;
1228     }
1229 }
1230
1231 sub get_included_guests {
1232     my ($job) = @_;
1233
1234     my $vmids = [];
1235     my $vmids_per_node = {};
1236
1237     my $vmlist = PVE::Cluster::get_vmlist();
1238
1239     if ($job->{pool}) {
1240         $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool});
1241     } elsif ($job->{vmid}) {
1242         $vmids = [ split_list($job->{vmid}) ];
1243     } elsif ($job->{all}) {
1244         # all or exclude
1245         my $exclude = check_vmids(split_list($job->{exclude}));
1246         my $excludehash = { map { $_ => 1 } @$exclude };
1247
1248         for my $id (keys %{$vmlist->{ids}}) {
1249             next if $excludehash->{$id};
1250             push @$vmids, $id;
1251         }
1252     } else {
1253         return $vmids_per_node;
1254     }
1255     $vmids = check_vmids(@$vmids);
1256
1257     for my $vmid (@$vmids) {
1258         if (defined($vmlist->{ids}->{$vmid})) {
1259             my $node = $vmlist->{ids}->{$vmid}->{node};
1260             next if (defined $job->{node} && $job->{node} ne $node);
1261
1262             push @{$vmids_per_node->{$node}}, $vmid;
1263         } else {
1264             push @{$vmids_per_node->{''}}, $vmid;
1265         }
1266     }
1267
1268     return $vmids_per_node;
1269 }
1270
1271 1;