PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Path;
   8 use IO::File;
   9 use IO::Select;
  10 use IPC::Open3;
  11 use POSIX qw(strftime);
  12 use Time::Local;
  13
  14 use PVE::Cluster qw(cfs_read_file);
  15 use PVE::DataCenterConfig;
  16 use PVE::Exception qw(raise_param_exc);
  17 use PVE::HA::Config;
  18 use PVE::HA::Env::PVE2;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::RPCEnvironment;
  21 use PVE::Storage;
  22 use PVE::VZDump::Common;
  23 use PVE::VZDump::Plugin;
  24 use PVE::Tools qw(extract_param split_list);
  25 use PVE::API2Tools;
  26
  27 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  28
  29 my $lockfile = '/var/run/vzdump.lock';
  30 my $pidfile = '/var/run/vzdump.pid';
  31 my $logdir = '/var/log/vzdump';
  32
  33 my @plugins = qw();
  34
  35 my $confdesc = PVE::VZDump::Common::get_confdesc();
  36
  37 # Load available plugins
  38 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
  39 foreach my $plug (@pve_vzdump_classes) {
  40     my $filename = "/usr/share/perl5/$plug.pm";
  41     $filename =~ s!::!/!g;
  42     if (-f $filename) {
  43         eval { require $filename; };
  44         if (!$@) {
  45             $plug->import ();
  46             push @plugins, $plug;
  47         } else {
  48             die $@;
  49         }
  50     }
  51 }
  52
  53 # helper functions
  54
  55 sub debugmsg {
  56     my ($mtype, $msg, $logfd, $syslog) = @_;
  57
  58     PVE::VZDump::Plugin::debugmsg(@_);
  59 }
  60
  61 sub run_command {
  62     my ($logfd, $cmdstr, %param) = @_;
  63
  64     my $logfunc = sub {
  65         my $line = shift;
  66         debugmsg ('info', $line, $logfd);
  67     };
  68
  69     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
  70 }
  71
  72 sub storage_info {
  73     my $storage = shift;
  74
  75     my $cfg = PVE::Storage::config();
  76     my $scfg = PVE::Storage::storage_config($cfg, $storage);
  77     my $type = $scfg->{type};
  78
  79     die "can't use storage type '$type' for backup\n"
  80         if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'
  81               || $type eq 'cifs' || $type eq 'cephfs' || $type eq 'pbs'));
  82     die "can't use storage '$storage' for backups - wrong content type\n"
  83         if (!$scfg->{content}->{backup});
  84
  85     PVE::Storage::activate_storage($cfg, $storage);
  86
  87     my $info = {
  88         scfg => $scfg,
  89         maxfiles => $scfg->{maxfiles},
  90     };
  91
  92     $info->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'})
  93         if defined($scfg->{'prune-backups'});
  94
  95     die "cannot have 'maxfiles' and 'prune-backups' configured at the same time\n"
  96         if defined($info->{'prune-backups'}) && defined($info->{maxfiles});
  97
  98     if ($type eq 'pbs') {
  99         $info->{pbs} = 1;
 100     } else {
 101         $info->{dumpdir} = PVE::Storage::get_backup_dir($cfg, $storage);
 102     }
 103
 104     return $info;
 105 }
 106
 107 sub format_size {
 108     my $size = shift;
 109
 110     my $kb = $size / 1024;
 111
 112     if ($kb < 1024) {
 113         return int ($kb) . "KB";
 114     }
 115
 116     my $mb = $size / (1024*1024);
 117     if ($mb < 1024) {
 118         return int ($mb) . "MB";
 119     }
 120     my $gb = $mb / 1024;
 121     if ($gb < 1024) {
 122         return sprintf ("%.2fGB", $gb);
 123     }
 124     my $tb = $gb / 1024;
 125     return sprintf ("%.2fTB", $tb);
 126 }
 127
 128 sub format_time {
 129     my $seconds = shift;
 130
 131     my $hours = int ($seconds/3600);
 132     $seconds = $seconds - $hours*3600;
 133     my $min = int ($seconds/60);
 134     $seconds = $seconds - $min*60;
 135
 136     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 137 }
 138
 139 sub encode8bit {
 140     my ($str) = @_;
 141
 142     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 143
 144     return $str;
 145 }
 146
 147 sub escape_html {
 148     my ($str) = @_;
 149
 150     $str =~ s/&/&amp;/g;
 151     $str =~ s/</&lt;/g;
 152     $str =~ s/>/&gt;/g;
 153
 154     return $str;
 155 }
 156
 157 sub check_bin {
 158     my ($bin)  = @_;
 159
 160     foreach my $p (split (/:/, $ENV{PATH})) {
 161         my $fn = "$p/$bin";
 162         if (-x $fn) {
 163             return $fn;
 164         }
 165     }
 166
 167     die "unable to find command '$bin'\n";
 168 }
 169
 170 sub check_vmids {
 171     my (@vmids) = @_;
 172
 173     my $res = [];
 174     for my $vmid (sort {$a <=> $b} @vmids) {
 175         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 176         $vmid = int ($vmid); # remove leading zeros
 177         next if !$vmid;
 178         push @$res, $vmid;
 179     }
 180
 181     return $res;
 182 }
 183
 184
 185 sub read_vzdump_defaults {
 186
 187     my $fn = "/etc/vzdump.conf";
 188
 189     my $defaults = {
 190         map {
 191             my $default = $confdesc->{$_}->{default};
 192              defined($default) ? ($_ => $default) : ()
 193         } keys %$confdesc
 194     };
 195
 196     my $raw;
 197     eval { $raw = PVE::Tools::file_get_contents($fn); };
 198     return $defaults if $@;
 199
 200     my $conf_schema = { type => 'object', properties => $confdesc, };
 201     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 202     if (my $excludes = $res->{'exclude-path'}) {
 203         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 204     }
 205     if (defined($res->{mailto})) {
 206         my @mailto = split_list($res->{mailto});
 207         $res->{mailto} = [ @mailto ];
 208     }
 209
 210     foreach my $key (keys %$defaults) {
 211         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 212     }
 213
 214     return $res;
 215 }
 216
 217 use constant MAX_MAIL_SIZE => 1024*1024;
 218 sub sendmail {
 219     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 220
 221     my $opts = $self->{opts};
 222
 223     my $mailto = $opts->{mailto};
 224
 225     return if !($mailto && scalar(@$mailto));
 226
 227     my $cmdline = $self->{cmdline};
 228
 229     my $ecount = 0;
 230     foreach my $task (@$tasklist) {
 231         $ecount++ if $task->{state} ne 'ok';
 232         chomp $task->{msg} if $task->{msg};
 233         $task->{backuptime} = 0 if !$task->{backuptime};
 234         $task->{size} = 0 if !$task->{size};
 235         $task->{target} = 'unknown' if !$task->{target};
 236         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 237
 238         if ($task->{state} eq 'todo') {
 239             $task->{msg} = 'aborted';
 240         }
 241     }
 242
 243     my $notify = $opts->{mailnotification} || 'always';
 244     return if (!$ecount && !$err && ($notify eq 'failure'));
 245
 246     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 247     if ($err) {
 248         if ($err =~ /\n/) {
 249             $stat .= ": multiple problems";
 250         } else {
 251             $stat .= ": $err";
 252             $err = undef;
 253         }
 254     }
 255
 256     my $hostname = `hostname -f` || PVE::INotify::nodename();
 257     chomp $hostname;
 258
 259     # text part
 260     my $text = $err ? "$err\n\n" : '';
 261     $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", qw(VMID STATUS TIME SIZE FILENAME));
 262     foreach my $task (@$tasklist) {
 263         my $vmid = $task->{vmid};
 264         if  ($task->{state} eq 'ok') {
 265
 266             $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", $vmid,
 267                                 $task->{state},
 268                                 format_time($task->{backuptime}),
 269                                 format_size ($task->{size}),
 270                                 $task->{target});
 271         } else {
 272             $text .= sprintf ("%-10s %-6s %10s %8.2fMB  %s\n", $vmid,
 273                                 $task->{state},
 274                                 format_time($task->{backuptime}),
 275                                 0, '-');
 276         }
 277     }
 278
 279     my $text_log_part;
 280     $text_log_part .= "\nDetailed backup logs:\n\n";
 281     $text_log_part .= "$cmdline\n\n";
 282
 283     $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
 284     foreach my $task (@$tasklist) {
 285         my $vmid = $task->{vmid};
 286         my $log = $task->{tmplog};
 287         if (!$log) {
 288             $text_log_part .= "$vmid: no log available\n\n";
 289             next;
 290         }
 291         if (open (TMP, "$log")) {
 292             while (my $line = <TMP>) {
 293                 next if $line =~ /^status: \d+/; # not useful in mails
 294                 $text_log_part .= encode8bit ("$vmid: $line");
 295             }
 296         } else {
 297             $text_log_part .= "$vmid: Could not open log file\n\n";
 298         }
 299         close (TMP);
 300         $text_log_part .= "\n";
 301     }
 302     $text_log_part .= $detail_post if defined($detail_post);
 303
 304     # html part
 305     my $html = "<html><body>\n";
 306     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 307     $html .= "<table border=1 cellpadding=3>\n";
 308     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 309
 310     my $ssize = 0;
 311
 312     foreach my $task (@$tasklist) {
 313         my $vmid = $task->{vmid};
 314         my $name = $task->{hostname};
 315
 316         if  ($task->{state} eq 'ok') {
 317
 318             $ssize += $task->{size};
 319
 320             $html .= sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 321                                 $vmid, $name,
 322                                 format_time($task->{backuptime}),
 323                                 format_size ($task->{size}),
 324                                 escape_html ($task->{target}));
 325         } else {
 326             $html .= sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 327                                 $vmid, $name, format_time($task->{backuptime}),
 328                                 escape_html ($task->{msg}));
 329         }
 330     }
 331
 332     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 333  format_time ($totaltime), format_size ($ssize));
 334
 335     $html .= "\n</table><br><br>\n";
 336     my $html_log_part;
 337     $html_log_part .= "Detailed backup logs:<br /><br />\n";
 338     $html_log_part .= "<pre>\n";
 339     $html_log_part .= escape_html($cmdline) . "\n\n";
 340
 341     $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 342     foreach my $task (@$tasklist) {
 343         my $vmid = $task->{vmid};
 344         my $log = $task->{tmplog};
 345         if (!$log) {
 346             $html_log_part .= "$vmid: no log available\n\n";
 347             next;
 348         }
 349         if (open (TMP, "$log")) {
 350             while (my $line = <TMP>) {
 351                 next if $line =~ /^status: \d+/; # not useful in mails
 352                 if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 353                     $html_log_part .= encode8bit ("$vmid: <font color=red>".
 354                         escape_html ($line) . "</font>");
 355                 } else {
 356                     $html_log_part .= encode8bit ("$vmid: " . escape_html ($line));
 357                 }
 358             }
 359         } else {
 360             $html_log_part .= "$vmid: Could not open log file\n\n";
 361         }
 362         close (TMP);
 363         $html_log_part .= "\n";
 364     }
 365     $html_log_part .= escape_html($detail_post) if defined($detail_post);
 366     $html_log_part .= "</pre>";
 367     my $html_end .= "\n</body></html>\n";
 368     # end html part
 369
 370     if (length($text) + length($text_log_part) +
 371         length($html) + length($html_log_part) < MAX_MAIL_SIZE)
 372     {
 373         $html .= $html_log_part;
 374         $text .= $text_log_part;
 375     } else {
 376         my $msg = "Log output was too long to be sent by mail. ".
 377             "See Task History for details!\n";
 378         $text .= $msg;
 379         $html .= "<p>$msg</p>";
 380         $html .= $html_end;
 381     }
 382
 383     my $subject = "vzdump backup status ($hostname) : $stat";
 384
 385     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 386     my $mailfrom = $dcconf->{email_from} || "root";
 387
 388     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 389 };
 390
 391 sub new {
 392     my ($class, $cmdline, $opts, $skiplist) = @_;
 393
 394     mkpath $logdir;
 395
 396     check_bin ('cp');
 397     check_bin ('df');
 398     check_bin ('sendmail');
 399     check_bin ('rsync');
 400     check_bin ('tar');
 401     check_bin ('mount');
 402     check_bin ('umount');
 403     check_bin ('cstream');
 404     check_bin ('ionice');
 405
 406     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 407         check_bin ('lvcreate');
 408         check_bin ('lvs');
 409         check_bin ('lvremove');
 410     }
 411
 412     my $defaults = read_vzdump_defaults();
 413
 414     $opts->{remove} = 1 if !defined($opts->{remove});
 415
 416     foreach my $k (keys %$defaults) {
 417         next if $k eq 'exclude-path' || $k eq 'maxfiles'; # dealt with separately
 418         if ($k eq 'dumpdir' || $k eq 'storage') {
 419             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 420                 !defined ($opts->{storage});
 421         } else {
 422             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 423         }
 424     }
 425
 426     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 427     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 428
 429     $skiplist = [] if !$skiplist;
 430     my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist };
 431
 432     my $findexcl = $self->{findexcl} = [];
 433     if ($defaults->{'exclude-path'}) {
 434         push @$findexcl, @{$defaults->{'exclude-path'}};
 435     }
 436
 437     if ($opts->{'exclude-path'}) {
 438         push @$findexcl, @{$opts->{'exclude-path'}};
 439     }
 440
 441     if ($opts->{stdexcludes}) {
 442         push @$findexcl, '/tmp/?*',
 443                          '/var/tmp/?*',
 444                          '/var/run/?*.pid';
 445     }
 446
 447     foreach my $p (@plugins) {
 448
 449         my $pd = $p->new ($self);
 450
 451         push @{$self->{plugins}}, $pd;
 452     }
 453
 454     if (defined($opts->{storage}) && $opts->{stdout}) {
 455         die "cannot use options 'storage' and 'stdout' at the same time\n";
 456     } elsif (defined($opts->{storage}) && defined($opts->{dumpdir})) {
 457         die "cannot use options 'storage' and 'dumpdir' at the same time\n";
 458     }
 459
 460     if (!$opts->{dumpdir} && !$opts->{storage}) {
 461         $opts->{storage} = 'local';
 462     }
 463
 464     my $errors = '';
 465
 466     if ($opts->{storage}) {
 467         my $info = eval { storage_info ($opts->{storage}) };
 468         if (my $err = $@) {
 469             $errors .= "could not get storage information for '$opts->{storage}': $err";
 470         } else {
 471             $opts->{dumpdir} = $info->{dumpdir};
 472             $opts->{scfg} = $info->{scfg};
 473             $opts->{pbs} = $info->{pbs};
 474
 475             if (!defined($opts->{'prune-backups'}) && !defined($opts->{maxfiles})) {
 476                 $opts->{'prune-backups'} = $info->{'prune-backups'};
 477                 $opts->{maxfiles} = $info->{maxfiles};
 478             }
 479         }
 480     } elsif ($opts->{dumpdir}) {
 481         $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
 482             if ! -d $opts->{dumpdir};
 483     } else {
 484         die "internal error";
 485     }
 486
 487     if (!defined($opts->{'prune-backups'})) {
 488         $opts->{maxfiles} //= $defaults->{maxfiles};
 489         $opts->{'prune-backups'} = { 'keep-last' => $opts->{maxfiles} };
 490         delete $opts->{maxfiles};
 491     }
 492
 493     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 494         $errors .= "\n" if $errors;
 495         $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
 496     }
 497
 498     if ($errors) {
 499         eval { $self->sendmail([], 0, $errors); };
 500         debugmsg ('err', $@) if $@;
 501         die "$errors\n";
 502     }
 503
 504     return $self;
 505 }
 506
 507 sub get_mount_info {
 508     my ($dir) = @_;
 509
 510     # Note: df 'available' can be negative, and percentage set to '-'
 511
 512     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 513
 514     my $res;
 515
 516     my $parser = sub {
 517         my $line = shift;
 518         if (my ($fsid, $fstype, undef, $mp) = $line =~
 519             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 520             $res = {
 521                 device => $fsid,
 522                 fstype => $fstype,
 523                 mountpoint => $mp,
 524             };
 525         }
 526     };
 527
 528     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 529     warn $@ if $@;
 530
 531     return $res;
 532 }
 533
 534 sub getlock {
 535     my ($self, $upid) = @_;
 536
 537     my $fh;
 538
 539     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 540
 541     die "missing UPID" if !$upid; # should not happen
 542
 543     if (!open (SERVER_FLCK, ">>$lockfile")) {
 544         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 545         die "can't open lock on file '$lockfile' - $!";
 546     }
 547
 548     if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 549
 550         if (!$maxwait) {
 551             debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
 552             die "can't acquire lock '$lockfile' (wait = 0)";
 553         }
 554
 555         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 556
 557         eval {
 558             alarm ($maxwait * 60);
 559
 560             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 561
 562             if (!flock (SERVER_FLCK, LOCK_EX)) {
 563                 my $err = $!;
 564                 close (SERVER_FLCK);
 565                 alarm (0);
 566                 die "$err\n";
 567             }
 568             alarm (0);
 569         };
 570         alarm (0);
 571
 572         my $err = $@;
 573
 574         if ($err) {
 575             debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
 576             die "can't acquire lock '$lockfile' - $err";
 577         }
 578
 579         debugmsg('info', "got global lock", undef, 1);
 580     }
 581
 582     PVE::Tools::file_set_contents($pidfile, $upid);
 583 }
 584
 585 sub run_hook_script {
 586     my ($self, $phase, $task, $logfd) = @_;
 587
 588     my $opts = $self->{opts};
 589
 590     my $script = $opts->{script};
 591     return if !$script;
 592
 593     if (!-x $script) {
 594         die "The hook script '$script' is not executable.\n";
 595     }
 596
 597     my $cmd = "$script $phase";
 598
 599     $cmd .= " $task->{mode} $task->{vmid}" if ($task);
 600
 601     local %ENV;
 602     # set immutable opts directly (so they are available in all phases)
 603     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 604     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 605
 606     foreach my $ek (qw(vmtype hostname target logfile)) {
 607         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 608     }
 609     # FIXME: for backwards compatibility - drop with PVE 7.0
 610     $ENV{TARFILE} = $task->{target} if $task->{target};
 611
 612     run_command ($logfd, $cmd);
 613 }
 614
 615 sub compressor_info {
 616     my ($opts) = @_;
 617     my $opt_compress = $opts->{compress};
 618
 619     if (!$opt_compress || $opt_compress eq '0') {
 620         return undef;
 621     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 622         return ('lzop', 'lzo');
 623     } elsif ($opt_compress eq 'gzip') {
 624         if ($opts->{pigz} > 0) {
 625             my $pigz_threads = $opts->{pigz};
 626             if ($pigz_threads == 1) {
 627                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 628                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 629             }
 630             return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
 631         } else {
 632             return ('gzip --rsyncable', 'gz');
 633         }
 634     } elsif ($opt_compress eq 'zstd') {
 635         my $zstd_threads = $opts->{zstd} // 1;
 636         if ($zstd_threads == 0) {
 637             my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 638             $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
 639         }
 640         return ("zstd --rsyncable --threads=${zstd_threads}", 'zst');
 641     } else {
 642         die "internal error - unknown compression option '$opt_compress'";
 643     }
 644 }
 645
 646 sub get_backup_file_list {
 647     my ($dir, $bkname, $exclude_fn) = @_;
 648
 649     my $bklist = [];
 650     foreach my $fn (<$dir/${bkname}-*>) {
 651         next if $exclude_fn && $fn eq $exclude_fn;
 652
 653         my $archive_info = eval { PVE::Storage::archive_info($fn) } // {};
 654         if ($archive_info->{is_std_name}) {
 655             my $filename = $archive_info->{filename};
 656             my $backup = {
 657                 'path' => "$dir/$filename",
 658                 'ctime' => $archive_info->{ctime},
 659             };
 660             push @{$bklist}, $backup;
 661         }
 662     }
 663
 664     return $bklist;
 665 }
 666
 667 sub exec_backup_task {
 668     my ($self, $task) = @_;
 669
 670     my $opts = $self->{opts};
 671
 672     my $cfg = PVE::Storage::config();
 673     my $vmid = $task->{vmid};
 674     my $plugin = $task->{plugin};
 675
 676     $task->{backup_time} = time();
 677
 678     my $pbs_group_name;
 679     my $pbs_snapshot_name;
 680
 681     my $vmstarttime = time ();
 682
 683     my $logfd;
 684
 685     my $cleanup = {};
 686
 687     my $log_vm_online_again = sub {
 688         return if !defined($task->{vmstoptime});
 689         $task->{vmconttime} //= time();
 690         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 691         $delay = '<1' if $delay < 1;
 692         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 693     };
 694
 695     eval {
 696         die "unable to find VM '$vmid'\n" if !$plugin;
 697
 698         my $vmtype = $plugin->type();
 699
 700         if ($self->{opts}->{pbs}) {
 701             if ($vmtype eq 'lxc') {
 702                 $pbs_group_name = "ct/$vmid";
 703             } elsif  ($vmtype eq 'qemu') {
 704                 $pbs_group_name = "vm/$vmid";
 705             } else {
 706                 die "pbs backup not implemented for plugin type '$vmtype'\n";
 707             }
 708             my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
 709             $pbs_snapshot_name = "$pbs_group_name/$btime";
 710         }
 711
 712         # for now we deny backups of a running ha managed service in *stop* mode
 713         # as it interferes with the HA stack (started services should not stop).
 714         if ($opts->{mode} eq 'stop' &&
 715             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 716         {
 717             die "Cannot execute a backup with stop mode on a HA managed and".
 718                 " enabled Service. Use snapshot mode or disable the Service.\n";
 719         }
 720
 721         my $tmplog = "$logdir/$vmtype-$vmid.log";
 722
 723         my $bkname = "vzdump-$vmtype-$vmid";
 724         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
 725
 726         my $prune_options = $opts->{'prune-backups'};
 727
 728         my $backup_limit = 0;
 729         foreach my $keep (values %{$prune_options}) {
 730             $backup_limit += $keep;
 731         }
 732
 733         if ($backup_limit && !$opts->{remove}) {
 734             my $count;
 735             if ($self->{opts}->{pbs}) {
 736                 my $res = PVE::Storage::PBSPlugin::run_client_cmd($opts->{scfg}, $opts->{storage}, 'snapshots', $pbs_group_name);
 737                 $count = scalar(@$res);
 738             } else {
 739                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 740                 $count = scalar(@$bklist);
 741             }
 742             die "There is a max backup limit of $backup_limit enforced by the".
 743                 " target storage or the vzdump parameters.".
 744                 " Either increase the limit or delete old backup(s).\n"
 745                 if $count >= $backup_limit;
 746         }
 747
 748         if (!$self->{opts}->{pbs}) {
 749             $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 750         }
 751
 752         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 753         my ($comp, $comp_ext) = compressor_info($opts);
 754         if ($comp && $comp_ext) {
 755             $ext .= ".${comp_ext}";
 756         }
 757
 758         if ($self->{opts}->{pbs}) {
 759             die "unable to pipe backup to stdout\n" if $opts->{stdout};
 760             $task->{target} = $pbs_snapshot_name;
 761         } else {
 762             if ($opts->{stdout}) {
 763                 $task->{target} = '-';
 764             } else {
 765                 $task->{target} = $task->{tmptar} = "$opts->{dumpdir}/$basename$ext";
 766                 $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 767                 unlink $task->{tmptar};
 768             }
 769         }
 770
 771         $task->{vmtype} = $vmtype;
 772
 773         my $pid = $$;
 774         if ($opts->{tmpdir}) {
 775             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp${pid}_$vmid/";
 776         } elsif ($self->{opts}->{pbs}) {
 777             $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 778         } else {
 779             # dumpdir is posix? then use it as temporary dir
 780             my $info = get_mount_info($opts->{dumpdir});
 781             if ($vmtype eq 'qemu' ||
 782                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 783                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 784             } else {
 785                 $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 786                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 787                           "using $task->{tmpdir} for temporary files", $logfd);
 788             }
 789         }
 790
 791         rmtree $task->{tmpdir};
 792         mkdir $task->{tmpdir};
 793         -d $task->{tmpdir} ||
 794             die "unable to create temporary directory '$task->{tmpdir}'";
 795
 796         $logfd = IO::File->new (">$tmplog") ||
 797             die "unable to create log file '$tmplog'";
 798
 799         $task->{dumpdir} = $opts->{dumpdir};
 800         $task->{storeid} = $opts->{storage};
 801         $task->{scfg} = $opts->{scfg};
 802         $task->{tmplog} = $tmplog;
 803
 804         unlink $task->{logfile} if defined($task->{logfile});
 805
 806         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 807         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
 808
 809         $plugin->set_logfd ($logfd);
 810
 811         # test is VM is running
 812         my ($running, $status_text) = $plugin->vm_status ($vmid);
 813
 814         debugmsg ('info', "status = ${status_text}", $logfd);
 815
 816         # lock VM (prevent config changes)
 817         $plugin->lock_vm ($vmid);
 818
 819         $cleanup->{unlock} = 1;
 820
 821         # prepare
 822
 823         my $mode = $running ? $task->{mode} : 'stop';
 824
 825         if ($mode eq 'snapshot') {
 826             my %saved_task = %$task;
 827             eval { $plugin->prepare ($task, $vmid, $mode); };
 828             if (my $err = $@) {
 829                 die $err if $err !~ m/^mode failure/;
 830                 debugmsg ('info',  $err, $logfd);
 831                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 832                 $mode = 'suspend'; # so prepare is called again below
 833                 %$task = %saved_task;
 834             }
 835         }
 836
 837         $cleanup->{prepared} = 1;
 838
 839         $task->{mode} = $mode;
 840
 841         debugmsg ('info', "backup mode: $mode", $logfd);
 842
 843         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
 844             if $opts->{bwlimit};
 845
 846         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 847
 848         if ($mode eq 'stop') {
 849
 850             $plugin->prepare ($task, $vmid, $mode);
 851
 852             $self->run_hook_script ('backup-start', $task, $logfd);
 853
 854             if ($running) {
 855                 debugmsg ('info', "stopping vm", $logfd);
 856                 $task->{vmstoptime} = time();
 857                 $self->run_hook_script ('pre-stop', $task, $logfd);
 858                 $plugin->stop_vm ($task, $vmid);
 859                 $cleanup->{restart} = 1;
 860             }
 861
 862
 863         } elsif ($mode eq 'suspend') {
 864
 865             $plugin->prepare ($task, $vmid, $mode);
 866
 867             $self->run_hook_script ('backup-start', $task, $logfd);
 868
 869             if ($vmtype eq 'lxc') {
 870                 # pre-suspend rsync
 871                 $plugin->copy_data_phase1($task, $vmid);
 872             }
 873
 874             debugmsg ('info', "suspending guest", $logfd);
 875             $task->{vmstoptime} = time ();
 876             $self->run_hook_script ('pre-stop', $task, $logfd);
 877             $plugin->suspend_vm ($task, $vmid);
 878             $cleanup->{resume} = 1;
 879
 880             if ($vmtype eq 'lxc') {
 881                 # post-suspend rsync
 882                 $plugin->copy_data_phase2($task, $vmid);
 883
 884                 debugmsg ('info', "resuming guest", $logfd);
 885                 $cleanup->{resume} = 0;
 886                 $self->run_hook_script('pre-restart', $task, $logfd);
 887                 $plugin->resume_vm($task, $vmid);
 888                 $self->run_hook_script('post-restart', $task, $logfd);
 889                 $log_vm_online_again->();
 890             }
 891
 892         } elsif ($mode eq 'snapshot') {
 893
 894             $self->run_hook_script ('backup-start', $task, $logfd);
 895
 896             my $snapshot_count = $task->{snapshot_count} || 0;
 897
 898             $self->run_hook_script ('pre-stop', $task, $logfd);
 899
 900             if ($snapshot_count > 1) {
 901                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 902                 $task->{vmstoptime} = time ();
 903                 $plugin->suspend_vm ($task, $vmid);
 904                 $cleanup->{resume} = 1;
 905             }
 906
 907             $plugin->snapshot ($task, $vmid);
 908
 909             $self->run_hook_script ('pre-restart', $task, $logfd);
 910
 911             if ($snapshot_count > 1) {
 912                 debugmsg ('info', "resume vm", $logfd);
 913                 $cleanup->{resume} = 0;
 914                 $plugin->resume_vm ($task, $vmid);
 915                 $log_vm_online_again->();
 916             }
 917
 918             $self->run_hook_script ('post-restart', $task, $logfd);
 919
 920         } else {
 921             die "internal error - unknown mode '$mode'\n";
 922         }
 923
 924         # assemble archive image
 925         $plugin->assemble ($task, $vmid);
 926
 927         # produce archive
 928
 929         if ($opts->{stdout}) {
 930             debugmsg ('info', "sending archive to stdout", $logfd);
 931             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 932             $self->run_hook_script ('backup-end', $task, $logfd);
 933             return;
 934         }
 935
 936         my $archive_txt = $self->{opts}->{pbs} ? 'Proxmox Backup Server' : 'vzdump';
 937         debugmsg('info', "creating $archive_txt archive '$task->{target}'", $logfd);
 938         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 939
 940         if ($self->{opts}->{pbs}) {
 941             # size is added to task struct in guest vzdump plugins
 942         } else {
 943             rename ($task->{tmptar}, $task->{target}) ||
 944                 die "unable to rename '$task->{tmptar}' to '$task->{target}'\n";
 945
 946             # determine size
 947             $task->{size} = (-s $task->{target}) || 0;
 948             my $cs = format_size ($task->{size});
 949             debugmsg ('info', "archive file size: $cs", $logfd);
 950         }
 951
 952         # purge older backup
 953         if ($opts->{remove}) {
 954             if (!defined($opts->{storage})) {
 955                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{target});
 956                 PVE::Storage::prune_mark_backup_group($bklist, $prune_options);
 957
 958                 foreach my $prune_entry (@{$bklist}) {
 959                     next if $prune_entry->{mark} ne 'remove';
 960
 961                     my $archive_path = $prune_entry->{path};
 962                     debugmsg ('info', "delete old backup '$archive_path'", $logfd);
 963                     PVE::Storage::archive_remove($archive_path);
 964                 }
 965             } else {
 966                 my $logfunc = sub { debugmsg($_[0], $_[1], $logfd) };
 967                 PVE::Storage::prune_backups($cfg, $opts->{storage}, $prune_options, $vmid, $vmtype, 0, $logfunc);
 968             }
 969         }
 970
 971         $self->run_hook_script ('backup-end', $task, $logfd);
 972     };
 973     my $err = $@;
 974
 975     if ($plugin) {
 976         # clean-up
 977
 978         if ($cleanup->{unlock}) {
 979             eval { $plugin->unlock_vm ($vmid); };
 980             warn $@ if $@;
 981         }
 982
 983         if ($cleanup->{prepared}) {
 984             # only call cleanup when necessary (when prepare was executed)
 985             eval { $plugin->cleanup ($task, $vmid) };
 986             warn $@ if $@;
 987         }
 988
 989         eval { $plugin->set_logfd (undef); };
 990         warn $@ if $@;
 991
 992         if ($cleanup->{resume} || $cleanup->{restart}) {
 993             eval {
 994                 $self->run_hook_script ('pre-restart', $task, $logfd);
 995                 if ($cleanup->{resume}) {
 996                     debugmsg ('info', "resume vm", $logfd);
 997                     $plugin->resume_vm ($task, $vmid);
 998                 } else {
 999                     my $running = $plugin->vm_status($vmid);
1000                     if (!$running) {
1001                         debugmsg ('info', "restarting vm", $logfd);
1002                         $plugin->start_vm ($task, $vmid);
1003                     }
1004                 }
1005                 $self->run_hook_script ('post-restart', $task, $logfd);
1006             };
1007             my $err = $@;
1008             if ($err) {
1009                 warn $err;
1010             } else {
1011                 $log_vm_online_again->();
1012             }
1013         }
1014     }
1015
1016     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
1017     warn $@ if $@;
1018
1019     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
1020     warn $@ if $@;
1021
1022     my $delay = $task->{backuptime} = time () - $vmstarttime;
1023
1024     if ($err) {
1025         $task->{state} = 'err';
1026         $task->{msg} = $err;
1027         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1028         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
1029
1030         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1031
1032     } else {
1033         $task->{state} = 'ok';
1034         my $tstr = format_time ($delay);
1035         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1036         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1037     }
1038
1039     close ($logfd) if $logfd;
1040
1041     if ($task->{tmplog}) {
1042         if ($self->{opts}->{pbs}) {
1043             if ($task->{state} eq 'ok') {
1044                 my $param = [$pbs_snapshot_name, $task->{tmplog}];
1045                 PVE::Storage::PBSPlugin::run_raw_client_cmd(
1046                     $opts->{scfg}, $opts->{storage}, 'upload-log', $param, errmsg => "upload log failed");
1047             }
1048         } elsif ($task->{logfile}) {
1049             system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1050         }
1051     }
1052
1053     eval { $self->run_hook_script ('log-end', $task); };
1054
1055     die $err if $err && $err =~ m/^interrupted by signal$/;
1056 }
1057
1058 sub exec_backup {
1059     my ($self, $rpcenv, $authuser) = @_;
1060
1061     my $opts = $self->{opts};
1062
1063     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1064
1065     if (scalar(@{$self->{skiplist}})) {
1066         my $skip_string = join(', ', sort { $a <=> $b } @{$self->{skiplist}});
1067         debugmsg ('info', "skip external VMs: $skip_string");
1068     }
1069
1070     my $tasklist = [];
1071     my $vzdump_plugins =  {};
1072     foreach my $plugin (@{$self->{plugins}}) {
1073         my $type = $plugin->type();
1074         next if exists $vzdump_plugins->{$type};
1075         $vzdump_plugins->{$type} = $plugin;
1076     }
1077
1078     my $vmlist = PVE::Cluster::get_vmlist();
1079     my $vmids = [ sort { $a <=> $b } @{$opts->{vmids}} ];
1080     foreach my $vmid (@{$vmids}) {
1081         my $plugin;
1082         if (defined($vmlist->{ids}->{$vmid})) {
1083             my $guest_type = $vmlist->{ids}->{$vmid}->{type};
1084             $plugin = $vzdump_plugins->{$guest_type};
1085             next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all});
1086         }
1087         push @$tasklist, {
1088             mode => $opts->{mode},
1089             plugin => $plugin,
1090             state => 'todo',
1091             vmid => $vmid,
1092         };
1093     }
1094
1095     # Use in-memory files for the outer hook logs to pass them to sendmail.
1096     my $job_start_log = '';
1097     my $job_end_log = '';
1098     open my $job_start_fd, '>', \$job_start_log;
1099     open my $job_end_fd, '>', \$job_end_log;
1100
1101     my $starttime = time();
1102     my $errcount = 0;
1103     eval {
1104
1105         $self->run_hook_script ('job-start', undef, $job_start_fd);
1106
1107         foreach my $task (@$tasklist) {
1108             $self->exec_backup_task ($task);
1109             $errcount += 1 if $task->{state} ne 'ok';
1110         }
1111
1112         $self->run_hook_script ('job-end', undef, $job_end_fd);
1113     };
1114     my $err = $@;
1115
1116     $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1117
1118     if ($err) {
1119         debugmsg ('err', "Backup job failed - $err", undef, 1);
1120     } else {
1121         if ($errcount) {
1122             debugmsg ('info', "Backup job finished with errors", undef, 1);
1123         } else {
1124             debugmsg ('info', "Backup job finished successfully", undef, 1);
1125         }
1126     }
1127
1128     close $job_start_fd;
1129     close $job_end_fd;
1130
1131     my $totaltime = time() - $starttime;
1132
1133     eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); };
1134     debugmsg ('err', $@) if $@;
1135
1136     die $err if $err;
1137
1138     die "job errors\n" if $errcount;
1139
1140     unlink $pidfile;
1141 }
1142
1143
1144 sub option_exists {
1145     my $key = shift;
1146     return defined($confdesc->{$key});
1147 }
1148
1149 sub verify_vzdump_parameters {
1150     my ($param, $check_missing) = @_;
1151
1152     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1153         if $param->{all} && $param->{vmid};
1154
1155     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1156         if $param->{exclude} && $param->{vmid};
1157
1158     raise_param_exc({ pool => "option conflicts with option 'vmid'"})
1159         if $param->{pool} && $param->{vmid};
1160
1161     raise_param_exc({ 'prune-backups' => "option conflicts with option 'maxfiles'"})
1162         if defined($param->{'prune-backups'}) && defined($param->{maxfiles});
1163
1164     $param->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $param->{'prune-backups'})
1165         if defined($param->{'prune-backups'});
1166
1167     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
1168
1169     warn "option 'size' is deprecated and will be removed in a future " .
1170          "release, please update your script/configuration!\n"
1171         if defined($param->{size});
1172
1173     return if !$check_missing;
1174
1175     raise_param_exc({ vmid => "property is missing"})
1176         if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
1177
1178 }
1179
1180 sub stop_running_backups {
1181     my($self) = @_;
1182
1183     my $upid = PVE::Tools::file_read_firstline($pidfile);
1184     return if !$upid;
1185
1186     my $task = PVE::Tools::upid_decode($upid);
1187
1188     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1189         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1190         kill(15, $task->{pid});
1191         # wait max 15 seconds to shut down (else, do nothing for now)
1192         my $i;
1193         for ($i = 15; $i > 0; $i--) {
1194             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1195             sleep (1);
1196         }
1197         die "stopping backup process $task->{pid} failed\n" if $i == 0;
1198     }
1199 }
1200
1201 sub get_included_guests {
1202     my ($job) = @_;
1203
1204     my $vmids = [];
1205     my $vmids_per_node = {};
1206
1207     my $vmlist = PVE::Cluster::get_vmlist();
1208
1209     if ($job->{pool}) {
1210         $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool});
1211     } elsif ($job->{vmid}) {
1212         $vmids = [ split_list($job->{vmid}) ];
1213     } elsif ($job->{all}) {
1214         # all or exclude
1215         my $exclude = check_vmids(split_list($job->{exclude}));
1216         my $excludehash = { map { $_ => 1 } @$exclude };
1217
1218         for my $id (keys %{$vmlist->{ids}}) {
1219             next if $excludehash->{$id};
1220             push @$vmids, $id;
1221         }
1222     } else {
1223         return $vmids_per_node;
1224     }
1225     $vmids = check_vmids(@$vmids);
1226
1227     for my $vmid (@$vmids) {
1228         if (defined($vmlist->{ids}->{$vmid})) {
1229             my $node = $vmlist->{ids}->{$vmid}->{node};
1230             next if (defined $job->{node} && $job->{node} ne $node);
1231
1232             push @{$vmids_per_node->{$node}}, $vmid;
1233         } else {
1234             push @{$vmids_per_node->{''}}, $vmid;
1235         }
1236     }
1237
1238     return $vmids_per_node;
1239 }
1240
1241 1;