PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Path;
   8 use IO::File;
   9 use IO::Select;
  10 use IPC::Open3;
  11 use POSIX qw(strftime);
  12 use Time::Local;
  13
  14 use PVE::Cluster qw(cfs_read_file);
  15 use PVE::DataCenterConfig;
  16 use PVE::Exception qw(raise_param_exc);
  17 use PVE::HA::Config;
  18 use PVE::HA::Env::PVE2;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::RPCEnvironment;
  21 use PVE::Storage;
  22 use PVE::VZDump::Common;
  23 use PVE::VZDump::Plugin;
  24 use PVE::Tools qw(extract_param split_list);
  25 use PVE::API2Tools;
  26
  27 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  28
  29 my $lockfile = '/var/run/vzdump.lock';
  30 my $pidfile = '/var/run/vzdump.pid';
  31 my $logdir = '/var/log/vzdump';
  32
  33 my @plugins = qw();
  34
  35 my $confdesc = PVE::VZDump::Common::get_confdesc();
  36
  37 # Load available plugins
  38 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
  39 foreach my $plug (@pve_vzdump_classes) {
  40     my $filename = "/usr/share/perl5/$plug.pm";
  41     $filename =~ s!::!/!g;
  42     if (-f $filename) {
  43         eval { require $filename; };
  44         if (!$@) {
  45             $plug->import ();
  46             push @plugins, $plug;
  47         } else {
  48             die $@;
  49         }
  50     }
  51 }
  52
  53 # helper functions
  54
  55 sub debugmsg {
  56     my ($mtype, $msg, $logfd, $syslog) = @_;
  57
  58     PVE::VZDump::Plugin::debugmsg(@_);
  59 }
  60
  61 sub run_command {
  62     my ($logfd, $cmdstr, %param) = @_;
  63
  64     my $logfunc = sub {
  65         my $line = shift;
  66         debugmsg ('info', $line, $logfd);
  67     };
  68
  69     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
  70 }
  71
  72 sub storage_info {
  73     my $storage = shift;
  74
  75     my $cfg = PVE::Storage::config();
  76     my $scfg = PVE::Storage::storage_config($cfg, $storage);
  77     my $type = $scfg->{type};
  78
  79     die "can't use storage type '$type' for backup\n"
  80         if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'
  81               || $type eq 'cifs' || $type eq 'cephfs' || $type eq 'pbs'));
  82     die "can't use storage '$storage' for backups - wrong content type\n"
  83         if (!$scfg->{content}->{backup});
  84
  85     PVE::Storage::activate_storage($cfg, $storage);
  86
  87     my $info = {
  88         scfg => $scfg,
  89         maxfiles => $scfg->{maxfiles},
  90     };
  91
  92     if ($type eq 'pbs') {
  93         $info->{pbs} = 1;
  94     } else {
  95         $info->{dumpdir} = PVE::Storage::get_backup_dir($cfg, $storage);
  96     }
  97
  98     return $info;
  99 }
 100
 101 sub format_size {
 102     my $size = shift;
 103
 104     my $kb = $size / 1024;
 105
 106     if ($kb < 1024) {
 107         return int ($kb) . "KB";
 108     }
 109
 110     my $mb = $size / (1024*1024);
 111     if ($mb < 1024) {
 112         return int ($mb) . "MB";
 113     }
 114     my $gb = $mb / 1024;
 115     if ($gb < 1024) {
 116         return sprintf ("%.2fGB", $gb);
 117     }
 118     my $tb = $gb / 1024;
 119     return sprintf ("%.2fTB", $tb);
 120 }
 121
 122 sub format_time {
 123     my $seconds = shift;
 124
 125     my $hours = int ($seconds/3600);
 126     $seconds = $seconds - $hours*3600;
 127     my $min = int ($seconds/60);
 128     $seconds = $seconds - $min*60;
 129
 130     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 131 }
 132
 133 sub encode8bit {
 134     my ($str) = @_;
 135
 136     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 137
 138     return $str;
 139 }
 140
 141 sub escape_html {
 142     my ($str) = @_;
 143
 144     $str =~ s/&/&amp;/g;
 145     $str =~ s/</&lt;/g;
 146     $str =~ s/>/&gt;/g;
 147
 148     return $str;
 149 }
 150
 151 sub check_bin {
 152     my ($bin)  = @_;
 153
 154     foreach my $p (split (/:/, $ENV{PATH})) {
 155         my $fn = "$p/$bin";
 156         if (-x $fn) {
 157             return $fn;
 158         }
 159     }
 160
 161     die "unable to find command '$bin'\n";
 162 }
 163
 164 sub check_vmids {
 165     my (@vmids) = @_;
 166
 167     my $res = [];
 168     for my $vmid (sort {$a <=> $b} @vmids) {
 169         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 170         $vmid = int ($vmid); # remove leading zeros
 171         next if !$vmid;
 172         push @$res, $vmid;
 173     }
 174
 175     return $res;
 176 }
 177
 178
 179 sub read_vzdump_defaults {
 180
 181     my $fn = "/etc/vzdump.conf";
 182
 183     my $defaults = {
 184         map {
 185             my $default = $confdesc->{$_}->{default};
 186              defined($default) ? ($_ => $default) : ()
 187         } keys %$confdesc
 188     };
 189
 190     my $raw;
 191     eval { $raw = PVE::Tools::file_get_contents($fn); };
 192     return $defaults if $@;
 193
 194     my $conf_schema = { type => 'object', properties => $confdesc, };
 195     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 196     if (my $excludes = $res->{'exclude-path'}) {
 197         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 198     }
 199     if (defined($res->{mailto})) {
 200         my @mailto = split_list($res->{mailto});
 201         $res->{mailto} = [ @mailto ];
 202     }
 203
 204     foreach my $key (keys %$defaults) {
 205         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 206     }
 207
 208     return $res;
 209 }
 210
 211 use constant MAX_MAIL_SIZE => 1024*1024;
 212 sub sendmail {
 213     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 214
 215     my $opts = $self->{opts};
 216
 217     my $mailto = $opts->{mailto};
 218
 219     return if !($mailto && scalar(@$mailto));
 220
 221     my $cmdline = $self->{cmdline};
 222
 223     my $ecount = 0;
 224     foreach my $task (@$tasklist) {
 225         $ecount++ if $task->{state} ne 'ok';
 226         chomp $task->{msg} if $task->{msg};
 227         $task->{backuptime} = 0 if !$task->{backuptime};
 228         $task->{size} = 0 if !$task->{size};
 229         $task->{target} = 'unknown' if !$task->{target};
 230         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 231
 232         if ($task->{state} eq 'todo') {
 233             $task->{msg} = 'aborted';
 234         }
 235     }
 236
 237     my $notify = $opts->{mailnotification} || 'always';
 238     return if (!$ecount && !$err && ($notify eq 'failure'));
 239
 240     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 241     if ($err) {
 242         if ($err =~ /\n/) {
 243             $stat .= ": multiple problems";
 244         } else {
 245             $stat .= ": $err";
 246             $err = undef;
 247         }
 248     }
 249
 250     my $hostname = `hostname -f` || PVE::INotify::nodename();
 251     chomp $hostname;
 252
 253     # text part
 254     my $text = $err ? "$err\n\n" : '';
 255     $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", qw(VMID STATUS TIME SIZE FILENAME));
 256     foreach my $task (@$tasklist) {
 257         my $vmid = $task->{vmid};
 258         if  ($task->{state} eq 'ok') {
 259
 260             $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", $vmid,
 261                                 $task->{state},
 262                                 format_time($task->{backuptime}),
 263                                 format_size ($task->{size}),
 264                                 $task->{target});
 265         } else {
 266             $text .= sprintf ("%-10s %-6s %10s %8.2fMB  %s\n", $vmid,
 267                                 $task->{state},
 268                                 format_time($task->{backuptime}),
 269                                 0, '-');
 270         }
 271     }
 272
 273     my $text_log_part;
 274     $text_log_part .= "\nDetailed backup logs:\n\n";
 275     $text_log_part .= "$cmdline\n\n";
 276
 277     $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
 278     foreach my $task (@$tasklist) {
 279         my $vmid = $task->{vmid};
 280         my $log = $task->{tmplog};
 281         if (!$log) {
 282             $text_log_part .= "$vmid: no log available\n\n";
 283             next;
 284         }
 285         if (open (TMP, "$log")) {
 286             while (my $line = <TMP>) {
 287                 next if $line =~ /^status: \d+/; # not useful in mails
 288                 $text_log_part .= encode8bit ("$vmid: $line");
 289             }
 290         } else {
 291             $text_log_part .= "$vmid: Could not open log file\n\n";
 292         }
 293         close (TMP);
 294         $text_log_part .= "\n";
 295     }
 296     $text_log_part .= $detail_post if defined($detail_post);
 297
 298     # html part
 299     my $html = "<html><body>\n";
 300     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 301     $html .= "<table border=1 cellpadding=3>\n";
 302     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 303
 304     my $ssize = 0;
 305
 306     foreach my $task (@$tasklist) {
 307         my $vmid = $task->{vmid};
 308         my $name = $task->{hostname};
 309
 310         if  ($task->{state} eq 'ok') {
 311
 312             $ssize += $task->{size};
 313
 314             $html .= sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 315                                 $vmid, $name,
 316                                 format_time($task->{backuptime}),
 317                                 format_size ($task->{size}),
 318                                 escape_html ($task->{target}));
 319         } else {
 320             $html .= sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 321                                 $vmid, $name, format_time($task->{backuptime}),
 322                                 escape_html ($task->{msg}));
 323         }
 324     }
 325
 326     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 327  format_time ($totaltime), format_size ($ssize));
 328
 329     $html .= "\n</table><br><br>\n";
 330     my $html_log_part;
 331     $html_log_part .= "Detailed backup logs:<br /><br />\n";
 332     $html_log_part .= "<pre>\n";
 333     $html_log_part .= escape_html($cmdline) . "\n\n";
 334
 335     $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 336     foreach my $task (@$tasklist) {
 337         my $vmid = $task->{vmid};
 338         my $log = $task->{tmplog};
 339         if (!$log) {
 340             $html_log_part .= "$vmid: no log available\n\n";
 341             next;
 342         }
 343         if (open (TMP, "$log")) {
 344             while (my $line = <TMP>) {
 345                 next if $line =~ /^status: \d+/; # not useful in mails
 346                 if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 347                     $html_log_part .= encode8bit ("$vmid: <font color=red>".
 348                         escape_html ($line) . "</font>");
 349                 } else {
 350                     $html_log_part .= encode8bit ("$vmid: " . escape_html ($line));
 351                 }
 352             }
 353         } else {
 354             $html_log_part .= "$vmid: Could not open log file\n\n";
 355         }
 356         close (TMP);
 357         $html_log_part .= "\n";
 358     }
 359     $html_log_part .= escape_html($detail_post) if defined($detail_post);
 360     $html_log_part .= "</pre>";
 361     my $html_end .= "\n</body></html>\n";
 362     # end html part
 363
 364     if (length($text) + length($text_log_part) +
 365         length($html) + length($html_log_part) < MAX_MAIL_SIZE)
 366     {
 367         $html .= $html_log_part;
 368         $text .= $text_log_part;
 369     } else {
 370         my $msg = "Log output was too long to be sent by mail. ".
 371             "See Task History for details!\n";
 372         $text .= $msg;
 373         $html .= "<p>$msg</p>";
 374         $html .= $html_end;
 375     }
 376
 377     my $subject = "vzdump backup status ($hostname) : $stat";
 378
 379     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 380     my $mailfrom = $dcconf->{email_from} || "root";
 381
 382     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 383 };
 384
 385 sub new {
 386     my ($class, $cmdline, $opts, $skiplist) = @_;
 387
 388     mkpath $logdir;
 389
 390     check_bin ('cp');
 391     check_bin ('df');
 392     check_bin ('sendmail');
 393     check_bin ('rsync');
 394     check_bin ('tar');
 395     check_bin ('mount');
 396     check_bin ('umount');
 397     check_bin ('cstream');
 398     check_bin ('ionice');
 399
 400     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 401         check_bin ('lvcreate');
 402         check_bin ('lvs');
 403         check_bin ('lvremove');
 404     }
 405
 406     my $defaults = read_vzdump_defaults();
 407
 408     $opts->{remove} = 1 if !defined($opts->{remove});
 409
 410     foreach my $k (keys %$defaults) {
 411         next if $k eq 'exclude-path' || $k eq 'maxfiles'; # dealt with separately
 412         if ($k eq 'dumpdir' || $k eq 'storage') {
 413             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 414                 !defined ($opts->{storage});
 415         } else {
 416             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 417         }
 418     }
 419
 420     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 421     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 422
 423     $skiplist = [] if !$skiplist;
 424     my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist };
 425
 426     my $findexcl = $self->{findexcl} = [];
 427     if ($defaults->{'exclude-path'}) {
 428         push @$findexcl, @{$defaults->{'exclude-path'}};
 429     }
 430
 431     if ($opts->{'exclude-path'}) {
 432         push @$findexcl, @{$opts->{'exclude-path'}};
 433     }
 434
 435     if ($opts->{stdexcludes}) {
 436         push @$findexcl, '/tmp/?*',
 437                          '/var/tmp/?*',
 438                          '/var/run/?*.pid';
 439     }
 440
 441     foreach my $p (@plugins) {
 442
 443         my $pd = $p->new ($self);
 444
 445         push @{$self->{plugins}}, $pd;
 446     }
 447
 448     if (defined($opts->{storage}) && $opts->{stdout}) {
 449         die "cannot use options 'storage' and 'stdout' at the same time\n";
 450     } elsif (defined($opts->{storage}) && defined($opts->{dumpdir})) {
 451         die "cannot use options 'storage' and 'dumpdir' at the same time\n";
 452     }
 453
 454     if (!$opts->{dumpdir} && !$opts->{storage}) {
 455         $opts->{storage} = 'local';
 456     }
 457
 458     my $errors = '';
 459
 460     if ($opts->{storage}) {
 461         my $info = eval { storage_info ($opts->{storage}) };
 462         $errors .= "could not get storage information for '$opts->{storage}': $@"
 463             if ($@);
 464         $opts->{dumpdir} = $info->{dumpdir};
 465         $opts->{scfg} = $info->{scfg};
 466         $opts->{pbs} = $info->{pbs};
 467         $opts->{maxfiles} //= $info->{maxfiles};
 468     } elsif ($opts->{dumpdir}) {
 469         $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
 470             if ! -d $opts->{dumpdir};
 471     } else {
 472         die "internal error";
 473     }
 474
 475     $opts->{maxfiles} //= $defaults->{maxfiles};
 476
 477     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 478         $errors .= "\n" if $errors;
 479         $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
 480     }
 481
 482     if ($errors) {
 483         eval { $self->sendmail([], 0, $errors); };
 484         debugmsg ('err', $@) if $@;
 485         die "$errors\n";
 486     }
 487
 488     return $self;
 489 }
 490
 491 sub get_mount_info {
 492     my ($dir) = @_;
 493
 494     # Note: df 'available' can be negative, and percentage set to '-'
 495
 496     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 497
 498     my $res;
 499
 500     my $parser = sub {
 501         my $line = shift;
 502         if (my ($fsid, $fstype, undef, $mp) = $line =~
 503             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 504             $res = {
 505                 device => $fsid,
 506                 fstype => $fstype,
 507                 mountpoint => $mp,
 508             };
 509         }
 510     };
 511
 512     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 513     warn $@ if $@;
 514
 515     return $res;
 516 }
 517
 518 sub getlock {
 519     my ($self, $upid) = @_;
 520
 521     my $fh;
 522
 523     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 524
 525     die "missimg UPID" if !$upid; # should not happen
 526
 527     if (!open (SERVER_FLCK, ">>$lockfile")) {
 528         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 529         die "can't open lock on file '$lockfile' - $!";
 530     }
 531
 532     if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 533
 534         if (!$maxwait) {
 535             debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
 536             die "can't acquire lock '$lockfile' (wait = 0)";
 537         }
 538
 539         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 540
 541         eval {
 542             alarm ($maxwait * 60);
 543
 544             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 545
 546             if (!flock (SERVER_FLCK, LOCK_EX)) {
 547                 my $err = $!;
 548                 close (SERVER_FLCK);
 549                 alarm (0);
 550                 die "$err\n";
 551             }
 552             alarm (0);
 553         };
 554         alarm (0);
 555
 556         my $err = $@;
 557
 558         if ($err) {
 559             debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
 560             die "can't acquire lock '$lockfile' - $err";
 561         }
 562
 563         debugmsg('info', "got global lock", undef, 1);
 564     }
 565
 566     PVE::Tools::file_set_contents($pidfile, $upid);
 567 }
 568
 569 sub run_hook_script {
 570     my ($self, $phase, $task, $logfd) = @_;
 571
 572     my $opts = $self->{opts};
 573
 574     my $script = $opts->{script};
 575     return if !$script;
 576
 577     if (!-x $script) {
 578         die "The hook script '$script' is not executable.\n";
 579     }
 580
 581     my $cmd = "$script $phase";
 582
 583     $cmd .= " $task->{mode} $task->{vmid}" if ($task);
 584
 585     local %ENV;
 586     # set immutable opts directly (so they are available in all phases)
 587     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 588     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 589
 590     foreach my $ek (qw(vmtype hostname target logfile)) {
 591         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 592     }
 593     # FIXME: for backwards compatibility - drop with PVE 7.0
 594     $ENV{TARFILE} = $task->{target} if $task->{target};
 595
 596     run_command ($logfd, $cmd);
 597 }
 598
 599 sub compressor_info {
 600     my ($opts) = @_;
 601     my $opt_compress = $opts->{compress};
 602
 603     if (!$opt_compress || $opt_compress eq '0') {
 604         return undef;
 605     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 606         return ('lzop', 'lzo');
 607     } elsif ($opt_compress eq 'gzip') {
 608         if ($opts->{pigz} > 0) {
 609             my $pigz_threads = $opts->{pigz};
 610             if ($pigz_threads == 1) {
 611                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 612                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 613             }
 614             return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
 615         } else {
 616             return ('gzip --rsyncable', 'gz');
 617         }
 618     } elsif ($opt_compress eq 'zstd') {
 619         my $zstd_threads = $opts->{zstd} // 1;
 620         if ($zstd_threads == 0) {
 621             my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 622             $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
 623         }
 624         return ("zstd --rsyncable --threads=${zstd_threads}", 'zst');
 625     } else {
 626         die "internal error - unknown compression option '$opt_compress'";
 627     }
 628 }
 629
 630 sub get_backup_file_list {
 631     my ($dir, $bkname, $exclude_fn) = @_;
 632
 633     my $bklist = [];
 634     foreach my $fn (<$dir/${bkname}-*>) {
 635         next if $exclude_fn && $fn eq $exclude_fn;
 636         if ($fn =~ m!/(${bkname}-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?)))$!) {
 637             $fn = "$dir/$1"; # untaint
 638             my $t = timelocal ($7, $6, $5, $4, $3 - 1, $2);
 639             push @$bklist, [$fn, $t];
 640         }
 641     }
 642
 643     return $bklist;
 644 }
 645
 646 sub exec_backup_task {
 647     my ($self, $task) = @_;
 648
 649     my $opts = $self->{opts};
 650
 651     my $vmid = $task->{vmid};
 652     my $plugin = $task->{plugin};
 653     my $vmtype = $plugin->type();
 654
 655     $task->{backup_time} = time();
 656
 657     my $pbs_group_name;
 658     my $pbs_snapshot_name;
 659
 660     if ($self->{opts}->{pbs}) {
 661         if ($vmtype eq 'lxc') {
 662             $pbs_group_name = "ct/$vmid";
 663         } elsif  ($vmtype eq 'qemu') {
 664             $pbs_group_name = "vm/$vmid";
 665         } else {
 666             die "pbs backup not implemented for plugin type '$vmtype'\n";
 667         }
 668         my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
 669         $pbs_snapshot_name = "$pbs_group_name/$btime";
 670     }
 671
 672     my $vmstarttime = time ();
 673
 674     my $logfd;
 675
 676     my $cleanup = {};
 677
 678     my $log_vm_online_again = sub {
 679         return if !defined($task->{vmstoptime});
 680         $task->{vmconttime} //= time();
 681         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 682         $delay = '<1' if $delay < 1;
 683         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 684     };
 685
 686     eval {
 687         die "unable to find VM '$vmid'\n" if !$plugin;
 688
 689         # for now we deny backups of a running ha managed service in *stop* mode
 690         # as it interferes with the HA stack (started services should not stop).
 691         if ($opts->{mode} eq 'stop' &&
 692             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 693         {
 694             die "Cannot execute a backup with stop mode on a HA managed and".
 695                 " enabled Service. Use snapshot mode or disable the Service.\n";
 696         }
 697
 698         my $tmplog = "$logdir/$vmtype-$vmid.log";
 699
 700         my $bkname = "vzdump-$vmtype-$vmid";
 701         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
 702
 703         my $maxfiles = $opts->{maxfiles};
 704
 705         if ($maxfiles && !$opts->{remove}) {
 706             my $count;
 707             if ($self->{opts}->{pbs}) {
 708                 my $res = PVE::Storage::PBSPlugin::run_client_cmd($opts->{scfg}, $opts->{storage}, 'snapshots', $pbs_group_name);
 709                 $count = scalar(@$res);
 710             } else {
 711                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 712                 $count = scalar(@$bklist);
 713             }
 714             die "There is a max backup limit of ($maxfiles) enforced by the".
 715                 " target storage or the vzdump parameters.".
 716                 " Either increase the limit or delete old backup(s).\n"
 717                 if $count >= $maxfiles;
 718         }
 719
 720         if (!$self->{opts}->{pbs}) {
 721             $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 722         }
 723
 724         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 725         my ($comp, $comp_ext) = compressor_info($opts);
 726         if ($comp && $comp_ext) {
 727             $ext .= ".${comp_ext}";
 728         }
 729
 730         if ($self->{opts}->{pbs}) {
 731             die "unable to pipe backup to stdout\n" if $opts->{stdout};
 732             $task->{target} = $pbs_snapshot_name;
 733         } else {
 734             if ($opts->{stdout}) {
 735                 $task->{target} = '-';
 736             } else {
 737                 $task->{target} = $task->{tmptar} = "$opts->{dumpdir}/$basename$ext";
 738                 $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 739                 unlink $task->{tmptar};
 740             }
 741         }
 742
 743         $task->{vmtype} = $vmtype;
 744
 745         if ($opts->{tmpdir}) {
 746             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp$$";
 747         } elsif ($self->{opts}->{pbs}) {
 748             $task->{tmpdir} = "/var/tmp/vzdumptmp$$"; #fixme
 749         } else {
 750             # dumpdir is posix? then use it as temporary dir
 751             my $info = get_mount_info($opts->{dumpdir});
 752             if ($vmtype eq 'qemu' ||
 753                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 754                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 755             } else {
 756                 $task->{tmpdir} = "/var/tmp/vzdumptmp$$";
 757                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 758                           "using $task->{tmpdir} for temporary files", $logfd);
 759             }
 760         }
 761
 762         rmtree $task->{tmpdir};
 763         mkdir $task->{tmpdir};
 764         -d $task->{tmpdir} ||
 765             die "unable to create temporary directory '$task->{tmpdir}'";
 766
 767         $logfd = IO::File->new (">$tmplog") ||
 768             die "unable to create log file '$tmplog'";
 769
 770         $task->{dumpdir} = $opts->{dumpdir};
 771         $task->{storeid} = $opts->{storage};
 772         $task->{scfg} = $opts->{scfg};
 773         $task->{tmplog} = $tmplog;
 774
 775         unlink $task->{logfile} if defined($task->{logfile});
 776
 777         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 778         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
 779
 780         $plugin->set_logfd ($logfd);
 781
 782         # test is VM is running
 783         my ($running, $status_text) = $plugin->vm_status ($vmid);
 784
 785         debugmsg ('info', "status = ${status_text}", $logfd);
 786
 787         # lock VM (prevent config changes)
 788         $plugin->lock_vm ($vmid);
 789
 790         $cleanup->{unlock} = 1;
 791
 792         # prepare
 793
 794         my $mode = $running ? $task->{mode} : 'stop';
 795
 796         if ($mode eq 'snapshot') {
 797             my %saved_task = %$task;
 798             eval { $plugin->prepare ($task, $vmid, $mode); };
 799             if (my $err = $@) {
 800                 die $err if $err !~ m/^mode failure/;
 801                 debugmsg ('info',  $err, $logfd);
 802                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 803                 $mode = 'suspend'; # so prepare is called again below
 804                 %$task = %saved_task;
 805             }
 806         }
 807
 808         $cleanup->{prepared} = 1;
 809
 810         $task->{mode} = $mode;
 811
 812         debugmsg ('info', "backup mode: $mode", $logfd);
 813
 814         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
 815             if $opts->{bwlimit};
 816
 817         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 818
 819         if ($mode eq 'stop') {
 820
 821             $plugin->prepare ($task, $vmid, $mode);
 822
 823             $self->run_hook_script ('backup-start', $task, $logfd);
 824
 825             if ($running) {
 826                 debugmsg ('info', "stopping vm", $logfd);
 827                 $task->{vmstoptime} = time();
 828                 $self->run_hook_script ('pre-stop', $task, $logfd);
 829                 $plugin->stop_vm ($task, $vmid);
 830                 $cleanup->{restart} = 1;
 831             }
 832
 833
 834         } elsif ($mode eq 'suspend') {
 835
 836             $plugin->prepare ($task, $vmid, $mode);
 837
 838             $self->run_hook_script ('backup-start', $task, $logfd);
 839
 840             if ($vmtype eq 'lxc') {
 841                 # pre-suspend rsync
 842                 $plugin->copy_data_phase1($task, $vmid);
 843             }
 844
 845             debugmsg ('info', "suspending guest", $logfd);
 846             $task->{vmstoptime} = time ();
 847             $self->run_hook_script ('pre-stop', $task, $logfd);
 848             $plugin->suspend_vm ($task, $vmid);
 849             $cleanup->{resume} = 1;
 850
 851             if ($vmtype eq 'lxc') {
 852                 # post-suspend rsync
 853                 $plugin->copy_data_phase2($task, $vmid);
 854
 855                 debugmsg ('info', "resuming guest", $logfd);
 856                 $cleanup->{resume} = 0;
 857                 $self->run_hook_script('pre-restart', $task, $logfd);
 858                 $plugin->resume_vm($task, $vmid);
 859                 $self->run_hook_script('post-restart', $task, $logfd);
 860                 $log_vm_online_again->();
 861             }
 862
 863         } elsif ($mode eq 'snapshot') {
 864
 865             $self->run_hook_script ('backup-start', $task, $logfd);
 866
 867             my $snapshot_count = $task->{snapshot_count} || 0;
 868
 869             $self->run_hook_script ('pre-stop', $task, $logfd);
 870
 871             if ($snapshot_count > 1) {
 872                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 873                 $task->{vmstoptime} = time ();
 874                 $plugin->suspend_vm ($task, $vmid);
 875                 $cleanup->{resume} = 1;
 876             }
 877
 878             $plugin->snapshot ($task, $vmid);
 879
 880             $self->run_hook_script ('pre-restart', $task, $logfd);
 881
 882             if ($snapshot_count > 1) {
 883                 debugmsg ('info', "resume vm", $logfd);
 884                 $cleanup->{resume} = 0;
 885                 $plugin->resume_vm ($task, $vmid);
 886                 $log_vm_online_again->();
 887             }
 888
 889             $self->run_hook_script ('post-restart', $task, $logfd);
 890
 891         } else {
 892             die "internal error - unknown mode '$mode'\n";
 893         }
 894
 895         # assemble archive image
 896         $plugin->assemble ($task, $vmid);
 897
 898         # produce archive
 899
 900         if ($opts->{stdout}) {
 901             debugmsg ('info', "sending archive to stdout", $logfd);
 902             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 903             $self->run_hook_script ('backup-end', $task, $logfd);
 904             return;
 905         }
 906
 907         my $archive_txt = $self->{opts}->{pbs} ? 'Proxmox Backup Server' : 'vzdump';
 908         debugmsg('info', "creating $archive_txt archive '$task->{target}'", $logfd);
 909         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 910
 911         if ($self->{opts}->{pbs}) {
 912             # size is added to task struct in guest vzdump plugins
 913         } else {
 914             rename ($task->{tmptar}, $task->{target}) ||
 915                 die "unable to rename '$task->{tmptar}' to '$task->{target}'\n";
 916
 917             # determine size
 918             $task->{size} = (-s $task->{target}) || 0;
 919             my $cs = format_size ($task->{size});
 920             debugmsg ('info', "archive file size: $cs", $logfd);
 921         }
 922
 923         # purge older backup
 924         if ($maxfiles && $opts->{remove}) {
 925
 926             if ($self->{opts}->{pbs}) {
 927                 my $args = [$pbs_group_name, '--quiet', '1', '--keep-last', $maxfiles];
 928                 my $logfunc = sub { my $line = shift; debugmsg ('info', $line, $logfd); };
 929                 PVE::Storage::PBSPlugin::run_raw_client_cmd(
 930                     $opts->{scfg}, $opts->{storage}, 'prune', $args, logfunc => $logfunc);
 931             } else {
 932                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{target});
 933                 $bklist = [ sort { $b->[1] <=> $a->[1] } @$bklist ];
 934
 935                 while (scalar (@$bklist) >= $maxfiles) {
 936                     my $d = pop @$bklist;
 937                     debugmsg ('info', "delete old backup '$d->[0]'", $logfd);
 938                     unlink $d->[0];
 939                     my $logfn = $d->[0];
 940                     $logfn =~ s/\.(tgz|((tar|vma)(\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?))$/\.log/;
 941                     unlink $logfn;
 942                 }
 943             }
 944         }
 945
 946         $self->run_hook_script ('backup-end', $task, $logfd);
 947     };
 948     my $err = $@;
 949
 950     if ($plugin) {
 951         # clean-up
 952
 953         if ($cleanup->{unlock}) {
 954             eval { $plugin->unlock_vm ($vmid); };
 955             warn $@ if $@;
 956         }
 957
 958         if ($cleanup->{prepared}) {
 959             # only call cleanup when necessary (when prepare was executed)
 960             eval { $plugin->cleanup ($task, $vmid) };
 961             warn $@ if $@;
 962         }
 963
 964         eval { $plugin->set_logfd (undef); };
 965         warn $@ if $@;
 966
 967         if ($cleanup->{resume} || $cleanup->{restart}) {
 968             eval {
 969                 $self->run_hook_script ('pre-restart', $task, $logfd);
 970                 if ($cleanup->{resume}) {
 971                     debugmsg ('info', "resume vm", $logfd);
 972                     $plugin->resume_vm ($task, $vmid);
 973                 } else {
 974                     my $running = $plugin->vm_status($vmid);
 975                     if (!$running) {
 976                         debugmsg ('info', "restarting vm", $logfd);
 977                         $plugin->start_vm ($task, $vmid);
 978                     }
 979                 }
 980                 $self->run_hook_script ('post-restart', $task, $logfd);
 981             };
 982             my $err = $@;
 983             if ($err) {
 984                 warn $err;
 985             } else {
 986                 $log_vm_online_again->();
 987             }
 988         }
 989     }
 990
 991     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
 992     warn $@ if $@;
 993
 994     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
 995     warn $@ if $@;
 996
 997     my $delay = $task->{backuptime} = time () - $vmstarttime;
 998
 999     if ($err) {
1000         $task->{state} = 'err';
1001         $task->{msg} = $err;
1002         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1003         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
1004
1005         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1006
1007     } else {
1008         $task->{state} = 'ok';
1009         my $tstr = format_time ($delay);
1010         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1011         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1012     }
1013
1014     close ($logfd) if $logfd;
1015
1016     if ($task->{tmplog}) {
1017         if ($self->{opts}->{pbs}) {
1018             if ($task->{state} eq 'ok') {
1019                 my $param = [$pbs_snapshot_name, $task->{tmplog}];
1020                 PVE::Storage::PBSPlugin::run_raw_client_cmd(
1021                     $opts->{scfg}, $opts->{storage}, 'upload-log', $param, errmsg => "upload log failed");
1022             }
1023         } elsif ($task->{logfile}) {
1024             system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1025         }
1026     }
1027
1028     eval { $self->run_hook_script ('log-end', $task); };
1029
1030     die $err if $err && $err =~ m/^interrupted by signal$/;
1031 }
1032
1033 sub exec_backup {
1034     my ($self, $rpcenv, $authuser) = @_;
1035
1036     my $opts = $self->{opts};
1037
1038     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1039     debugmsg ('info', "skip external VMs: " . join(', ', @{$self->{skiplist}}))
1040         if scalar(@{$self->{skiplist}});
1041
1042     my $tasklist = [];
1043     my $vzdump_plugins =  {};
1044     foreach my $plugin (@{$self->{plugins}}) {
1045         my $type = $plugin->type();
1046         next if exists $vzdump_plugins->{$type};
1047         $vzdump_plugins->{$type} = $plugin;
1048     }
1049
1050     my $vmlist = PVE::Cluster::get_vmlist();
1051     foreach my $vmid (@{$opts->{vmids}}) {
1052         my $guest_type = $vmlist->{ids}->{$vmid}->{type};
1053         my $plugin = $vzdump_plugins->{$guest_type};
1054         next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all});
1055         push @$tasklist, {
1056             mode => $opts->{mode},
1057             plugin => $plugin,
1058             state => 'todo',
1059             vmid => $vmid,
1060         };
1061     }
1062
1063     # Use in-memory files for the outer hook logs to pass them to sendmail.
1064     my $job_start_log = '';
1065     my $job_end_log = '';
1066     open my $job_start_fd, '>', \$job_start_log;
1067     open my $job_end_fd, '>', \$job_end_log;
1068
1069     my $starttime = time();
1070     my $errcount = 0;
1071     eval {
1072
1073         $self->run_hook_script ('job-start', undef, $job_start_fd);
1074
1075         foreach my $task (@$tasklist) {
1076             $self->exec_backup_task ($task);
1077             $errcount += 1 if $task->{state} ne 'ok';
1078         }
1079
1080         $self->run_hook_script ('job-end', undef, $job_end_fd);
1081     };
1082     my $err = $@;
1083
1084     $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1085
1086     if ($err) {
1087         debugmsg ('err', "Backup job failed - $err", undef, 1);
1088     } else {
1089         if ($errcount) {
1090             debugmsg ('info', "Backup job finished with errors", undef, 1);
1091         } else {
1092             debugmsg ('info', "Backup job finished successfully", undef, 1);
1093         }
1094     }
1095
1096     close $job_start_fd;
1097     close $job_end_fd;
1098
1099     my $totaltime = time() - $starttime;
1100
1101     eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); };
1102     debugmsg ('err', $@) if $@;
1103
1104     die $err if $err;
1105
1106     die "job errors\n" if $errcount;
1107
1108     unlink $pidfile;
1109 }
1110
1111
1112 sub option_exists {
1113     my $key = shift;
1114     return defined($confdesc->{$key});
1115 }
1116
1117 sub verify_vzdump_parameters {
1118     my ($param, $check_missing) = @_;
1119
1120     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1121         if $param->{all} && $param->{vmid};
1122
1123     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1124         if $param->{exclude} && $param->{vmid};
1125
1126     raise_param_exc({ pool => "option conflicts with option 'vmid'"})
1127         if $param->{pool} && $param->{vmid};
1128
1129     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
1130
1131     warn "option 'size' is deprecated and will be removed in a future " .
1132          "release, please update your script/configuration!\n"
1133         if defined($param->{size});
1134
1135     return if !$check_missing;
1136
1137     raise_param_exc({ vmid => "property is missing"})
1138         if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
1139
1140 }
1141
1142 sub stop_running_backups {
1143     my($self) = @_;
1144
1145     my $upid = PVE::Tools::file_read_firstline($pidfile);
1146     return if !$upid;
1147
1148     my $task = PVE::Tools::upid_decode($upid);
1149
1150     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1151         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1152         kill(15, $task->{pid});
1153         # wait max 15 seconds to shut down (else, do nothing for now)
1154         my $i;
1155         for ($i = 15; $i > 0; $i--) {
1156             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1157             sleep (1);
1158         }
1159         die "stopping backup process $task->{pid} failed\n" if $i == 0;
1160     }
1161 }
1162
1163 sub get_included_guests {
1164     my ($job) = @_;
1165
1166     my $nodename = PVE::INotify::nodename();
1167     my $vmids = [];
1168     my $vmids_per_node = {};
1169
1170     my $vmlist = PVE::Cluster::get_vmlist();
1171
1172     if ($job->{pool}) {
1173         $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool});
1174     } elsif ($job->{vmid}) {
1175         $vmids = [ split_list($job->{vmid}) ];
1176     } elsif ($job->{all}) {
1177         # all or exclude
1178         my $exclude = check_vmids(split_list($job->{exclude}));
1179         my $excludehash = { map { $_ => 1 } @$exclude };
1180
1181         for my $id (keys %{$vmlist->{ids}}) {
1182             next if $excludehash->{$id};
1183             push @$vmids, $id;
1184         }
1185     } else {
1186         return $vmids_per_node;
1187     }
1188     $vmids = check_vmids(@$vmids);
1189
1190     for my $vmid (@$vmids) {
1191         my $node = $vmlist->{ids}->{$vmid}->{node};
1192         next if (defined $job->{node} && $job->{node} ne $node);
1193
1194         push @{$vmids_per_node->{$node}}, $vmid;
1195     }
1196
1197     return $vmids_per_node;
1198 }
1199
1200 1;