PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Path;
   8 use IO::File;
   9 use IO::Select;
  10 use IPC::Open3;
  11 use POSIX qw(strftime);
  12 use Time::Local;
  13
  14 use PVE::Cluster qw(cfs_read_file);
  15 use PVE::DataCenterConfig;
  16 use PVE::Exception qw(raise_param_exc);
  17 use PVE::HA::Config;
  18 use PVE::HA::Env::PVE2;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::RPCEnvironment;
  21 use PVE::Storage;
  22 use PVE::VZDump::Common;
  23 use PVE::VZDump::Plugin;
  24
  25 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  26
  27 my $lockfile = '/var/run/vzdump.lock';
  28 my $pidfile = '/var/run/vzdump.pid';
  29 my $logdir = '/var/log/vzdump';
  30
  31 my @plugins = qw();
  32
  33 my $confdesc = PVE::VZDump::Common::get_confdesc();
  34
  35 # Load available plugins
  36 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
  37 foreach my $plug (@pve_vzdump_classes) {
  38     my $filename = "/usr/share/perl5/$plug.pm";
  39     $filename =~ s!::!/!g;
  40     if (-f $filename) {
  41         eval { require $filename; };
  42         if (!$@) {
  43             $plug->import ();
  44             push @plugins, $plug;
  45         } else {
  46             die $@;
  47         }
  48     }
  49 }
  50
  51 # helper functions
  52
  53 sub debugmsg {
  54     my ($mtype, $msg, $logfd, $syslog) = @_;
  55
  56     PVE::VZDump::Plugin::debugmsg(@_);
  57 }
  58
  59 sub run_command {
  60     my ($logfd, $cmdstr, %param) = @_;
  61
  62     my $logfunc = sub {
  63         my $line = shift;
  64         debugmsg ('info', $line, $logfd);
  65     };
  66
  67     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
  68 }
  69
  70 sub storage_info {
  71     my $storage = shift;
  72
  73     my $cfg = PVE::Storage::config();
  74     my $scfg = PVE::Storage::storage_config($cfg, $storage);
  75     my $type = $scfg->{type};
  76
  77     die "can't use storage type '$type' for backup\n"
  78         if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'
  79               || $type eq 'cifs' || $type eq 'cephfs' || $type eq 'pbs'));
  80     die "can't use storage '$storage' for backups - wrong content type\n"
  81         if (!$scfg->{content}->{backup});
  82
  83     PVE::Storage::activate_storage($cfg, $storage);
  84
  85     if ($type eq 'pbs') {
  86         return {
  87             scfg => $scfg,
  88             maxfiles => $scfg->{maxfiles},
  89         };
  90     } else {
  91         return {
  92             scfg => $scfg,
  93             dumpdir => PVE::Storage::get_backup_dir($cfg, $storage),
  94             maxfiles => $scfg->{maxfiles},
  95         };
  96     }
  97 }
  98
  99 sub format_size {
 100     my $size = shift;
 101
 102     my $kb = $size / 1024;
 103
 104     if ($kb < 1024) {
 105         return int ($kb) . "KB";
 106     }
 107
 108     my $mb = $size / (1024*1024);
 109
 110     if ($mb < 1024) {
 111         return int ($mb) . "MB";
 112     } else {
 113         my $gb = $mb / 1024;
 114         return sprintf ("%.2fGB", $gb);
 115     }
 116 }
 117
 118 sub format_time {
 119     my $seconds = shift;
 120
 121     my $hours = int ($seconds/3600);
 122     $seconds = $seconds - $hours*3600;
 123     my $min = int ($seconds/60);
 124     $seconds = $seconds - $min*60;
 125
 126     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 127 }
 128
 129 sub encode8bit {
 130     my ($str) = @_;
 131
 132     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 133
 134     return $str;
 135 }
 136
 137 sub escape_html {
 138     my ($str) = @_;
 139
 140     $str =~ s/&/&amp;/g;
 141     $str =~ s/</&lt;/g;
 142     $str =~ s/>/&gt;/g;
 143
 144     return $str;
 145 }
 146
 147 sub check_bin {
 148     my ($bin)  = @_;
 149
 150     foreach my $p (split (/:/, $ENV{PATH})) {
 151         my $fn = "$p/$bin";
 152         if (-x $fn) {
 153             return $fn;
 154         }
 155     }
 156
 157     die "unable to find command '$bin'\n";
 158 }
 159
 160 sub check_vmids {
 161     my (@vmids) = @_;
 162
 163     my $res = [];
 164     foreach my $vmid (@vmids) {
 165         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 166         $vmid = int ($vmid); # remove leading zeros
 167         next if !$vmid;
 168         push @$res, $vmid;
 169     }
 170
 171     return $res;
 172 }
 173
 174
 175 sub read_vzdump_defaults {
 176
 177     my $fn = "/etc/vzdump.conf";
 178
 179     my $defaults = {
 180         map {
 181             my $default = $confdesc->{$_}->{default};
 182              defined($default) ? ($_ => $default) : ()
 183         } keys %$confdesc
 184     };
 185
 186     my $raw;
 187     eval { $raw = PVE::Tools::file_get_contents($fn); };
 188     return $defaults if $@;
 189
 190     my $conf_schema = { type => 'object', properties => $confdesc, };
 191     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 192     if (my $excludes = $res->{'exclude-path'}) {
 193         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 194     }
 195     if (defined($res->{mailto})) {
 196         my @mailto = PVE::Tools::split_list($res->{mailto});
 197         $res->{mailto} = [ @mailto ];
 198     }
 199
 200     foreach my $key (keys %$defaults) {
 201         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 202     }
 203
 204     return $res;
 205 }
 206
 207 use constant MAX_MAIL_SIZE => 1024*1024;
 208 sub sendmail {
 209     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 210
 211     my $opts = $self->{opts};
 212
 213     my $mailto = $opts->{mailto};
 214
 215     return if !($mailto && scalar(@$mailto));
 216
 217     my $cmdline = $self->{cmdline};
 218
 219     my $ecount = 0;
 220     foreach my $task (@$tasklist) {
 221         $ecount++ if $task->{state} ne 'ok';
 222         chomp $task->{msg} if $task->{msg};
 223         $task->{backuptime} = 0 if !$task->{backuptime};
 224         $task->{size} = 0 if !$task->{size};
 225         $task->{tarfile} = 'unknown' if !$task->{tarfile};
 226         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 227
 228         if ($task->{state} eq 'todo') {
 229             $task->{msg} = 'aborted';
 230         }
 231     }
 232
 233     my $notify = $opts->{mailnotification} || 'always';
 234     return if (!$ecount && !$err && ($notify eq 'failure'));
 235
 236     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 237     if ($err) {
 238         if ($err =~ /\n/) {
 239             $stat .= ": multiple problems";
 240         } else {
 241             $stat .= ": $err";
 242             $err = undef;
 243         }
 244     }
 245
 246     my $hostname = `hostname -f` || PVE::INotify::nodename();
 247     chomp $hostname;
 248
 249     # text part
 250     my $text = $err ? "$err\n\n" : '';
 251     $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", qw(VMID STATUS TIME SIZE FILENAME));
 252     foreach my $task (@$tasklist) {
 253         my $vmid = $task->{vmid};
 254         if  ($task->{state} eq 'ok') {
 255
 256             $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", $vmid,
 257                                 $task->{state},
 258                                 format_time($task->{backuptime}),
 259                                 format_size ($task->{size}),
 260                                 $task->{tarfile});
 261         } else {
 262             $text .= sprintf ("%-10s %-6s %10s %8.2fMB  %s\n", $vmid,
 263                                 $task->{state},
 264                                 format_time($task->{backuptime}),
 265                                 0, '-');
 266         }
 267     }
 268
 269     my $text_log_part;
 270     $text_log_part .= "\nDetailed backup logs:\n\n";
 271     $text_log_part .= "$cmdline\n\n";
 272
 273     $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
 274     foreach my $task (@$tasklist) {
 275         my $vmid = $task->{vmid};
 276         my $log = $task->{tmplog};
 277         if (!$log) {
 278             $text_log_part .= "$vmid: no log available\n\n";
 279             next;
 280         }
 281         if (open (TMP, "$log")) {
 282             while (my $line = <TMP>) {
 283                 next if $line =~ /^status: \d+/; # not useful in mails
 284                 $text_log_part .= encode8bit ("$vmid: $line");
 285             }
 286         } else {
 287             $text_log_part .= "$vmid: Could not open log file\n\n";
 288         }
 289         close (TMP);
 290         $text_log_part .= "\n";
 291     }
 292     $text_log_part .= $detail_post if defined($detail_post);
 293
 294     # html part
 295     my $html = "<html><body>\n";
 296     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 297     $html .= "<table border=1 cellpadding=3>\n";
 298     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 299
 300     my $ssize = 0;
 301
 302     foreach my $task (@$tasklist) {
 303         my $vmid = $task->{vmid};
 304         my $name = $task->{hostname};
 305
 306         if  ($task->{state} eq 'ok') {
 307
 308             $ssize += $task->{size};
 309
 310             $html .= sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 311                                 $vmid, $name,
 312                                 format_time($task->{backuptime}),
 313                                 format_size ($task->{size}),
 314                                 escape_html ($task->{tarfile}));
 315         } else {
 316             $html .= sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 317                                 $vmid, $name, format_time($task->{backuptime}),
 318                                 escape_html ($task->{msg}));
 319         }
 320     }
 321
 322     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 323  format_time ($totaltime), format_size ($ssize));
 324
 325     $html .= "\n</table><br><br>\n";
 326     my $html_log_part;
 327     $html_log_part .= "Detailed backup logs:<br /><br />\n";
 328     $html_log_part .= "<pre>\n";
 329     $html_log_part .= escape_html($cmdline) . "\n\n";
 330
 331     $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 332     foreach my $task (@$tasklist) {
 333         my $vmid = $task->{vmid};
 334         my $log = $task->{tmplog};
 335         if (!$log) {
 336             $html_log_part .= "$vmid: no log available\n\n";
 337             next;
 338         }
 339         if (open (TMP, "$log")) {
 340             while (my $line = <TMP>) {
 341                 next if $line =~ /^status: \d+/; # not useful in mails
 342                 if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 343                     $html_log_part .= encode8bit ("$vmid: <font color=red>".
 344                         escape_html ($line) . "</font>");
 345                 } else {
 346                     $html_log_part .= encode8bit ("$vmid: " . escape_html ($line));
 347                 }
 348             }
 349         } else {
 350             $html_log_part .= "$vmid: Could not open log file\n\n";
 351         }
 352         close (TMP);
 353         $html_log_part .= "\n";
 354     }
 355     $html_log_part .= escape_html($detail_post) if defined($detail_post);
 356     $html_log_part .= "</pre>";
 357     my $html_end .= "\n</body></html>\n";
 358     # end html part
 359
 360     if (length($text) + length($text_log_part) +
 361         length($html) + length($html_log_part) < MAX_MAIL_SIZE)
 362     {
 363         $html .= $html_log_part;
 364         $text .= $text_log_part;
 365     } else {
 366         my $msg = "Log output was too long to be sent by mail. ".
 367             "See Task History for details!\n";
 368         $text .= $msg;
 369         $html .= "<p>$msg</p>";
 370         $html .= $html_end;
 371     }
 372
 373     my $subject = "vzdump backup status ($hostname) : $stat";
 374
 375     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 376     my $mailfrom = $dcconf->{email_from} || "root";
 377
 378     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 379 };
 380
 381 sub new {
 382     my ($class, $cmdline, $opts, $skiplist) = @_;
 383
 384     mkpath $logdir;
 385
 386     check_bin ('cp');
 387     check_bin ('df');
 388     check_bin ('sendmail');
 389     check_bin ('rsync');
 390     check_bin ('tar');
 391     check_bin ('mount');
 392     check_bin ('umount');
 393     check_bin ('cstream');
 394     check_bin ('ionice');
 395
 396     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 397         check_bin ('lvcreate');
 398         check_bin ('lvs');
 399         check_bin ('lvremove');
 400     }
 401
 402     my $defaults = read_vzdump_defaults();
 403
 404     my $maxfiles = $opts->{maxfiles}; # save here, because we overwrite with default
 405
 406     $opts->{remove} = 1 if !defined($opts->{remove});
 407
 408     foreach my $k (keys %$defaults) {
 409         next if $k eq 'exclude-path'; # dealt with separately
 410         if ($k eq 'dumpdir' || $k eq 'storage') {
 411             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 412                 !defined ($opts->{storage});
 413         } else {
 414             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 415         }
 416     }
 417
 418     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 419     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 420
 421     $skiplist = [] if !$skiplist;
 422     my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist };
 423
 424     my $findexcl = $self->{findexcl} = [];
 425     if ($defaults->{'exclude-path'}) {
 426         push @$findexcl, @{$defaults->{'exclude-path'}};
 427     }
 428
 429     if ($opts->{'exclude-path'}) {
 430         push @$findexcl, @{$opts->{'exclude-path'}};
 431     }
 432
 433     if ($opts->{stdexcludes}) {
 434         push @$findexcl, '/tmp/?*',
 435                          '/var/tmp/?*',
 436                          '/var/run/?*.pid';
 437     }
 438
 439     foreach my $p (@plugins) {
 440
 441         my $pd = $p->new ($self);
 442
 443         push @{$self->{plugins}}, $pd;
 444     }
 445
 446     if (defined($opts->{storage}) && $opts->{stdout}) {
 447         die "unable to use option 'storage' with option 'stdout'\n";
 448     }
 449
 450     if (!$opts->{dumpdir} && !$opts->{storage}) {
 451         $opts->{storage} = 'local';
 452     }
 453
 454     my $errors = '';
 455
 456     if ($opts->{storage}) {
 457         my $info = eval { storage_info ($opts->{storage}) };
 458         $errors .= "could not get storage information for '$opts->{storage}': $@"
 459             if ($@);
 460         $opts->{dumpdir} = $info->{dumpdir};
 461         $opts->{scfg} = $info->{scfg};
 462         $maxfiles //= $info->{maxfiles};
 463     } elsif ($opts->{dumpdir}) {
 464         $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
 465             if ! -d $opts->{dumpdir};
 466     } else {
 467         die "internal error";
 468     }
 469
 470     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 471         $errors .= "\n" if $errors;
 472         $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
 473     }
 474
 475     if ($errors) {
 476         eval { $self->sendmail([], 0, $errors); };
 477         debugmsg ('err', $@) if $@;
 478         die "$errors\n";
 479     }
 480
 481     $opts->{maxfiles} = $maxfiles if defined($maxfiles);
 482
 483     return $self;
 484
 485 }
 486
 487 sub get_mount_info {
 488     my ($dir) = @_;
 489
 490     # Note: df 'available' can be negative, and percentage set to '-'
 491
 492     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 493
 494     my $res;
 495
 496     my $parser = sub {
 497         my $line = shift;
 498         if (my ($fsid, $fstype, undef, $mp) = $line =~
 499             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 500             $res = {
 501                 device => $fsid,
 502                 fstype => $fstype,
 503                 mountpoint => $mp,
 504             };
 505         }
 506     };
 507
 508     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 509     warn $@ if $@;
 510
 511     return $res;
 512 }
 513
 514 sub getlock {
 515     my ($self, $upid) = @_;
 516
 517     my $fh;
 518
 519     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 520
 521     die "missimg UPID" if !$upid; # should not happen
 522
 523     if (!open (SERVER_FLCK, ">>$lockfile")) {
 524         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 525         die "can't open lock on file '$lockfile' - $!";
 526     }
 527
 528     if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 529
 530         if (!$maxwait) {
 531             debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
 532             die "can't acquire lock '$lockfile' (wait = 0)";
 533         }
 534
 535         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 536
 537         eval {
 538             alarm ($maxwait * 60);
 539
 540             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 541
 542             if (!flock (SERVER_FLCK, LOCK_EX)) {
 543                 my $err = $!;
 544                 close (SERVER_FLCK);
 545                 alarm (0);
 546                 die "$err\n";
 547             }
 548             alarm (0);
 549         };
 550         alarm (0);
 551
 552         my $err = $@;
 553
 554         if ($err) {
 555             debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
 556             die "can't acquire lock '$lockfile' - $err";
 557         }
 558
 559         debugmsg('info', "got global lock", undef, 1);
 560     }
 561
 562     PVE::Tools::file_set_contents($pidfile, $upid);
 563 }
 564
 565 sub run_hook_script {
 566     my ($self, $phase, $task, $logfd) = @_;
 567
 568     my $opts = $self->{opts};
 569
 570     my $script = $opts->{script};
 571     return if !$script;
 572
 573     if (!-x $script) {
 574         die "The hook script '$script' is not executable.\n";
 575     }
 576
 577     my $cmd = "$script $phase";
 578
 579     $cmd .= " $task->{mode} $task->{vmid}" if ($task);
 580
 581     local %ENV;
 582     # set immutable opts directly (so they are available in all phases)
 583     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 584     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 585
 586     foreach my $ek (qw(vmtype hostname tarfile logfile)) {
 587         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 588     }
 589
 590     run_command ($logfd, $cmd);
 591 }
 592
 593 sub compressor_info {
 594     my ($opts) = @_;
 595     my $opt_compress = $opts->{compress};
 596
 597     if (!$opt_compress || $opt_compress eq '0') {
 598         return undef;
 599     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 600         return ('lzop', 'lzo');
 601     } elsif ($opt_compress eq 'gzip') {
 602         if ($opts->{pigz} > 0) {
 603             my $pigz_threads = $opts->{pigz};
 604             if ($pigz_threads == 1) {
 605                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 606                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 607             }
 608             return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
 609         } else {
 610             return ('gzip --rsyncable', 'gz');
 611         }
 612     } else {
 613         die "internal error - unknown compression option '$opt_compress'";
 614     }
 615 }
 616
 617 sub get_backup_file_list {
 618     my ($dir, $bkname, $exclude_fn) = @_;
 619
 620     my $bklist = [];
 621     foreach my $fn (<$dir/${bkname}-*>) {
 622         next if $exclude_fn && $fn eq $exclude_fn;
 623         if ($fn =~ m!/(${bkname}-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?)))$!) {
 624             $fn = "$dir/$1"; # untaint
 625             my $t = timelocal ($7, $6, $5, $4, $3 - 1, $2);
 626             push @$bklist, [$fn, $t];
 627         }
 628     }
 629
 630     return $bklist;
 631 }
 632
 633 sub exec_backup_task {
 634     my ($self, $task) = @_;
 635
 636     my $opts = $self->{opts};
 637
 638     my $vmid = $task->{vmid};
 639     my $plugin = $task->{plugin};
 640     my $vmtype = $plugin->type();
 641
 642     $task->{backup_time} = time();
 643
 644     my $pbs_group_name;
 645     my $pbs_snapshot_name;
 646
 647     if ($opts->{scfg}->{type} eq 'pbs') {
 648         if ($vmtype eq 'lxc') {
 649             $pbs_group_name = "ct/$vmid";
 650         } elsif  ($vmtype eq 'qemu') {
 651             $pbs_group_name = "vm/$vmid";
 652         } else {
 653             die "pbs backup not implemented for plugin type '$vmtype'\n";
 654         }
 655         my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
 656         $pbs_snapshot_name = "$pbs_group_name/$btime";
 657     }
 658
 659     my $vmstarttime = time ();
 660
 661     my $logfd;
 662
 663     my $cleanup = {};
 664
 665     my $log_vm_online_again = sub {
 666         return if !defined($task->{vmstoptime});
 667         $task->{vmconttime} //= time();
 668         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 669         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 670     };
 671
 672     eval {
 673         die "unable to find VM '$vmid'\n" if !$plugin;
 674
 675         # for now we deny backups of a running ha managed service in *stop* mode
 676         # as it interferes with the HA stack (started services should not stop).
 677         if ($opts->{mode} eq 'stop' &&
 678             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 679         {
 680             die "Cannot execute a backup with stop mode on a HA managed and".
 681                 " enabled Service. Use snapshot mode or disable the Service.\n";
 682         }
 683
 684         my $tmplog = "$logdir/$vmtype-$vmid.log";
 685
 686         my $bkname = "vzdump-$vmtype-$vmid";
 687         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
 688
 689         my $maxfiles = $opts->{maxfiles};
 690
 691         if ($maxfiles && !$opts->{remove}) {
 692             my $count;
 693             if ($opts->{scfg}->{type} eq 'pbs') {
 694                 my $res = PVE::Storage::PBSPlugin::run_client_cmd($opts->{scfg}, $opts->{storage}, 'snapshots', $pbs_group_name);
 695                 $count = scalar(@$res);
 696             } else {
 697                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 698                 $count = scalar(@$bklist);
 699             }
 700             die "There is a max backup limit of ($maxfiles) enforced by the".
 701                 " target storage or the vzdump parameters.".
 702                 " Either increase the limit or delete old backup(s).\n"
 703                 if $count >= $maxfiles;
 704         }
 705
 706         if ($opts->{scfg}->{type} ne 'pbs') {
 707             $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 708         }
 709
 710         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 711         my ($comp, $comp_ext) = compressor_info($opts);
 712         if ($comp && $comp_ext) {
 713             $ext .= ".${comp_ext}";
 714         }
 715
 716         if ($opts->{scfg}->{type} eq 'pbs') {
 717             die "unable to pipe backup to stdout\n" if $opts->{stdout};
 718         } else {
 719             if ($opts->{stdout}) {
 720                 $task->{tarfile} = '-';
 721             } else {
 722                 my $tarfile = $task->{tarfile} = "$opts->{dumpdir}/$basename$ext";
 723                 $task->{tmptar} = $task->{tarfile};
 724                 $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 725                 unlink $task->{tmptar};
 726             }
 727         }
 728
 729         $task->{vmtype} = $vmtype;
 730
 731         if ($opts->{scfg}->{type} eq 'pbs') {
 732             $task->{tmpdir} = "/var/tmp/vzdumptmp$$"; #fixme
 733         } elsif ($opts->{tmpdir}) {
 734             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp$$";
 735         } else {
 736             # dumpdir is posix? then use it as temporary dir
 737             my $info = get_mount_info($opts->{dumpdir});
 738             if ($vmtype eq 'qemu' ||
 739                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 740                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 741             } else {
 742                 $task->{tmpdir} = "/var/tmp/vzdumptmp$$";
 743                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 744                           "using $task->{tmpdir} for temporary files", $logfd);
 745             }
 746         }
 747
 748         rmtree $task->{tmpdir};
 749         mkdir $task->{tmpdir};
 750         -d $task->{tmpdir} ||
 751             die "unable to create temporary directory '$task->{tmpdir}'";
 752
 753         $logfd = IO::File->new (">$tmplog") ||
 754             die "unable to create log file '$tmplog'";
 755
 756         $task->{dumpdir} = $opts->{dumpdir};
 757         $task->{storeid} = $opts->{storage};
 758         $task->{scfg} = $opts->{scfg};
 759         $task->{tmplog} = $tmplog;
 760
 761         unlink $task->{logfile} if defined($task->{logfile});
 762
 763         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 764         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
 765
 766         $plugin->set_logfd ($logfd);
 767
 768         # test is VM is running
 769         my ($running, $status_text) = $plugin->vm_status ($vmid);
 770
 771         debugmsg ('info', "status = ${status_text}", $logfd);
 772
 773         # lock VM (prevent config changes)
 774         $plugin->lock_vm ($vmid);
 775
 776         $cleanup->{unlock} = 1;
 777
 778         # prepare
 779
 780         my $mode = $running ? $task->{mode} : 'stop';
 781
 782         if ($mode eq 'snapshot') {
 783             my %saved_task = %$task;
 784             eval { $plugin->prepare ($task, $vmid, $mode); };
 785             if (my $err = $@) {
 786                 die $err if $err !~ m/^mode failure/;
 787                 debugmsg ('info',  $err, $logfd);
 788                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 789                 $mode = 'suspend'; # so prepare is called again below
 790                 %$task = %saved_task;
 791             }
 792         }
 793
 794         $cleanup->{prepared} = 1;
 795
 796         $task->{mode} = $mode;
 797
 798         debugmsg ('info', "backup mode: $mode", $logfd);
 799
 800         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
 801             if $opts->{bwlimit};
 802
 803         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 804
 805         if ($mode eq 'stop') {
 806
 807             $plugin->prepare ($task, $vmid, $mode);
 808
 809             $self->run_hook_script ('backup-start', $task, $logfd);
 810
 811             if ($running) {
 812                 debugmsg ('info', "stopping vm", $logfd);
 813                 $task->{vmstoptime} = time();
 814                 $self->run_hook_script ('pre-stop', $task, $logfd);
 815                 $plugin->stop_vm ($task, $vmid);
 816                 $cleanup->{restart} = 1;
 817             }
 818
 819
 820         } elsif ($mode eq 'suspend') {
 821
 822             $plugin->prepare ($task, $vmid, $mode);
 823
 824             $self->run_hook_script ('backup-start', $task, $logfd);
 825
 826             if ($vmtype eq 'lxc') {
 827                 # pre-suspend rsync
 828                 $plugin->copy_data_phase1($task, $vmid);
 829             }
 830
 831             debugmsg ('info', "suspend vm", $logfd);
 832             $task->{vmstoptime} = time ();
 833             $self->run_hook_script ('pre-stop', $task, $logfd);
 834             $plugin->suspend_vm ($task, $vmid);
 835             $cleanup->{resume} = 1;
 836
 837             if ($vmtype eq 'lxc') {
 838                 # post-suspend rsync
 839                 $plugin->copy_data_phase2($task, $vmid);
 840
 841                 debugmsg ('info', "resume vm", $logfd);
 842                 $cleanup->{resume} = 0;
 843                 $self->run_hook_script('pre-restart', $task, $logfd);
 844                 $plugin->resume_vm($task, $vmid);
 845                 $self->run_hook_script('post-restart', $task, $logfd);
 846                 $log_vm_online_again->();
 847             }
 848
 849         } elsif ($mode eq 'snapshot') {
 850
 851             $self->run_hook_script ('backup-start', $task, $logfd);
 852
 853             my $snapshot_count = $task->{snapshot_count} || 0;
 854
 855             $self->run_hook_script ('pre-stop', $task, $logfd);
 856
 857             if ($snapshot_count > 1) {
 858                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 859                 $task->{vmstoptime} = time ();
 860                 $plugin->suspend_vm ($task, $vmid);
 861                 $cleanup->{resume} = 1;
 862             }
 863
 864             $plugin->snapshot ($task, $vmid);
 865
 866             $self->run_hook_script ('pre-restart', $task, $logfd);
 867
 868             if ($snapshot_count > 1) {
 869                 debugmsg ('info', "resume vm", $logfd);
 870                 $cleanup->{resume} = 0;
 871                 $plugin->resume_vm ($task, $vmid);
 872                 $log_vm_online_again->();
 873             }
 874
 875             $self->run_hook_script ('post-restart', $task, $logfd);
 876
 877         } else {
 878             die "internal error - unknown mode '$mode'\n";
 879         }
 880
 881         # assemble archive image
 882         $plugin->assemble ($task, $vmid);
 883
 884         # produce archive
 885
 886         if ($opts->{stdout}) {
 887             debugmsg ('info', "sending archive to stdout", $logfd);
 888             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 889             $self->run_hook_script ('backup-end', $task, $logfd);
 890             return;
 891         }
 892
 893         # fixme: ??
 894         if ($opts->{scfg}->{type} eq 'pbs') {
 895             debugmsg ('info', "creating pbs archive on storage '$opts->{storage}'", $logfd);
 896         } else {
 897             debugmsg ('info', "creating archive '$task->{tarfile}'", $logfd);
 898         }
 899         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 900
 901         if ($opts->{scfg}->{type} eq 'pbs') {
 902             # fixme: log size ?
 903             debugmsg ('info', "pbs upload finished", $logfd);
 904         } else {
 905             rename ($task->{tmptar}, $task->{tarfile}) ||
 906                 die "unable to rename '$task->{tmptar}' to '$task->{tarfile}'\n";
 907
 908             # determine size
 909             $task->{size} = (-s $task->{tarfile}) || 0;
 910             my $cs = format_size ($task->{size});
 911             debugmsg ('info', "archive file size: $cs", $logfd);
 912         }
 913
 914         # purge older backup
 915         if ($maxfiles && $opts->{remove}) {
 916
 917             if ($opts->{scfg}->{type} eq 'pbs') {
 918                 my $args = [$pbs_group_name, '--keep-last', $maxfiles];
 919                 my $logfunc = sub { my $line = shift; debugmsg ('info', $line, $logfd); };
 920                 PVE::Storage::PBSPlugin::run_raw_client_cmd(
 921                     $opts->{scfg}, $opts->{storage}, 'prune', $args, logfunc => $logfunc);
 922             } else {
 923                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{tarfile});
 924                 $bklist = [ sort { $b->[1] <=> $a->[1] } @$bklist ];
 925
 926                 while (scalar (@$bklist) >= $maxfiles) {
 927                     my $d = pop @$bklist;
 928                     debugmsg ('info', "delete old backup '$d->[0]'", $logfd);
 929                     unlink $d->[0];
 930                     my $logfn = $d->[0];
 931                     $logfn =~ s/\.(tgz|((tar|vma)(\.(gz|lzo))?))$/\.log/;
 932                     unlink $logfn;
 933                 }
 934             }
 935         }
 936
 937         $self->run_hook_script ('backup-end', $task, $logfd);
 938     };
 939     my $err = $@;
 940
 941     if ($plugin) {
 942         # clean-up
 943
 944         if ($cleanup->{unlock}) {
 945             eval { $plugin->unlock_vm ($vmid); };
 946             warn $@ if $@;
 947         }
 948
 949         if ($cleanup->{prepared}) {
 950             # only call cleanup when necessary (when prepare was executed)
 951             eval { $plugin->cleanup ($task, $vmid) };
 952             warn $@ if $@;
 953         }
 954
 955         eval { $plugin->set_logfd (undef); };
 956         warn $@ if $@;
 957
 958         if ($cleanup->{resume} || $cleanup->{restart}) {
 959             eval {
 960                 $self->run_hook_script ('pre-restart', $task, $logfd);
 961                 if ($cleanup->{resume}) {
 962                     debugmsg ('info', "resume vm", $logfd);
 963                     $plugin->resume_vm ($task, $vmid);
 964                 } else {
 965                     my $running = $plugin->vm_status($vmid);
 966                     if (!$running) {
 967                         debugmsg ('info', "restarting vm", $logfd);
 968                         $plugin->start_vm ($task, $vmid);
 969                     }
 970                 }
 971                 $self->run_hook_script ('post-restart', $task, $logfd);
 972             };
 973             my $err = $@;
 974             if ($err) {
 975                 warn $err;
 976             } else {
 977                 $log_vm_online_again->();
 978             }
 979         }
 980     }
 981
 982     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
 983     warn $@ if $@;
 984
 985     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
 986     warn $@ if $@;
 987
 988     my $delay = $task->{backuptime} = time () - $vmstarttime;
 989
 990     if ($err) {
 991         $task->{state} = 'err';
 992         $task->{msg} = $err;
 993         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
 994         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
 995
 996         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
 997
 998     } else {
 999         $task->{state} = 'ok';
1000         my $tstr = format_time ($delay);
1001         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1002         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1003     }
1004
1005     close ($logfd) if $logfd;
1006
1007     if ($task->{tmplog}) {
1008         if ($opts->{scfg}->{type} eq 'pbs') {
1009             if ($task->{state} eq 'ok') {
1010                 my $param = [$pbs_snapshot_name, $task->{tmplog}];
1011                 PVE::Storage::PBSPlugin::run_raw_client_cmd(
1012                     $opts->{scfg}, $opts->{storage}, 'upload-log', $param, errmsg => "upload log failed");
1013             }
1014         } elsif ($task->{logfile}) {
1015             system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1016         }
1017     }
1018
1019     eval { $self->run_hook_script ('log-end', $task); };
1020
1021     die $err if $err && $err =~ m/^interrupted by signal$/;
1022 }
1023
1024 sub exec_backup {
1025     my ($self, $rpcenv, $authuser) = @_;
1026
1027     my $opts = $self->{opts};
1028
1029     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1030     debugmsg ('info', "skip external VMs: " . join(', ', @{$self->{skiplist}}))
1031         if scalar(@{$self->{skiplist}});
1032
1033     my $tasklist = [];
1034
1035     if ($opts->{all}) {
1036         foreach my $plugin (@{$self->{plugins}}) {
1037             my $vmlist = $plugin->vmlist();
1038             foreach my $vmid (sort @$vmlist) {
1039                 next if grep { $_ eq  $vmid } @{$opts->{exclude}};
1040                 next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], 1);
1041                 push @$tasklist, { vmid => $vmid,  state => 'todo', plugin => $plugin, mode => $opts->{mode} };
1042             }
1043         }
1044     } else {
1045         foreach my $vmid (sort @{$opts->{vmids}}) {
1046             my $plugin;
1047             foreach my $pg (@{$self->{plugins}}) {
1048                 my $vmlist = $pg->vmlist();
1049                 if (grep { $_ eq  $vmid } @$vmlist) {
1050                     $plugin = $pg;
1051                     last;
1052                 }
1053             }
1054             $rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ]);
1055             push @$tasklist, { vmid => $vmid,  state => 'todo', plugin => $plugin, mode => $opts->{mode} };
1056         }
1057     }
1058
1059     # Use in-memory files for the outer hook logs to pass them to sendmail.
1060     my $job_start_log = '';
1061     my $job_end_log = '';
1062     open my $job_start_fd, '>', \$job_start_log;
1063     open my $job_end_fd, '>', \$job_end_log;
1064
1065     my $starttime = time();
1066     my $errcount = 0;
1067     eval {
1068
1069         $self->run_hook_script ('job-start', undef, $job_start_fd);
1070
1071         foreach my $task (@$tasklist) {
1072             $self->exec_backup_task ($task);
1073             $errcount += 1 if $task->{state} ne 'ok';
1074         }
1075
1076         $self->run_hook_script ('job-end', undef, $job_end_fd);
1077     };
1078     my $err = $@;
1079
1080     $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1081
1082     if ($err) {
1083         debugmsg ('err', "Backup job failed - $err", undef, 1);
1084     } else {
1085         if ($errcount) {
1086             debugmsg ('info', "Backup job finished with errors", undef, 1);
1087         } else {
1088             debugmsg ('info', "Backup job finished successfully", undef, 1);
1089         }
1090     }
1091
1092     close $job_start_fd;
1093     close $job_end_fd;
1094
1095     my $totaltime = time() - $starttime;
1096
1097     eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); };
1098     debugmsg ('err', $@) if $@;
1099
1100     die $err if $err;
1101
1102     die "job errors\n" if $errcount;
1103
1104     unlink $pidfile;
1105 }
1106
1107
1108 sub option_exists {
1109     my $key = shift;
1110     return defined($confdesc->{$key});
1111 }
1112
1113 sub verify_vzdump_parameters {
1114     my ($param, $check_missing) = @_;
1115
1116     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1117         if $param->{all} && $param->{vmid};
1118
1119     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1120         if $param->{exclude} && $param->{vmid};
1121
1122     raise_param_exc({ pool => "option conflicts with option 'vmid'"})
1123         if $param->{pool} && $param->{vmid};
1124
1125     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
1126
1127     warn "option 'size' is deprecated and will be removed in a future " .
1128          "release, please update your script/configuration!\n"
1129         if defined($param->{size});
1130
1131     return if !$check_missing;
1132
1133     raise_param_exc({ vmid => "property is missing"})
1134         if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
1135
1136 }
1137
1138 sub stop_running_backups {
1139     my($self) = @_;
1140
1141     my $upid = PVE::Tools::file_read_firstline($pidfile);
1142     return if !$upid;
1143
1144     my $task = PVE::Tools::upid_decode($upid);
1145
1146     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1147         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1148         kill(15, $task->{pid});
1149         # wait max 15 seconds to shut down (else, do nothing for now)
1150         my $i;
1151         for ($i = 15; $i > 0; $i--) {
1152             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1153             sleep (1);
1154         }
1155         die "stopping backup process $task->{pid} failed\n" if $i == 0;
1156     }
1157 }
1158
1159 1;