PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Path;
   8 use IO::File;
   9 use IO::Select;
  10 use IPC::Open3;
  11 use POSIX qw(strftime);
  12 use Time::Local;
  13
  14 use PVE::Cluster qw(cfs_read_file);
  15 use PVE::DataCenterConfig;
  16 use PVE::Exception qw(raise_param_exc);
  17 use PVE::HA::Config;
  18 use PVE::HA::Env::PVE2;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::RPCEnvironment;
  21 use PVE::Storage;
  22 use PVE::VZDump::Common;
  23 use PVE::VZDump::Plugin;
  24
  25 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  26
  27 my $lockfile = '/var/run/vzdump.lock';
  28 my $pidfile = '/var/run/vzdump.pid';
  29 my $logdir = '/var/log/vzdump';
  30
  31 my @plugins = qw();
  32
  33 my $confdesc = PVE::VZDump::Common::get_confdesc();
  34
  35 # Load available plugins
  36 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
  37 foreach my $plug (@pve_vzdump_classes) {
  38     my $filename = "/usr/share/perl5/$plug.pm";
  39     $filename =~ s!::!/!g;
  40     if (-f $filename) {
  41         eval { require $filename; };
  42         if (!$@) {
  43             $plug->import ();
  44             push @plugins, $plug;
  45         } else {
  46             die $@;
  47         }
  48     }
  49 }
  50
  51 # helper functions
  52
  53 sub debugmsg {
  54     my ($mtype, $msg, $logfd, $syslog) = @_;
  55
  56     PVE::VZDump::Plugin::debugmsg(@_);
  57 }
  58
  59 sub run_command {
  60     my ($logfd, $cmdstr, %param) = @_;
  61
  62     my $logfunc = sub {
  63         my $line = shift;
  64         debugmsg ('info', $line, $logfd);
  65     };
  66
  67     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
  68 }
  69
  70 sub storage_info {
  71     my $storage = shift;
  72
  73     my $cfg = PVE::Storage::config();
  74     my $scfg = PVE::Storage::storage_config($cfg, $storage);
  75     my $type = $scfg->{type};
  76
  77     die "can't use storage type '$type' for backup\n"
  78         if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'
  79               || $type eq 'cifs' || $type eq 'cephfs' || $type eq 'pbs'));
  80     die "can't use storage '$storage' for backups - wrong content type\n"
  81         if (!$scfg->{content}->{backup});
  82
  83     PVE::Storage::activate_storage($cfg, $storage);
  84
  85     if ($type eq 'pbs') {
  86         return {
  87             scfg => $scfg,
  88             maxfiles => $scfg->{maxfiles},
  89         };
  90     } else {
  91         return {
  92             scfg => $scfg,
  93             dumpdir => PVE::Storage::get_backup_dir($cfg, $storage),
  94             maxfiles => $scfg->{maxfiles},
  95         };
  96     }
  97 }
  98
  99 sub format_size {
 100     my $size = shift;
 101
 102     my $kb = $size / 1024;
 103
 104     if ($kb < 1024) {
 105         return int ($kb) . "KB";
 106     }
 107
 108     my $mb = $size / (1024*1024);
 109
 110     if ($mb < 1024) {
 111         return int ($mb) . "MB";
 112     } else {
 113         my $gb = $mb / 1024;
 114         return sprintf ("%.2fGB", $gb);
 115     }
 116 }
 117
 118 sub format_time {
 119     my $seconds = shift;
 120
 121     my $hours = int ($seconds/3600);
 122     $seconds = $seconds - $hours*3600;
 123     my $min = int ($seconds/60);
 124     $seconds = $seconds - $min*60;
 125
 126     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 127 }
 128
 129 sub encode8bit {
 130     my ($str) = @_;
 131
 132     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 133
 134     return $str;
 135 }
 136
 137 sub escape_html {
 138     my ($str) = @_;
 139
 140     $str =~ s/&/&amp;/g;
 141     $str =~ s/</&lt;/g;
 142     $str =~ s/>/&gt;/g;
 143
 144     return $str;
 145 }
 146
 147 sub check_bin {
 148     my ($bin)  = @_;
 149
 150     foreach my $p (split (/:/, $ENV{PATH})) {
 151         my $fn = "$p/$bin";
 152         if (-x $fn) {
 153             return $fn;
 154         }
 155     }
 156
 157     die "unable to find command '$bin'\n";
 158 }
 159
 160 sub check_vmids {
 161     my (@vmids) = @_;
 162
 163     my $res = [];
 164     foreach my $vmid (@vmids) {
 165         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 166         $vmid = int ($vmid); # remove leading zeros
 167         next if !$vmid;
 168         push @$res, $vmid;
 169     }
 170
 171     return $res;
 172 }
 173
 174
 175 sub read_vzdump_defaults {
 176
 177     my $fn = "/etc/vzdump.conf";
 178
 179     my $defaults = {
 180         map {
 181             my $default = $confdesc->{$_}->{default};
 182              defined($default) ? ($_ => $default) : ()
 183         } keys %$confdesc
 184     };
 185
 186     my $raw;
 187     eval { $raw = PVE::Tools::file_get_contents($fn); };
 188     return $defaults if $@;
 189
 190     my $conf_schema = { type => 'object', properties => $confdesc, };
 191     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 192     if (my $excludes = $res->{'exclude-path'}) {
 193         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 194     }
 195     if (defined($res->{mailto})) {
 196         my @mailto = PVE::Tools::split_list($res->{mailto});
 197         $res->{mailto} = [ @mailto ];
 198     }
 199
 200     foreach my $key (keys %$defaults) {
 201         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 202     }
 203
 204     return $res;
 205 }
 206
 207 use constant MAX_MAIL_SIZE => 1024*1024;
 208 sub sendmail {
 209     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 210
 211     my $opts = $self->{opts};
 212
 213     my $mailto = $opts->{mailto};
 214
 215     return if !($mailto && scalar(@$mailto));
 216
 217     my $cmdline = $self->{cmdline};
 218
 219     my $ecount = 0;
 220     foreach my $task (@$tasklist) {
 221         $ecount++ if $task->{state} ne 'ok';
 222         chomp $task->{msg} if $task->{msg};
 223         $task->{backuptime} = 0 if !$task->{backuptime};
 224         $task->{size} = 0 if !$task->{size};
 225         $task->{tarfile} = 'unknown' if !$task->{tarfile};
 226         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 227
 228         if ($task->{state} eq 'todo') {
 229             $task->{msg} = 'aborted';
 230         }
 231     }
 232
 233     my $notify = $opts->{mailnotification} || 'always';
 234     return if (!$ecount && !$err && ($notify eq 'failure'));
 235
 236     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 237     if ($err) {
 238         if ($err =~ /\n/) {
 239             $stat .= ": multiple problems";
 240         } else {
 241             $stat .= ": $err";
 242             $err = undef;
 243         }
 244     }
 245
 246     my $hostname = `hostname -f` || PVE::INotify::nodename();
 247     chomp $hostname;
 248
 249     # text part
 250     my $text = $err ? "$err\n\n" : '';
 251     $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", qw(VMID STATUS TIME SIZE FILENAME));
 252     foreach my $task (@$tasklist) {
 253         my $vmid = $task->{vmid};
 254         if  ($task->{state} eq 'ok') {
 255
 256             $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", $vmid,
 257                                 $task->{state},
 258                                 format_time($task->{backuptime}),
 259                                 format_size ($task->{size}),
 260                                 $task->{tarfile});
 261         } else {
 262             $text .= sprintf ("%-10s %-6s %10s %8.2fMB  %s\n", $vmid,
 263                                 $task->{state},
 264                                 format_time($task->{backuptime}),
 265                                 0, '-');
 266         }
 267     }
 268
 269     my $text_log_part;
 270     $text_log_part .= "\nDetailed backup logs:\n\n";
 271     $text_log_part .= "$cmdline\n\n";
 272
 273     $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
 274     foreach my $task (@$tasklist) {
 275         my $vmid = $task->{vmid};
 276         my $log = $task->{tmplog};
 277         if (!$log) {
 278             $text_log_part .= "$vmid: no log available\n\n";
 279             next;
 280         }
 281         if (open (TMP, "$log")) {
 282             while (my $line = <TMP>) {
 283                 next if $line =~ /^status: \d+/; # not useful in mails
 284                 $text_log_part .= encode8bit ("$vmid: $line");
 285             }
 286         } else {
 287             $text_log_part .= "$vmid: Could not open log file\n\n";
 288         }
 289         close (TMP);
 290         $text_log_part .= "\n";
 291     }
 292     $text_log_part .= $detail_post if defined($detail_post);
 293
 294     # html part
 295     my $html = "<html><body>\n";
 296     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 297     $html .= "<table border=1 cellpadding=3>\n";
 298     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 299
 300     my $ssize = 0;
 301
 302     foreach my $task (@$tasklist) {
 303         my $vmid = $task->{vmid};
 304         my $name = $task->{hostname};
 305
 306         if  ($task->{state} eq 'ok') {
 307
 308             $ssize += $task->{size};
 309
 310             $html .= sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 311                                 $vmid, $name,
 312                                 format_time($task->{backuptime}),
 313                                 format_size ($task->{size}),
 314                                 escape_html ($task->{tarfile}));
 315         } else {
 316             $html .= sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 317                                 $vmid, $name, format_time($task->{backuptime}),
 318                                 escape_html ($task->{msg}));
 319         }
 320     }
 321
 322     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 323  format_time ($totaltime), format_size ($ssize));
 324
 325     $html .= "\n</table><br><br>\n";
 326     my $html_log_part;
 327     $html_log_part .= "Detailed backup logs:<br /><br />\n";
 328     $html_log_part .= "<pre>\n";
 329     $html_log_part .= escape_html($cmdline) . "\n\n";
 330
 331     $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 332     foreach my $task (@$tasklist) {
 333         my $vmid = $task->{vmid};
 334         my $log = $task->{tmplog};
 335         if (!$log) {
 336             $html_log_part .= "$vmid: no log available\n\n";
 337             next;
 338         }
 339         if (open (TMP, "$log")) {
 340             while (my $line = <TMP>) {
 341                 next if $line =~ /^status: \d+/; # not useful in mails
 342                 if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 343                     $html_log_part .= encode8bit ("$vmid: <font color=red>".
 344                         escape_html ($line) . "</font>");
 345                 } else {
 346                     $html_log_part .= encode8bit ("$vmid: " . escape_html ($line));
 347                 }
 348             }
 349         } else {
 350             $html_log_part .= "$vmid: Could not open log file\n\n";
 351         }
 352         close (TMP);
 353         $html_log_part .= "\n";
 354     }
 355     $html_log_part .= escape_html($detail_post) if defined($detail_post);
 356     $html_log_part .= "</pre>";
 357     my $html_end .= "\n</body></html>\n";
 358     # end html part
 359
 360     if (length($text) + length($text_log_part) +
 361         length($html) + length($html_log_part) < MAX_MAIL_SIZE)
 362     {
 363         $html .= $html_log_part;
 364         $text .= $text_log_part;
 365     } else {
 366         my $msg = "Log output was too long to be sent by mail. ".
 367             "See Task History for details!\n";
 368         $text .= $msg;
 369         $html .= "<p>$msg</p>";
 370         $html .= $html_end;
 371     }
 372
 373     my $subject = "vzdump backup status ($hostname) : $stat";
 374
 375     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 376     my $mailfrom = $dcconf->{email_from} || "root";
 377
 378     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 379 };
 380
 381 sub new {
 382     my ($class, $cmdline, $opts, $skiplist) = @_;
 383
 384     mkpath $logdir;
 385
 386     check_bin ('cp');
 387     check_bin ('df');
 388     check_bin ('sendmail');
 389     check_bin ('rsync');
 390     check_bin ('tar');
 391     check_bin ('mount');
 392     check_bin ('umount');
 393     check_bin ('cstream');
 394     check_bin ('ionice');
 395
 396     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 397         check_bin ('lvcreate');
 398         check_bin ('lvs');
 399         check_bin ('lvremove');
 400     }
 401
 402     my $defaults = read_vzdump_defaults();
 403
 404     my $maxfiles = $opts->{maxfiles}; # save here, because we overwrite with default
 405
 406     $opts->{remove} = 1 if !defined($opts->{remove});
 407
 408     foreach my $k (keys %$defaults) {
 409         next if $k eq 'exclude-path'; # dealt with separately
 410         if ($k eq 'dumpdir' || $k eq 'storage') {
 411             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 412                 !defined ($opts->{storage});
 413         } else {
 414             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 415         }
 416     }
 417
 418     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 419     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 420
 421     $skiplist = [] if !$skiplist;
 422     my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist };
 423
 424     my $findexcl = $self->{findexcl} = [];
 425     if ($defaults->{'exclude-path'}) {
 426         push @$findexcl, @{$defaults->{'exclude-path'}};
 427     }
 428
 429     if ($opts->{'exclude-path'}) {
 430         push @$findexcl, @{$opts->{'exclude-path'}};
 431     }
 432
 433     if ($opts->{stdexcludes}) {
 434         push @$findexcl, '/tmp/?*',
 435                          '/var/tmp/?*',
 436                          '/var/run/?*.pid';
 437     }
 438
 439     foreach my $p (@plugins) {
 440
 441         my $pd = $p->new ($self);
 442
 443         push @{$self->{plugins}}, $pd;
 444     }
 445
 446     if (defined($opts->{storage}) && $opts->{stdout}) {
 447         die "unable to use option 'storage' with option 'stdout'\n";
 448     }
 449
 450     if (!$opts->{dumpdir} && !$opts->{storage}) {
 451         $opts->{storage} = 'local';
 452     }
 453
 454     my $errors = '';
 455
 456     if ($opts->{storage}) {
 457         my $info = eval { storage_info ($opts->{storage}) };
 458         $errors .= "could not get storage information for '$opts->{storage}': $@"
 459             if ($@);
 460         $opts->{dumpdir} = $info->{dumpdir};
 461         $opts->{scfg} = $info->{scfg};
 462         $maxfiles //= $info->{maxfiles};
 463     } elsif ($opts->{dumpdir}) {
 464         $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
 465             if ! -d $opts->{dumpdir};
 466     } else {
 467         die "internal error";
 468     }
 469
 470     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 471         $errors .= "\n" if $errors;
 472         $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
 473     }
 474
 475     if ($errors) {
 476         eval { $self->sendmail([], 0, $errors); };
 477         debugmsg ('err', $@) if $@;
 478         die "$errors\n";
 479     }
 480
 481     $opts->{maxfiles} = $maxfiles if defined($maxfiles);
 482
 483     return $self;
 484
 485 }
 486
 487 sub get_mount_info {
 488     my ($dir) = @_;
 489
 490     # Note: df 'available' can be negative, and percentage set to '-'
 491
 492     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 493
 494     my $res;
 495
 496     my $parser = sub {
 497         my $line = shift;
 498         if (my ($fsid, $fstype, undef, $mp) = $line =~
 499             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 500             $res = {
 501                 device => $fsid,
 502                 fstype => $fstype,
 503                 mountpoint => $mp,
 504             };
 505         }
 506     };
 507
 508     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 509     warn $@ if $@;
 510
 511     return $res;
 512 }
 513
 514 sub getlock {
 515     my ($self, $upid) = @_;
 516
 517     my $fh;
 518
 519     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 520
 521     die "missimg UPID" if !$upid; # should not happen
 522
 523     if (!open (SERVER_FLCK, ">>$lockfile")) {
 524         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 525         die "can't open lock on file '$lockfile' - $!";
 526     }
 527
 528     if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 529
 530         if (!$maxwait) {
 531             debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
 532             die "can't acquire lock '$lockfile' (wait = 0)";
 533         }
 534
 535         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 536
 537         eval {
 538             alarm ($maxwait * 60);
 539
 540             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 541
 542             if (!flock (SERVER_FLCK, LOCK_EX)) {
 543                 my $err = $!;
 544                 close (SERVER_FLCK);
 545                 alarm (0);
 546                 die "$err\n";
 547             }
 548             alarm (0);
 549         };
 550         alarm (0);
 551
 552         my $err = $@;
 553
 554         if ($err) {
 555             debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
 556             die "can't acquire lock '$lockfile' - $err";
 557         }
 558
 559         debugmsg('info', "got global lock", undef, 1);
 560     }
 561
 562     PVE::Tools::file_set_contents($pidfile, $upid);
 563 }
 564
 565 sub run_hook_script {
 566     my ($self, $phase, $task, $logfd) = @_;
 567
 568     my $opts = $self->{opts};
 569
 570     my $script = $opts->{script};
 571
 572     return if !$script;
 573
 574     if (!-x $script) {
 575             die "The hook-script '$script' is not executable.\n"
 576     }
 577
 578
 579
 580     my $cmd = "$script $phase";
 581
 582     $cmd .= " $task->{mode} $task->{vmid}" if ($task);
 583
 584     local %ENV;
 585
 586     # set immutable opts directly (so they are available in all phases)
 587     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 588     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 589
 590     foreach my $ek (qw(vmtype hostname tarfile logfile)) {
 591         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 592     }
 593
 594     run_command ($logfd, $cmd);
 595 }
 596
 597 sub compressor_info {
 598     my ($opts) = @_;
 599     my $opt_compress = $opts->{compress};
 600
 601     if (!$opt_compress || $opt_compress eq '0') {
 602         return undef;
 603     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 604         return ('lzop', 'lzo');
 605     } elsif ($opt_compress eq 'gzip') {
 606         if ($opts->{pigz} > 0) {
 607             my $pigz_threads = $opts->{pigz};
 608             if ($pigz_threads == 1) {
 609                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 610                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 611             }
 612             return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
 613         } else {
 614             return ('gzip --rsyncable', 'gz');
 615         }
 616     } else {
 617         die "internal error - unknown compression option '$opt_compress'";
 618     }
 619 }
 620
 621 sub get_backup_file_list {
 622     my ($dir, $bkname, $exclude_fn) = @_;
 623
 624     my $bklist = [];
 625     foreach my $fn (<$dir/${bkname}-*>) {
 626         next if $exclude_fn && $fn eq $exclude_fn;
 627         if ($fn =~ m!/(${bkname}-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?)))$!) {
 628             $fn = "$dir/$1"; # untaint
 629             my $t = timelocal ($7, $6, $5, $4, $3 - 1, $2);
 630             push @$bklist, [$fn, $t];
 631         }
 632     }
 633
 634     return $bklist;
 635 }
 636
 637 sub exec_backup_task {
 638     my ($self, $task) = @_;
 639
 640     my $opts = $self->{opts};
 641
 642     my $vmid = $task->{vmid};
 643     my $plugin = $task->{plugin};
 644     my $vmtype = $plugin->type();
 645
 646     $task->{backup_time} = time();
 647
 648     my $pbs_group_name;
 649     my $pbs_snapshot_name;
 650
 651     if ($opts->{scfg}->{type} eq 'pbs') {
 652         if ($vmtype eq 'lxc') {
 653             $pbs_group_name = "ct/$vmid";
 654         } elsif  ($vmtype eq 'qemu') {
 655             $pbs_group_name = "vm/$vmid";
 656         } else {
 657             die "pbs backup not implemented for plugin type '$vmtype'\n";
 658         }
 659         my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
 660         $pbs_snapshot_name = "$pbs_group_name/$btime";
 661     }
 662
 663     my $vmstarttime = time ();
 664
 665     my $logfd;
 666
 667     my $cleanup = {};
 668
 669     my $log_vm_online_again = sub {
 670         return if !defined($task->{vmstoptime});
 671         $task->{vmconttime} //= time();
 672         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 673         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 674     };
 675
 676     eval {
 677         die "unable to find VM '$vmid'\n" if !$plugin;
 678
 679         # for now we deny backups of a running ha managed service in *stop* mode
 680         # as it interferes with the HA stack (started services should not stop).
 681         if ($opts->{mode} eq 'stop' &&
 682             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 683         {
 684             die "Cannot execute a backup with stop mode on a HA managed and".
 685                 " enabled Service. Use snapshot mode or disable the Service.\n";
 686         }
 687
 688         my $tmplog = "$logdir/$vmtype-$vmid.log";
 689
 690         my $bkname = "vzdump-$vmtype-$vmid";
 691         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
 692
 693         my $maxfiles = $opts->{maxfiles};
 694
 695         if ($maxfiles && !$opts->{remove}) {
 696             my $count;
 697             if ($opts->{scfg}->{type} eq 'pbs') {
 698                 my $res = PVE::Storage::PBSPlugin::run_client_cmd($opts->{scfg}, $opts->{storage}, 'snapshots', $pbs_group_name);
 699                 $count = scalar(@$res);
 700             } else {
 701                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 702                 $count = scalar(@$bklist);
 703             }
 704             die "There is a max backup limit of ($maxfiles) enforced by the".
 705                 " target storage or the vzdump parameters.".
 706                 " Either increase the limit or delete old backup(s).\n"
 707                 if $count >= $maxfiles;
 708         }
 709
 710         if ($opts->{scfg}->{type} ne 'pbs') {
 711             $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 712         }
 713
 714         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 715         my ($comp, $comp_ext) = compressor_info($opts);
 716         if ($comp && $comp_ext) {
 717             $ext .= ".${comp_ext}";
 718         }
 719
 720         if ($opts->{scfg}->{type} eq 'pbs') {
 721             die "unable to pipe backup to stdout\n" if $opts->{stdout};
 722         } else {
 723             if ($opts->{stdout}) {
 724                 $task->{tarfile} = '-';
 725             } else {
 726                 my $tarfile = $task->{tarfile} = "$opts->{dumpdir}/$basename$ext";
 727                 $task->{tmptar} = $task->{tarfile};
 728                 $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 729                 unlink $task->{tmptar};
 730             }
 731         }
 732
 733         $task->{vmtype} = $vmtype;
 734
 735         if ($opts->{scfg}->{type} eq 'pbs') {
 736             $task->{tmpdir} = "/var/tmp/vzdumptmp$$"; #fixme
 737         } elsif ($opts->{tmpdir}) {
 738             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp$$";
 739         } else {
 740             # dumpdir is posix? then use it as temporary dir
 741             my $info = get_mount_info($opts->{dumpdir});
 742             if ($vmtype eq 'qemu' ||
 743                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 744                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 745             } else {
 746                 $task->{tmpdir} = "/var/tmp/vzdumptmp$$";
 747                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 748                           "using $task->{tmpdir} for temporary files", $logfd);
 749             }
 750         }
 751
 752         rmtree $task->{tmpdir};
 753         mkdir $task->{tmpdir};
 754         -d $task->{tmpdir} ||
 755             die "unable to create temporary directory '$task->{tmpdir}'";
 756
 757         $logfd = IO::File->new (">$tmplog") ||
 758             die "unable to create log file '$tmplog'";
 759
 760         $task->{dumpdir} = $opts->{dumpdir};
 761         $task->{storeid} = $opts->{storage};
 762         $task->{scfg} = $opts->{scfg};
 763         $task->{tmplog} = $tmplog;
 764
 765         unlink $task->{logfile} if defined($task->{logfile});
 766
 767         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 768         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
 769
 770         $plugin->set_logfd ($logfd);
 771
 772         # test is VM is running
 773         my ($running, $status_text) = $plugin->vm_status ($vmid);
 774
 775         debugmsg ('info', "status = ${status_text}", $logfd);
 776
 777         # lock VM (prevent config changes)
 778         $plugin->lock_vm ($vmid);
 779
 780         $cleanup->{unlock} = 1;
 781
 782         # prepare
 783
 784         my $mode = $running ? $task->{mode} : 'stop';
 785
 786         if ($mode eq 'snapshot') {
 787             my %saved_task = %$task;
 788             eval { $plugin->prepare ($task, $vmid, $mode); };
 789             if (my $err = $@) {
 790                 die $err if $err !~ m/^mode failure/;
 791                 debugmsg ('info',  $err, $logfd);
 792                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 793                 $mode = 'suspend'; # so prepare is called again below
 794                 %$task = %saved_task;
 795             }
 796         }
 797
 798         $cleanup->{prepared} = 1;
 799
 800         $task->{mode} = $mode;
 801
 802         debugmsg ('info', "backup mode: $mode", $logfd);
 803
 804         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
 805             if $opts->{bwlimit};
 806
 807         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 808
 809         if ($mode eq 'stop') {
 810
 811             $plugin->prepare ($task, $vmid, $mode);
 812
 813             $self->run_hook_script ('backup-start', $task, $logfd);
 814
 815             if ($running) {
 816                 debugmsg ('info', "stopping vm", $logfd);
 817                 $task->{vmstoptime} = time();
 818                 $self->run_hook_script ('pre-stop', $task, $logfd);
 819                 $plugin->stop_vm ($task, $vmid);
 820                 $cleanup->{restart} = 1;
 821             }
 822
 823
 824         } elsif ($mode eq 'suspend') {
 825
 826             $plugin->prepare ($task, $vmid, $mode);
 827
 828             $self->run_hook_script ('backup-start', $task, $logfd);
 829
 830             if ($vmtype eq 'lxc') {
 831                 # pre-suspend rsync
 832                 $plugin->copy_data_phase1($task, $vmid);
 833             }
 834
 835             debugmsg ('info', "suspend vm", $logfd);
 836             $task->{vmstoptime} = time ();
 837             $self->run_hook_script ('pre-stop', $task, $logfd);
 838             $plugin->suspend_vm ($task, $vmid);
 839             $cleanup->{resume} = 1;
 840
 841             if ($vmtype eq 'lxc') {
 842                 # post-suspend rsync
 843                 $plugin->copy_data_phase2($task, $vmid);
 844
 845                 debugmsg ('info', "resume vm", $logfd);
 846                 $cleanup->{resume} = 0;
 847                 $self->run_hook_script('pre-restart', $task, $logfd);
 848                 $plugin->resume_vm($task, $vmid);
 849                 $self->run_hook_script('post-restart', $task, $logfd);
 850                 $log_vm_online_again->();
 851             }
 852
 853         } elsif ($mode eq 'snapshot') {
 854
 855             $self->run_hook_script ('backup-start', $task, $logfd);
 856
 857             my $snapshot_count = $task->{snapshot_count} || 0;
 858
 859             $self->run_hook_script ('pre-stop', $task, $logfd);
 860
 861             if ($snapshot_count > 1) {
 862                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 863                 $task->{vmstoptime} = time ();
 864                 $plugin->suspend_vm ($task, $vmid);
 865                 $cleanup->{resume} = 1;
 866             }
 867
 868             $plugin->snapshot ($task, $vmid);
 869
 870             $self->run_hook_script ('pre-restart', $task, $logfd);
 871
 872             if ($snapshot_count > 1) {
 873                 debugmsg ('info', "resume vm", $logfd);
 874                 $cleanup->{resume} = 0;
 875                 $plugin->resume_vm ($task, $vmid);
 876                 $log_vm_online_again->();
 877             }
 878
 879             $self->run_hook_script ('post-restart', $task, $logfd);
 880
 881         } else {
 882             die "internal error - unknown mode '$mode'\n";
 883         }
 884
 885         # assemble archive image
 886         $plugin->assemble ($task, $vmid);
 887
 888         # produce archive
 889
 890         if ($opts->{stdout}) {
 891             debugmsg ('info', "sending archive to stdout", $logfd);
 892             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 893             $self->run_hook_script ('backup-end', $task, $logfd);
 894             return;
 895         }
 896
 897         # fixme: ??
 898         if ($opts->{scfg}->{type} eq 'pbs') {
 899             debugmsg ('info', "creating pbs archive on storage '$opts->{storage}'", $logfd);
 900         } else {
 901             debugmsg ('info', "creating archive '$task->{tarfile}'", $logfd);
 902         }
 903         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 904
 905         if ($opts->{scfg}->{type} eq 'pbs') {
 906             # fixme: log size ?
 907             debugmsg ('info', "pbs upload finished", $logfd);
 908         } else {
 909             rename ($task->{tmptar}, $task->{tarfile}) ||
 910                 die "unable to rename '$task->{tmptar}' to '$task->{tarfile}'\n";
 911
 912             # determine size
 913             $task->{size} = (-s $task->{tarfile}) || 0;
 914             my $cs = format_size ($task->{size});
 915             debugmsg ('info', "archive file size: $cs", $logfd);
 916         }
 917
 918         # purge older backup
 919         if ($maxfiles && $opts->{remove}) {
 920
 921             if ($opts->{scfg}->{type} eq 'pbs') {
 922                 my $args = [$pbs_group_name, '--keep-last', $maxfiles];
 923                 my $logfunc = sub { my $line = shift; debugmsg ('info', $line, $logfd); };
 924                 PVE::Storage::PBSPlugin::run_raw_client_cmd(
 925                     $opts->{scfg}, $opts->{storage}, 'prune', $args, logfunc => $logfunc);
 926             } else {
 927                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{tarfile});
 928                 $bklist = [ sort { $b->[1] <=> $a->[1] } @$bklist ];
 929
 930                 while (scalar (@$bklist) >= $maxfiles) {
 931                     my $d = pop @$bklist;
 932                     debugmsg ('info', "delete old backup '$d->[0]'", $logfd);
 933                     unlink $d->[0];
 934                     my $logfn = $d->[0];
 935                     $logfn =~ s/\.(tgz|((tar|vma)(\.(gz|lzo))?))$/\.log/;
 936                     unlink $logfn;
 937                 }
 938             }
 939         }
 940
 941         $self->run_hook_script ('backup-end', $task, $logfd);
 942     };
 943     my $err = $@;
 944
 945     if ($plugin) {
 946         # clean-up
 947
 948         if ($cleanup->{unlock}) {
 949             eval { $plugin->unlock_vm ($vmid); };
 950             warn $@ if $@;
 951         }
 952
 953         if ($cleanup->{prepared}) {
 954             # only call cleanup when necessary (when prepare was executed)
 955             eval { $plugin->cleanup ($task, $vmid) };
 956             warn $@ if $@;
 957         }
 958
 959         eval { $plugin->set_logfd (undef); };
 960         warn $@ if $@;
 961
 962         if ($cleanup->{resume} || $cleanup->{restart}) {
 963             eval {
 964                 $self->run_hook_script ('pre-restart', $task, $logfd);
 965                 if ($cleanup->{resume}) {
 966                     debugmsg ('info', "resume vm", $logfd);
 967                     $plugin->resume_vm ($task, $vmid);
 968                 } else {
 969                     my $running = $plugin->vm_status($vmid);
 970                     if (!$running) {
 971                         debugmsg ('info', "restarting vm", $logfd);
 972                         $plugin->start_vm ($task, $vmid);
 973                     }
 974                 }
 975                 $self->run_hook_script ('post-restart', $task, $logfd);
 976             };
 977             my $err = $@;
 978             if ($err) {
 979                 warn $err;
 980             } else {
 981                 $log_vm_online_again->();
 982             }
 983         }
 984     }
 985
 986     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
 987     warn $@ if $@;
 988
 989     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
 990     warn $@ if $@;
 991
 992     my $delay = $task->{backuptime} = time () - $vmstarttime;
 993
 994     if ($err) {
 995         $task->{state} = 'err';
 996         $task->{msg} = $err;
 997         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
 998         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
 999
1000         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1001
1002     } else {
1003         $task->{state} = 'ok';
1004         my $tstr = format_time ($delay);
1005         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1006         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1007     }
1008
1009     close ($logfd) if $logfd;
1010
1011     if ($task->{tmplog}) {
1012         if ($opts->{scfg}->{type} eq 'pbs') {
1013             if ($task->{state} eq 'ok') {
1014                 my $param = [$pbs_snapshot_name, $task->{tmplog}];
1015                 PVE::Storage::PBSPlugin::run_raw_client_cmd(
1016                     $opts->{scfg}, $opts->{storage}, 'upload-log', $param, errmsg => "upload log failed");
1017             }
1018         } elsif ($task->{logfile}) {
1019             system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1020         }
1021     }
1022
1023     eval { $self->run_hook_script ('log-end', $task); };
1024
1025     die $err if $err && $err =~ m/^interrupted by signal$/;
1026 }
1027
1028 sub exec_backup {
1029     my ($self, $rpcenv, $authuser) = @_;
1030
1031     my $opts = $self->{opts};
1032
1033     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1034     debugmsg ('info', "skip external VMs: " . join(', ', @{$self->{skiplist}}))
1035         if scalar(@{$self->{skiplist}});
1036
1037     my $tasklist = [];
1038
1039     if ($opts->{all}) {
1040         foreach my $plugin (@{$self->{plugins}}) {
1041             my $vmlist = $plugin->vmlist();
1042             foreach my $vmid (sort @$vmlist) {
1043                 next if grep { $_ eq  $vmid } @{$opts->{exclude}};
1044                 next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], 1);
1045                 push @$tasklist, { vmid => $vmid,  state => 'todo', plugin => $plugin, mode => $opts->{mode} };
1046             }
1047         }
1048     } else {
1049         foreach my $vmid (sort @{$opts->{vmids}}) {
1050             my $plugin;
1051             foreach my $pg (@{$self->{plugins}}) {
1052                 my $vmlist = $pg->vmlist();
1053                 if (grep { $_ eq  $vmid } @$vmlist) {
1054                     $plugin = $pg;
1055                     last;
1056                 }
1057             }
1058             $rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ]);
1059             push @$tasklist, { vmid => $vmid,  state => 'todo', plugin => $plugin, mode => $opts->{mode} };
1060         }
1061     }
1062
1063     # Use in-memory files for the outer hook logs to pass them to sendmail.
1064     my $job_start_log = '';
1065     my $job_end_log = '';
1066     open my $job_start_fd, '>', \$job_start_log;
1067     open my $job_end_fd, '>', \$job_end_log;
1068
1069     my $starttime = time();
1070     my $errcount = 0;
1071     eval {
1072
1073         $self->run_hook_script ('job-start', undef, $job_start_fd);
1074
1075         foreach my $task (@$tasklist) {
1076             $self->exec_backup_task ($task);
1077             $errcount += 1 if $task->{state} ne 'ok';
1078         }
1079
1080         $self->run_hook_script ('job-end', undef, $job_end_fd);
1081     };
1082     my $err = $@;
1083
1084     $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1085
1086     if ($err) {
1087         debugmsg ('err', "Backup job failed - $err", undef, 1);
1088     } else {
1089         if ($errcount) {
1090             debugmsg ('info', "Backup job finished with errors", undef, 1);
1091         } else {
1092             debugmsg ('info', "Backup job finished successfully", undef, 1);
1093         }
1094     }
1095
1096     close $job_start_fd;
1097     close $job_end_fd;
1098
1099     my $totaltime = time() - $starttime;
1100
1101     eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); };
1102     debugmsg ('err', $@) if $@;
1103
1104     die $err if $err;
1105
1106     die "job errors\n" if $errcount;
1107
1108     unlink $pidfile;
1109 }
1110
1111
1112 sub option_exists {
1113     my $key = shift;
1114     return defined($confdesc->{$key});
1115 }
1116
1117 sub verify_vzdump_parameters {
1118     my ($param, $check_missing) = @_;
1119
1120     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1121         if $param->{all} && $param->{vmid};
1122
1123     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1124         if $param->{exclude} && $param->{vmid};
1125
1126     raise_param_exc({ pool => "option conflicts with option 'vmid'"})
1127         if $param->{pool} && $param->{vmid};
1128
1129     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
1130
1131     warn "option 'size' is deprecated and will be removed in a future " .
1132          "release, please update your script/configuration!\n"
1133         if defined($param->{size});
1134
1135     return if !$check_missing;
1136
1137     raise_param_exc({ vmid => "property is missing"})
1138         if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
1139
1140 }
1141
1142 sub stop_running_backups {
1143     my($self) = @_;
1144
1145     my $upid = PVE::Tools::file_read_firstline($pidfile);
1146     return if !$upid;
1147
1148     my $task = PVE::Tools::upid_decode($upid);
1149
1150     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1151         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1152         kill(15, $task->{pid});
1153         # wait max 15 seconds to shut down (else, do nothing for now)
1154         my $i;
1155         for ($i = 15; $i > 0; $i--) {
1156             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1157             sleep (1);
1158         }
1159         die "stopping backup process $task->{pid} failed\n" if $i == 0;
1160     }
1161 }
1162
1163 1;