PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5 use Fcntl ':flock';
   6 use PVE::Exception qw(raise_param_exc);
   7 use IO::File;
   8 use IO::Select;
   9 use IPC::Open3;
  10 use File::Path;
  11 use PVE::RPCEnvironment;
  12 use PVE::Storage;
  13 use PVE::Cluster qw(cfs_read_file);
  14 use POSIX qw(strftime);
  15 use Time::Local;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::HA::Env::PVE2;
  18 use PVE::HA::Config;
  19 use PVE::VZDump::Plugin;
  20
  21 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  22
  23 my $lockfile = '/var/run/vzdump.lock';
  24 my $pidfile = '/var/run/vzdump.pid';
  25 my $logdir = '/var/log/vzdump';
  26
  27 my @plugins = qw();
  28
  29 my $confdesc = {
  30     vmid => {
  31         type => 'string', format => 'pve-vmid-list',
  32         description => "The ID of the guest system you want to backup.",
  33         completion => \&PVE::Cluster::complete_local_vmid,
  34         optional => 1,
  35     },
  36     node => get_standard_option('pve-node', {
  37         description => "Only run if executed on this node.",
  38         completion => \&PVE::Cluster::get_nodelist,
  39         optional => 1,
  40     }),
  41     all => {
  42         type => 'boolean',
  43         description => "Backup all known guest systems on this host.",
  44         optional => 1,
  45         default => 0,
  46     },
  47     stdexcludes => {
  48         type => 'boolean',
  49         description => "Exclude temporary files and logs.",
  50         optional => 1,
  51         default => 1,
  52     },
  53     compress => {
  54         type => 'string',
  55         description => "Compress dump file.",
  56         optional => 1,
  57         enum => ['0', '1', 'gzip', 'lzo'],
  58         default => '0',
  59     },
  60     pigz=> {
  61         type => "integer",
  62         description => "Use pigz instead of gzip when N>0.".
  63             " N=1 uses half of cores, N>1 uses N as thread count.",
  64         optional => 1,
  65         default => 0,
  66     },
  67     quiet => {
  68         type => 'boolean',
  69         description => "Be quiet.",
  70         optional => 1,
  71         default => 0,
  72     },
  73     mode => {
  74         type => 'string',
  75         description => "Backup mode.",
  76         optional => 1,
  77         default => 'snapshot',
  78         enum => [ 'snapshot', 'suspend', 'stop' ],
  79     },
  80     exclude => {
  81         type => 'string', format => 'pve-vmid-list',
  82         description => "Exclude specified guest systems (assumes --all)",
  83         optional => 1,
  84     },
  85     'exclude-path' => {
  86         type => 'string', format => 'string-alist',
  87         description => "Exclude certain files/directories (shell globs).",
  88         optional => 1,
  89     },
  90     mailto => {
  91         type => 'string', format => 'string-list',
  92         description => "Comma-separated list of email addresses that should" .
  93             " receive email notifications.",
  94         optional => 1,
  95     },
  96     mailnotification => {
  97         type => 'string',
  98         description => "Specify when to send an email",
  99         optional => 1,
 100         enum => [ 'always', 'failure' ],
 101         default => 'always',
 102     },
 103     tmpdir => {
 104         type => 'string',
 105         description => "Store temporary files to specified directory.",
 106         optional => 1,
 107     },
 108     dumpdir => {
 109         type => 'string',
 110         description => "Store resulting files to specified directory.",
 111         optional => 1,
 112     },
 113     script => {
 114         type => 'string',
 115         description => "Use specified hook script.",
 116         optional => 1,
 117     },
 118     storage => get_standard_option('pve-storage-id', {
 119         description => "Store resulting file to this storage.",
 120         completion => \&complete_backup_storage,
 121         optional => 1,
 122     }),
 123     stop => {
 124         type => 'boolean',
 125         description => "Stop running backup jobs on this host.",
 126         optional => 1,
 127         default => 0,
 128     },
 129     size => {
 130         type => 'integer',
 131         description => "Unused, will be removed in a future release.",
 132         optional => 1,
 133         minimum => 500,
 134         default => 1024,
 135     },
 136     bwlimit => {
 137         type => 'integer',
 138         description => "Limit I/O bandwidth (KBytes per second).",
 139         optional => 1,
 140         minimum => 0,
 141         default => 0,
 142     },
 143     ionice => {
 144         type => 'integer',
 145         description => "Set CFQ ionice priority.",
 146         optional => 1,
 147         minimum => 0,
 148         maximum => 8,
 149         default => 7,
 150     },
 151     lockwait => {
 152         type => 'integer',
 153         description => "Maximal time to wait for the global lock (minutes).",
 154         optional => 1,
 155         minimum => 0,
 156         default => 3*60, # 3 hours
 157     },
 158     stopwait => {
 159         type => 'integer',
 160         description => "Maximal time to wait until a guest system is stopped (minutes).",
 161         optional => 1,
 162         minimum => 0,
 163         default => 10, # 10 minutes
 164     },
 165     maxfiles => {
 166         type => 'integer',
 167         description => "Maximal number of backup files per guest system.",
 168         optional => 1,
 169         minimum => 1,
 170         default => 1,
 171     },
 172     remove => {
 173         type => 'boolean',
 174         description => "Remove old backup files if there are more than 'maxfiles' backup files.",
 175         optional => 1,
 176         default => 1,
 177     },
 178     pool => {
 179         type => 'string',
 180         description => 'Backup all known guest systems included in the specified pool.',
 181         optional => 1,
 182     }
 183 };
 184
 185 # Load available plugins
 186 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
 187 foreach my $plug (@pve_vzdump_classes) {
 188     my $filename = "/usr/share/perl5/$plug.pm";
 189     $filename =~ s!::!/!g;
 190     if (-f $filename) {
 191         eval { require $filename; };
 192         if (!$@) {
 193             $plug->import ();
 194             push @plugins, $plug;
 195         } else {
 196             die $@;
 197         }
 198     }
 199 }
 200
 201 # helper functions
 202
 203 sub debugmsg {
 204     my ($mtype, $msg, $logfd, $syslog) = @_;
 205
 206     PVE::VZDump::Plugin::debugmsg(@_);
 207 }
 208
 209 sub run_command {
 210     my ($logfd, $cmdstr, %param) = @_;
 211
 212     my $logfunc = sub {
 213         my $line = shift;
 214         debugmsg ('info', $line, $logfd);
 215     };
 216
 217     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
 218 }
 219
 220 sub storage_info {
 221     my $storage = shift;
 222
 223     my $cfg = PVE::Storage::config();
 224     my $scfg = PVE::Storage::storage_config($cfg, $storage);
 225     my $type = $scfg->{type};
 226
 227     die "can't use storage type '$type' for backup\n"
 228         if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'
 229               || $type eq 'cifs' || $type eq 'cephfs'));
 230     die "can't use storage '$storage' for backups - wrong content type\n"
 231         if (!$scfg->{content}->{backup});
 232
 233     PVE::Storage::activate_storage($cfg, $storage);
 234
 235     return {
 236         dumpdir => PVE::Storage::get_backup_dir($cfg, $storage),
 237         maxfiles => $scfg->{maxfiles},
 238     };
 239 }
 240
 241 sub format_size {
 242     my $size = shift;
 243
 244     my $kb = $size / 1024;
 245
 246     if ($kb < 1024) {
 247         return int ($kb) . "KB";
 248     }
 249
 250     my $mb = $size / (1024*1024);
 251
 252     if ($mb < 1024) {
 253         return int ($mb) . "MB";
 254     } else {
 255         my $gb = $mb / 1024;
 256         return sprintf ("%.2fGB", $gb);
 257     }
 258 }
 259
 260 sub format_time {
 261     my $seconds = shift;
 262
 263     my $hours = int ($seconds/3600);
 264     $seconds = $seconds - $hours*3600;
 265     my $min = int ($seconds/60);
 266     $seconds = $seconds - $min*60;
 267
 268     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 269 }
 270
 271 sub encode8bit {
 272     my ($str) = @_;
 273
 274     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 275
 276     return $str;
 277 }
 278
 279 sub escape_html {
 280     my ($str) = @_;
 281
 282     $str =~ s/&/&amp;/g;
 283     $str =~ s/</&lt;/g;
 284     $str =~ s/>/&gt;/g;
 285
 286     return $str;
 287 }
 288
 289 sub check_bin {
 290     my ($bin)  = @_;
 291
 292     foreach my $p (split (/:/, $ENV{PATH})) {
 293         my $fn = "$p/$bin";
 294         if (-x $fn) {
 295             return $fn;
 296         }
 297     }
 298
 299     die "unable to find command '$bin'\n";
 300 }
 301
 302 sub check_vmids {
 303     my (@vmids) = @_;
 304
 305     my $res = [];
 306     foreach my $vmid (@vmids) {
 307         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 308         $vmid = int ($vmid); # remove leading zeros
 309         next if !$vmid;
 310         push @$res, $vmid;
 311     }
 312
 313     return $res;
 314 }
 315
 316
 317 sub read_vzdump_defaults {
 318
 319     my $fn = "/etc/vzdump.conf";
 320
 321     my $defaults = {
 322         map {
 323             my $default = $confdesc->{$_}->{default};
 324              defined($default) ? ($_ => $default) : ()
 325         } keys %$confdesc
 326     };
 327
 328     my $raw;
 329     eval { $raw = PVE::Tools::file_get_contents($fn); };
 330     return $defaults if $@;
 331
 332     my $conf_schema = { type => 'object', properties => $confdesc, };
 333     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 334     if (my $excludes = $res->{'exclude-path'}) {
 335         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 336     }
 337     if (defined($res->{mailto})) {
 338         my @mailto = PVE::Tools::split_list($res->{mailto});
 339         $res->{mailto} = [ @mailto ];
 340     }
 341
 342     foreach my $key (keys %$defaults) {
 343         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 344     }
 345
 346     return $res;
 347 }
 348
 349 sub sendmail {
 350     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 351
 352     my $opts = $self->{opts};
 353
 354     my $mailto = $opts->{mailto};
 355
 356     return if !($mailto && scalar(@$mailto));
 357
 358     my $cmdline = $self->{cmdline};
 359
 360     my $ecount = 0;
 361     foreach my $task (@$tasklist) {
 362         $ecount++ if $task->{state} ne 'ok';
 363         chomp $task->{msg} if $task->{msg};
 364         $task->{backuptime} = 0 if !$task->{backuptime};
 365         $task->{size} = 0 if !$task->{size};
 366         $task->{tarfile} = 'unknown' if !$task->{tarfile};
 367         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 368
 369         if ($task->{state} eq 'todo') {
 370             $task->{msg} = 'aborted';
 371         }
 372     }
 373
 374     my $notify = $opts->{mailnotification} || 'always';
 375     return if (!$ecount && !$err && ($notify eq 'failure'));
 376
 377     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 378     if ($err) {
 379         if ($err =~ /\n/) {
 380             $stat .= ": multiple problems";
 381         } else {
 382             $stat .= ": $err";
 383             $err = undef;
 384         }
 385     }
 386
 387     my $hostname = `hostname -f` || PVE::INotify::nodename();
 388     chomp $hostname;
 389
 390     # text part
 391     my $text = $err ? "$err\n\n" : '';
 392     $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", qw(VMID STATUS TIME SIZE FILENAME));
 393     foreach my $task (@$tasklist) {
 394         my $vmid = $task->{vmid};
 395         if  ($task->{state} eq 'ok') {
 396
 397             $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", $vmid,
 398                                 $task->{state},
 399                                 format_time($task->{backuptime}),
 400                                 format_size ($task->{size}),
 401                                 $task->{tarfile});
 402         } else {
 403             $text .= sprintf ("%-10s %-6s %10s %8.2fMB  %s\n", $vmid,
 404                                 $task->{state},
 405                                 format_time($task->{backuptime}),
 406                                 0, '-');
 407         }
 408     }
 409
 410     $text .= "\nDetailed backup logs:\n\n";
 411     $text .= "$cmdline\n\n";
 412
 413     $text .= $detail_pre . "\n" if defined($detail_pre);
 414     foreach my $task (@$tasklist) {
 415         my $vmid = $task->{vmid};
 416         my $log = $task->{tmplog};
 417         if (!$log) {
 418             $text .= "$vmid: no log available\n\n";
 419             next;
 420         }
 421         open (TMP, "$log");
 422         while (my $line = <TMP>) { $text .= encode8bit ("$vmid: $line"); }
 423         close (TMP);
 424         $text .= "\n";
 425     }
 426     $text .= $detail_post if defined($detail_post);
 427
 428     # html part
 429     my $html = "<html><body>\n";
 430     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 431     $html .= "<table border=1 cellpadding=3>\n";
 432     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 433
 434     my $ssize = 0;
 435
 436     foreach my $task (@$tasklist) {
 437         my $vmid = $task->{vmid};
 438         my $name = $task->{hostname};
 439
 440         if  ($task->{state} eq 'ok') {
 441
 442             $ssize += $task->{size};
 443
 444             $html .= sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 445                                 $vmid, $name,
 446                                 format_time($task->{backuptime}),
 447                                 format_size ($task->{size}),
 448                                 escape_html ($task->{tarfile}));
 449         } else {
 450             $html .= sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 451                                 $vmid, $name, format_time($task->{backuptime}),
 452                                 escape_html ($task->{msg}));
 453         }
 454     }
 455
 456     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 457  format_time ($totaltime), format_size ($ssize));
 458
 459     $html .= "</table><br><br>\n";
 460     $html .= "Detailed backup logs:<br /><br />\n";
 461     $html .= "<pre>\n";
 462     $html .= escape_html($cmdline) . "\n\n";
 463
 464     $html .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 465     foreach my $task (@$tasklist) {
 466         my $vmid = $task->{vmid};
 467         my $log = $task->{tmplog};
 468         if (!$log) {
 469             $html .= "$vmid: no log available\n\n";
 470             next;
 471         }
 472         open (TMP, "$log");
 473         while (my $line = <TMP>) {
 474             if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 475                 $html .= encode8bit ("$vmid: <font color=red>".
 476                                        escape_html ($line) . "</font>");
 477             } else {
 478                 $html .= encode8bit ("$vmid: " . escape_html ($line));
 479             }
 480         }
 481         close (TMP);
 482         $html .= "\n";
 483     }
 484     $html .= escape_html($detail_post) if defined($detail_post);
 485     $html .= "</pre></body></html>\n";
 486     # end html part
 487
 488     my $subject = "vzdump backup status ($hostname) : $stat";
 489
 490     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 491     my $mailfrom = $dcconf->{email_from} || "root";
 492
 493     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 494 };
 495
 496 sub new {
 497     my ($class, $cmdline, $opts, $skiplist) = @_;
 498
 499     mkpath $logdir;
 500
 501     check_bin ('cp');
 502     check_bin ('df');
 503     check_bin ('sendmail');
 504     check_bin ('rsync');
 505     check_bin ('tar');
 506     check_bin ('mount');
 507     check_bin ('umount');
 508     check_bin ('cstream');
 509     check_bin ('ionice');
 510
 511     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 512         check_bin ('lvcreate');
 513         check_bin ('lvs');
 514         check_bin ('lvremove');
 515     }
 516
 517     my $defaults = read_vzdump_defaults();
 518
 519     my $maxfiles = $opts->{maxfiles}; # save here, because we overwrite with default
 520
 521     $opts->{remove} = 1 if !defined($opts->{remove});
 522
 523     foreach my $k (keys %$defaults) {
 524         next if $k eq 'exclude-path'; # dealt with separately
 525         if ($k eq 'dumpdir' || $k eq 'storage') {
 526             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 527                 !defined ($opts->{storage});
 528         } else {
 529             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 530         }
 531     }
 532
 533     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 534     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 535
 536     $skiplist = [] if !$skiplist;
 537     my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist };
 538
 539     my $findexcl = $self->{findexcl} = [];
 540     if ($defaults->{'exclude-path'}) {
 541         push @$findexcl, @{$defaults->{'exclude-path'}};
 542     }
 543
 544     if ($opts->{'exclude-path'}) {
 545         push @$findexcl, @{$opts->{'exclude-path'}};
 546     }
 547
 548     if ($opts->{stdexcludes}) {
 549         push @$findexcl, '/tmp/?*',
 550                          '/var/tmp/?*',
 551                          '/var/run/?*.pid';
 552     }
 553
 554     foreach my $p (@plugins) {
 555
 556         my $pd = $p->new ($self);
 557
 558         push @{$self->{plugins}}, $pd;
 559     }
 560
 561     if (!$opts->{dumpdir} && !$opts->{storage}) {
 562         $opts->{storage} = 'local';
 563     }
 564
 565     my $errors = '';
 566
 567     if ($opts->{storage}) {
 568         my $info = eval { storage_info ($opts->{storage}) };
 569         $errors .= "could not get storage information for '$opts->{storage}': $@"
 570             if ($@);
 571         $opts->{dumpdir} = $info->{dumpdir};
 572         $maxfiles //= $info->{maxfiles};
 573     } elsif ($opts->{dumpdir}) {
 574         $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
 575             if ! -d $opts->{dumpdir};
 576     } else {
 577         die "internal error";
 578     }
 579
 580     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 581         $errors .= "\n" if $errors;
 582         $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
 583     }
 584
 585     if ($errors) {
 586         eval { $self->sendmail([], 0, $errors); };
 587         debugmsg ('err', $@) if $@;
 588         die "$errors\n";
 589     }
 590
 591     $opts->{maxfiles} = $maxfiles if defined($maxfiles);
 592
 593     return $self;
 594
 595 }
 596
 597 sub get_mount_info {
 598     my ($dir) = @_;
 599
 600     # Note: df 'available' can be negative, and percentage set to '-'
 601
 602     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 603
 604     my $res;
 605
 606     my $parser = sub {
 607         my $line = shift;
 608         if (my ($fsid, $fstype, undef, $mp) = $line =~
 609             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 610             $res = {
 611                 device => $fsid,
 612                 fstype => $fstype,
 613                 mountpoint => $mp,
 614             };
 615         }
 616     };
 617
 618     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 619     warn $@ if $@;
 620
 621     return $res;
 622 }
 623
 624 sub getlock {
 625     my ($self, $upid) = @_;
 626
 627     my $fh;
 628
 629     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 630
 631     die "missimg UPID" if !$upid; # should not happen
 632
 633     if (!open (SERVER_FLCK, ">>$lockfile")) {
 634         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 635         die "can't open lock on file '$lockfile' - $!";
 636     }
 637
 638     if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 639
 640         if (!$maxwait) {
 641             debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
 642             die "can't acquire lock '$lockfile' (wait = 0)";
 643         }
 644
 645         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 646
 647         eval {
 648             alarm ($maxwait * 60);
 649
 650             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 651
 652             if (!flock (SERVER_FLCK, LOCK_EX)) {
 653                 my $err = $!;
 654                 close (SERVER_FLCK);
 655                 alarm (0);
 656                 die "$err\n";
 657             }
 658             alarm (0);
 659         };
 660         alarm (0);
 661
 662         my $err = $@;
 663
 664         if ($err) {
 665             debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
 666             die "can't acquire lock '$lockfile' - $err";
 667         }
 668
 669         debugmsg('info', "got global lock", undef, 1);
 670     }
 671
 672     PVE::Tools::file_set_contents($pidfile, $upid);
 673 }
 674
 675 sub run_hook_script {
 676     my ($self, $phase, $task, $logfd) = @_;
 677
 678     my $opts = $self->{opts};
 679
 680     my $script = $opts->{script};
 681
 682     return if !$script;
 683
 684     my $cmd = "$script $phase";
 685
 686     $cmd .= " $task->{mode} $task->{vmid}" if ($task);
 687
 688     local %ENV;
 689
 690     # set immutable opts directly (so they are available in all phases)
 691     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 692     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 693
 694     foreach my $ek (qw(vmtype hostname tarfile logfile)) {
 695         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 696     }
 697
 698     run_command ($logfd, $cmd);
 699 }
 700
 701 sub compressor_info {
 702     my ($opts) = @_;
 703     my $opt_compress = $opts->{compress};
 704
 705     if (!$opt_compress || $opt_compress eq '0') {
 706         return undef;
 707     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 708         return ('lzop', 'lzo');
 709     } elsif ($opt_compress eq 'gzip') {
 710         if ($opts->{pigz} > 0) {
 711             my $pigz_threads = $opts->{pigz};
 712             if ($pigz_threads == 1) {
 713                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 714                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 715             }
 716             return ("pigz -p ${pigz_threads}", 'gz');
 717         } else {
 718             return ('gzip', 'gz');
 719         }
 720     } else {
 721         die "internal error - unknown compression option '$opt_compress'";
 722     }
 723 }
 724
 725 sub get_backup_file_list {
 726     my ($dir, $bkname, $exclude_fn) = @_;
 727
 728     my $bklist = [];
 729     foreach my $fn (<$dir/${bkname}-*>) {
 730         next if $exclude_fn && $fn eq $exclude_fn;
 731         if ($fn =~ m!/(${bkname}-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?)))$!) {
 732             $fn = "$dir/$1"; # untaint
 733             my $t = timelocal ($7, $6, $5, $4, $3 - 1, $2);
 734             push @$bklist, [$fn, $t];
 735         }
 736     }
 737
 738     return $bklist;
 739 }
 740
 741 sub exec_backup_task {
 742     my ($self, $task) = @_;
 743
 744     my $opts = $self->{opts};
 745
 746     my $vmid = $task->{vmid};
 747     my $plugin = $task->{plugin};
 748
 749     my $vmstarttime = time ();
 750
 751     my $logfd;
 752
 753     my $cleanup = {};
 754
 755     my $log_vm_online_again = sub {
 756         return if !defined($task->{vmstoptime});
 757         $task->{vmconttime} //= time();
 758         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 759         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 760     };
 761
 762     eval {
 763         die "unable to find VM '$vmid'\n" if !$plugin;
 764
 765         # for now we deny backups of a running ha managed service in *stop* mode
 766         # as it interferes with the HA stack (started services should not stop).
 767         if ($opts->{mode} eq 'stop' &&
 768             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 769         {
 770             die "Cannot execute a backup with stop mode on a HA managed and".
 771                 " enabled Service. Use snapshot mode or disable the Service.\n";
 772         }
 773
 774         my $vmtype = $plugin->type();
 775
 776         my $tmplog = "$logdir/$vmtype-$vmid.log";
 777
 778         my $bkname = "vzdump-$vmtype-$vmid";
 779         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime());
 780
 781         my $maxfiles = $opts->{maxfiles};
 782
 783         if ($maxfiles && !$opts->{remove}) {
 784             my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 785             die "There is a max backup limit of ($maxfiles) enforced by the".
 786             " target storage or the vzdump parameters.".
 787             " Either increase the limit or delete old backup(s).\n"
 788                 if scalar(@$bklist) >= $maxfiles;
 789         }
 790
 791         my $logfile = $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 792
 793         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 794         my ($comp, $comp_ext) = compressor_info($opts);
 795         if ($comp && $comp_ext) {
 796             $ext .= ".${comp_ext}";
 797         }
 798
 799         if ($opts->{stdout}) {
 800             $task->{tarfile} = '-';
 801         } else {
 802             my $tarfile = $task->{tarfile} = "$opts->{dumpdir}/$basename$ext";
 803             $task->{tmptar} = $task->{tarfile};
 804             $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 805             unlink $task->{tmptar};
 806         }
 807
 808         $task->{vmtype} = $vmtype;
 809
 810         if ($opts->{tmpdir}) {
 811             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp$$";
 812         } else {
 813             # dumpdir is posix? then use it as temporary dir
 814             my $info = get_mount_info($opts->{dumpdir});
 815             if ($vmtype eq 'qemu' ||
 816                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 817                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 818             } else {
 819                 $task->{tmpdir} = "/var/tmp/vzdumptmp$$";
 820                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 821                           "using $task->{tmpdir} for temporary files", $logfd);
 822             }
 823         }
 824
 825         rmtree $task->{tmpdir};
 826         mkdir $task->{tmpdir};
 827         -d $task->{tmpdir} ||
 828             die "unable to create temporary directory '$task->{tmpdir}'";
 829
 830         $logfd = IO::File->new (">$tmplog") ||
 831             die "unable to create log file '$tmplog'";
 832
 833         $task->{dumpdir} = $opts->{dumpdir};
 834         $task->{storeid} = $opts->{storage};
 835         $task->{tmplog} = $tmplog;
 836
 837         unlink $logfile;
 838
 839         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 840         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
 841
 842         $plugin->set_logfd ($logfd);
 843
 844         # test is VM is running
 845         my ($running, $status_text) = $plugin->vm_status ($vmid);
 846
 847         debugmsg ('info', "status = ${status_text}", $logfd);
 848
 849         # lock VM (prevent config changes)
 850         $plugin->lock_vm ($vmid);
 851
 852         $cleanup->{unlock} = 1;
 853
 854         # prepare
 855
 856         my $mode = $running ? $task->{mode} : 'stop';
 857
 858         if ($mode eq 'snapshot') {
 859             my %saved_task = %$task;
 860             eval { $plugin->prepare ($task, $vmid, $mode); };
 861             if (my $err = $@) {
 862                 die $err if $err !~ m/^mode failure/;
 863                 debugmsg ('info',  $err, $logfd);
 864                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 865                 $mode = 'suspend'; # so prepare is called again below
 866                 %$task = %saved_task;
 867             }
 868         }
 869
 870         $cleanup->{prepared} = 1;
 871
 872         $task->{mode} = $mode;
 873
 874         debugmsg ('info', "backup mode: $mode", $logfd);
 875
 876         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
 877             if $opts->{bwlimit};
 878
 879         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 880
 881         if ($mode eq 'stop') {
 882
 883             $plugin->prepare ($task, $vmid, $mode);
 884
 885             $self->run_hook_script ('backup-start', $task, $logfd);
 886
 887             if ($running) {
 888                 debugmsg ('info', "stopping vm", $logfd);
 889                 $task->{vmstoptime} = time();
 890                 $self->run_hook_script ('pre-stop', $task, $logfd);
 891                 $plugin->stop_vm ($task, $vmid);
 892                 $cleanup->{restart} = 1;
 893             }
 894
 895
 896         } elsif ($mode eq 'suspend') {
 897
 898             $plugin->prepare ($task, $vmid, $mode);
 899
 900             $self->run_hook_script ('backup-start', $task, $logfd);
 901
 902             if ($vmtype eq 'lxc') {
 903                 # pre-suspend rsync
 904                 $plugin->copy_data_phase1($task, $vmid);
 905             }
 906
 907             debugmsg ('info', "suspend vm", $logfd);
 908             $task->{vmstoptime} = time ();
 909             $self->run_hook_script ('pre-stop', $task, $logfd);
 910             $plugin->suspend_vm ($task, $vmid);
 911             $cleanup->{resume} = 1;
 912
 913             if ($vmtype eq 'lxc') {
 914                 # post-suspend rsync
 915                 $plugin->copy_data_phase2($task, $vmid);
 916
 917                 debugmsg ('info', "resume vm", $logfd);
 918                 $cleanup->{resume} = 0;
 919                 $self->run_hook_script('pre-restart', $task, $logfd);
 920                 $plugin->resume_vm($task, $vmid);
 921                 $self->run_hook_script('post-restart', $task, $logfd);
 922                 $log_vm_online_again->();
 923             }
 924
 925         } elsif ($mode eq 'snapshot') {
 926
 927             $self->run_hook_script ('backup-start', $task, $logfd);
 928
 929             my $snapshot_count = $task->{snapshot_count} || 0;
 930
 931             $self->run_hook_script ('pre-stop', $task, $logfd);
 932
 933             if ($snapshot_count > 1) {
 934                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 935                 $task->{vmstoptime} = time ();
 936                 $plugin->suspend_vm ($task, $vmid);
 937                 $cleanup->{resume} = 1;
 938             }
 939
 940             $plugin->snapshot ($task, $vmid);
 941
 942             $self->run_hook_script ('pre-restart', $task, $logfd);
 943
 944             if ($snapshot_count > 1) {
 945                 debugmsg ('info', "resume vm", $logfd);
 946                 $cleanup->{resume} = 0;
 947                 $plugin->resume_vm ($task, $vmid);
 948                 $log_vm_online_again->();
 949             }
 950
 951             $self->run_hook_script ('post-restart', $task, $logfd);
 952
 953         } else {
 954             die "internal error - unknown mode '$mode'\n";
 955         }
 956
 957         # assemble archive image
 958         $plugin->assemble ($task, $vmid);
 959
 960         # produce archive
 961
 962         if ($opts->{stdout}) {
 963             debugmsg ('info', "sending archive to stdout", $logfd);
 964             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 965             $self->run_hook_script ('backup-end', $task, $logfd);
 966             return;
 967         }
 968
 969         debugmsg ('info', "creating archive '$task->{tarfile}'", $logfd);
 970         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 971
 972         rename ($task->{tmptar}, $task->{tarfile}) ||
 973             die "unable to rename '$task->{tmptar}' to '$task->{tarfile}'\n";
 974
 975         # determine size
 976         $task->{size} = (-s $task->{tarfile}) || 0;
 977         my $cs = format_size ($task->{size});
 978         debugmsg ('info', "archive file size: $cs", $logfd);
 979
 980         # purge older backup
 981
 982         if ($maxfiles && $opts->{remove}) {
 983             my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{tarfile});
 984             $bklist = [ sort { $b->[1] <=> $a->[1] } @$bklist ];
 985
 986             while (scalar (@$bklist) >= $maxfiles) {
 987                 my $d = pop @$bklist;
 988                 debugmsg ('info', "delete old backup '$d->[0]'", $logfd);
 989                 unlink $d->[0];
 990                 my $logfn = $d->[0];
 991                 $logfn =~ s/\.(tgz|((tar|vma)(\.(gz|lzo))?))$/\.log/;
 992                 unlink $logfn;
 993             }
 994         }
 995
 996         $self->run_hook_script ('backup-end', $task, $logfd);
 997     };
 998     my $err = $@;
 999
1000     if ($plugin) {
1001         # clean-up
1002
1003         if ($cleanup->{unlock}) {
1004             eval { $plugin->unlock_vm ($vmid); };
1005             warn $@ if $@;
1006         }
1007
1008         if ($cleanup->{prepared}) {
1009             # only call cleanup when necessary (when prepare was executed)
1010             eval { $plugin->cleanup ($task, $vmid) };
1011             warn $@ if $@;
1012         }
1013
1014         eval { $plugin->set_logfd (undef); };
1015         warn $@ if $@;
1016
1017         if ($cleanup->{resume} || $cleanup->{restart}) {
1018             eval {
1019                 $self->run_hook_script ('pre-restart', $task, $logfd);
1020                 if ($cleanup->{resume}) {
1021                     debugmsg ('info', "resume vm", $logfd);
1022                     $plugin->resume_vm ($task, $vmid);
1023                 } else {
1024                     my $running = $plugin->vm_status($vmid);
1025                     if (!$running) {
1026                         debugmsg ('info', "restarting vm", $logfd);
1027                         $plugin->start_vm ($task, $vmid);
1028                     }
1029                 }
1030                 $self->run_hook_script ('post-restart', $task, $logfd);
1031             };
1032             my $err = $@;
1033             if ($err) {
1034                 warn $err;
1035             } else {
1036                 $log_vm_online_again->();
1037             }
1038         }
1039     }
1040
1041     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
1042     warn $@ if $@;
1043
1044     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
1045     warn $@ if $@;
1046
1047     my $delay = $task->{backuptime} = time () - $vmstarttime;
1048
1049     if ($err) {
1050         $task->{state} = 'err';
1051         $task->{msg} = $err;
1052         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1053         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
1054
1055         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1056
1057     } else {
1058         $task->{state} = 'ok';
1059         my $tstr = format_time ($delay);
1060         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1061         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1062     }
1063
1064     close ($logfd) if $logfd;
1065
1066     if ($task->{tmplog} && $task->{logfile}) {
1067         system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1068     }
1069
1070     eval { $self->run_hook_script ('log-end', $task); };
1071
1072     die $err if $err && $err =~ m/^interrupted by signal$/;
1073 }
1074
1075 sub exec_backup {
1076     my ($self, $rpcenv, $authuser) = @_;
1077
1078     my $opts = $self->{opts};
1079
1080     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1081     debugmsg ('info', "skip external VMs: " . join(', ', @{$self->{skiplist}}))
1082         if scalar(@{$self->{skiplist}});
1083
1084     my $tasklist = [];
1085
1086     if ($opts->{all}) {
1087         foreach my $plugin (@{$self->{plugins}}) {
1088             my $vmlist = $plugin->vmlist();
1089             foreach my $vmid (sort @$vmlist) {
1090                 next if grep { $_ eq  $vmid } @{$opts->{exclude}};
1091                 next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], 1);
1092                 push @$tasklist, { vmid => $vmid,  state => 'todo', plugin => $plugin, mode => $opts->{mode} };
1093             }
1094         }
1095     } else {
1096         foreach my $vmid (sort @{$opts->{vmids}}) {
1097             my $plugin;
1098             foreach my $pg (@{$self->{plugins}}) {
1099                 my $vmlist = $pg->vmlist();
1100                 if (grep { $_ eq  $vmid } @$vmlist) {
1101                     $plugin = $pg;
1102                     last;
1103                 }
1104             }
1105             $rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ]);
1106             push @$tasklist, { vmid => $vmid,  state => 'todo', plugin => $plugin, mode => $opts->{mode} };
1107         }
1108     }
1109
1110     # Use in-memory files for the outer hook logs to pass them to sendmail.
1111     my $job_start_log = '';
1112     my $job_end_log = '';
1113     open my $job_start_fd, '>', \$job_start_log;
1114     open my $job_end_fd, '>', \$job_end_log;
1115
1116     my $starttime = time();
1117     my $errcount = 0;
1118     eval {
1119
1120         $self->run_hook_script ('job-start', undef, $job_start_fd);
1121
1122         foreach my $task (@$tasklist) {
1123             $self->exec_backup_task ($task);
1124             $errcount += 1 if $task->{state} ne 'ok';
1125         }
1126
1127         $self->run_hook_script ('job-end', undef, $job_end_fd);
1128     };
1129     my $err = $@;
1130
1131     $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1132
1133     if ($err) {
1134         debugmsg ('err', "Backup job failed - $err", undef, 1);
1135     } else {
1136         if ($errcount) {
1137             debugmsg ('info', "Backup job finished with errors", undef, 1);
1138         } else {
1139             debugmsg ('info', "Backup job finished successfully", undef, 1);
1140         }
1141     }
1142
1143     close $job_start_fd;
1144     close $job_end_fd;
1145
1146     my $totaltime = time() - $starttime;
1147
1148     eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); };
1149     debugmsg ('err', $@) if $@;
1150
1151     die $err if $err;
1152
1153     die "job errors\n" if $errcount;
1154
1155     unlink $pidfile;
1156 }
1157
1158
1159 sub option_exists {
1160     my $key = shift;
1161     return defined($confdesc->{$key});
1162 }
1163
1164 # add JSON properties for create and set function
1165 sub json_config_properties {
1166     my $prop = shift;
1167
1168     foreach my $opt (keys %$confdesc) {
1169         $prop->{$opt} = $confdesc->{$opt};
1170     }
1171
1172     return $prop;
1173 }
1174
1175 sub verify_vzdump_parameters {
1176     my ($param, $check_missing) = @_;
1177
1178     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1179         if $param->{all} && $param->{vmid};
1180
1181     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1182         if $param->{exclude} && $param->{vmid};
1183
1184     raise_param_exc({ pool => "option conflicts with option 'vmid'"})
1185         if $param->{pool} && $param->{vmid};
1186
1187     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
1188
1189     warn "option 'size' is deprecated and will be removed in a future " .
1190          "release, please update your script/configuration!\n"
1191         if defined($param->{size});
1192
1193     return if !$check_missing;
1194
1195     raise_param_exc({ vmid => "property is missing"})
1196         if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
1197
1198 }
1199
1200 sub stop_running_backups {
1201     my($self) = @_;
1202
1203     my $upid = PVE::Tools::file_read_firstline($pidfile);
1204     return if !$upid;
1205
1206     my $task = PVE::Tools::upid_decode($upid);
1207
1208     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1209         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1210         kill(15, $task->{pid});
1211         # wait max 15 seconds to shut down (else, do nothing for now)
1212         my $i;
1213         for ($i = 15; $i > 0; $i--) {
1214             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1215             sleep (1);
1216         }
1217         die "stopping backup process $task->{pid} failed\n" if $i == 0;
1218     }
1219 }
1220
1221 sub command_line {
1222     my ($param) = @_;
1223
1224     my $cmd = "vzdump";
1225
1226     if ($param->{vmid}) {
1227         $cmd .= " " . join(' ', PVE::Tools::split_list($param->{vmid}));
1228     }
1229
1230     foreach my $p (keys %$param) {
1231         next if $p eq 'id' || $p eq 'vmid' || $p eq 'starttime' ||
1232                 $p eq 'dow' || $p eq 'stdout' || $p eq 'enabled';
1233         my $v = $param->{$p};
1234         my $pd = $confdesc->{$p} || die "no such vzdump option '$p'\n";
1235         if ($p eq 'exclude-path') {
1236             foreach my $path (split(/\0/, $v || '')) {
1237                 $cmd .= " --$p " . PVE::Tools::shellquote($path);
1238             }
1239         } else {
1240             $cmd .= " --$p " . PVE::Tools::shellquote($v) if defined($v) && $v ne '';
1241         }
1242     }
1243
1244     return $cmd;
1245 }
1246
1247 # bash completion helpers
1248 sub complete_backup_storage {
1249
1250     my $cfg = PVE::Storage::config();
1251     my $ids = $cfg->{ids};
1252
1253     my $nodename = PVE::INotify::nodename();
1254
1255     my $res = [];
1256     foreach my $sid (keys %$ids) {
1257         my $scfg = $ids->{$sid};
1258         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $nodename, 1);
1259         next if !$scfg->{content}->{backup};
1260         push @$res, $sid;
1261     }
1262
1263     return $res;
1264 }
1265
1266 1;