PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5 use Fcntl ':flock';
   6 use PVE::Exception qw(raise_param_exc);
   7 use IO::File;
   8 use IO::Select;
   9 use IPC::Open3;
  10 use File::Path;
  11 use PVE::RPCEnvironment;
  12 use PVE::Storage;
  13 use PVE::Cluster qw(cfs_read_file);
  14 use POSIX qw(strftime);
  15 use Time::Local;
  16 use PVE::JSONSchema qw(get_standard_option);
  17 use PVE::HA::Env::PVE2;
  18 use PVE::HA::Config;
  19 use PVE::VZDump::Plugin;
  20
  21 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  22
  23 my $lockfile = '/var/run/vzdump.lock';
  24 my $pidfile = '/var/run/vzdump.pid';
  25 my $logdir = '/var/log/vzdump';
  26
  27 my @plugins = qw();
  28
  29 my $confdesc = {
  30     vmid => {
  31         type => 'string', format => 'pve-vmid-list',
  32         description => "The ID of the guest system you want to backup.",
  33         completion => \&PVE::Cluster::complete_local_vmid,
  34         optional => 1,
  35     },
  36     node => get_standard_option('pve-node', {
  37         description => "Only run if executed on this node.",
  38         completion => \&PVE::Cluster::get_nodelist,
  39         optional => 1,
  40     }),
  41     all => {
  42         type => 'boolean',
  43         description => "Backup all known guest systems on this host.",
  44         optional => 1,
  45         default => 0,
  46     },
  47     stdexcludes => {
  48         type => 'boolean',
  49         description => "Exclude temporary files and logs.",
  50         optional => 1,
  51         default => 1,
  52     },
  53     compress => {
  54         type => 'string',
  55         description => "Compress dump file.",
  56         optional => 1,
  57         enum => ['0', '1', 'gzip', 'lzo'],
  58         default => '0',
  59     },
  60     pigz=> {
  61         type => "integer",
  62         description => "Use pigz instead of gzip when N>0.".
  63             " N=1 uses half of cores, N>1 uses N as thread count.",
  64         optional => 1,
  65         default => 0,
  66     },
  67     quiet => {
  68         type => 'boolean',
  69         description => "Be quiet.",
  70         optional => 1,
  71         default => 0,
  72     },
  73     mode => {
  74         type => 'string',
  75         description => "Backup mode.",
  76         optional => 1,
  77         default => 'snapshot',
  78         enum => [ 'snapshot', 'suspend', 'stop' ],
  79     },
  80     exclude => {
  81         type => 'string', format => 'pve-vmid-list',
  82         description => "Exclude specified guest systems (assumes --all)",
  83         optional => 1,
  84     },
  85     'exclude-path' => {
  86         type => 'string', format => 'string-alist',
  87         description => "Exclude certain files/directories (shell globs).",
  88         optional => 1,
  89     },
  90     mailto => {
  91         type => 'string', format => 'string-list',
  92         description => "Comma-separated list of email addresses that should" .
  93             " receive email notifications.",
  94         optional => 1,
  95     },
  96     mailnotification => {
  97         type => 'string',
  98         description => "Specify when to send an email",
  99         optional => 1,
 100         enum => [ 'always', 'failure' ],
 101         default => 'always',
 102     },
 103     tmpdir => {
 104         type => 'string',
 105         description => "Store temporary files to specified directory.",
 106         optional => 1,
 107     },
 108     dumpdir => {
 109         type => 'string',
 110         description => "Store resulting files to specified directory.",
 111         optional => 1,
 112     },
 113     script => {
 114         type => 'string',
 115         description => "Use specified hook script.",
 116         optional => 1,
 117     },
 118     storage => get_standard_option('pve-storage-id', {
 119         description => "Store resulting file to this storage.",
 120         completion => \&complete_backup_storage,
 121         optional => 1,
 122     }),
 123     stop => {
 124         type => 'boolean',
 125         description => "Stop runnig backup jobs on this host.",
 126         optional => 1,
 127         default => 0,
 128     },
 129     size => {
 130         type => 'integer',
 131         description => "Unused, will be removed in a future release.",
 132         optional => 1,
 133         minimum => 500,
 134         default => 1024,
 135     },
 136     bwlimit => {
 137         type => 'integer',
 138         description => "Limit I/O bandwidth (KBytes per second).",
 139         optional => 1,
 140         minimum => 0,
 141         default => 0,
 142     },
 143     ionice => {
 144         type => 'integer',
 145         description => "Set CFQ ionice priority.",
 146         optional => 1,
 147         minimum => 0,
 148         maximum => 8,
 149         default => 7,
 150     },
 151     lockwait => {
 152         type => 'integer',
 153         description => "Maximal time to wait for the global lock (minutes).",
 154         optional => 1,
 155         minimum => 0,
 156         default => 3*60, # 3 hours
 157     },
 158     stopwait => {
 159         type => 'integer',
 160         description => "Maximal time to wait until a guest system is stopped (minutes).",
 161         optional => 1,
 162         minimum => 0,
 163         default => 10, # 10 minutes
 164     },
 165     maxfiles => {
 166         type => 'integer',
 167         description => "Maximal number of backup files per guest system.",
 168         optional => 1,
 169         minimum => 1,
 170         default => 1,
 171     },
 172     remove => {
 173         type => 'boolean',
 174         description => "Remove old backup files if there are more than 'maxfiles' backup files.",
 175         optional => 1,
 176         default => 1,
 177     },
 178 };
 179
 180 # Load available plugins
 181 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
 182 foreach my $plug (@pve_vzdump_classes) {
 183     my $filename = "/usr/share/perl5/$plug.pm";
 184     $filename =~ s!::!/!g;
 185     if (-f $filename) {
 186         eval { require $filename; };
 187         if (!$@) {
 188             $plug->import ();
 189             push @plugins, $plug;
 190         } else {
 191             die $@;
 192         }
 193     }
 194 }
 195
 196 # helper functions
 197
 198 sub debugmsg {
 199     my ($mtype, $msg, $logfd, $syslog) = @_;
 200
 201     PVE::VZDump::Plugin::debugmsg(@_);
 202 }
 203
 204 sub run_command {
 205     my ($logfd, $cmdstr, %param) = @_;
 206
 207     my $logfunc = sub {
 208         my $line = shift;
 209         debugmsg ('info', $line, $logfd);
 210     };
 211
 212     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
 213 }
 214
 215 sub storage_info {
 216     my $storage = shift;
 217
 218     my $cfg = PVE::Storage::config();
 219     my $scfg = PVE::Storage::storage_config($cfg, $storage);
 220     my $type = $scfg->{type};
 221
 222     die "can't use storage type '$type' for backup\n"
 223         if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'
 224               || $type eq 'cifs' || $type eq 'cephfs'));
 225     die "can't use storage '$storage' for backups - wrong content type\n"
 226         if (!$scfg->{content}->{backup});
 227
 228     PVE::Storage::activate_storage($cfg, $storage);
 229
 230     return {
 231         dumpdir => PVE::Storage::get_backup_dir($cfg, $storage),
 232         maxfiles => $scfg->{maxfiles},
 233     };
 234 }
 235
 236 sub format_size {
 237     my $size = shift;
 238
 239     my $kb = $size / 1024;
 240
 241     if ($kb < 1024) {
 242         return int ($kb) . "KB";
 243     }
 244
 245     my $mb = $size / (1024*1024);
 246
 247     if ($mb < 1024) {
 248         return int ($mb) . "MB";
 249     } else {
 250         my $gb = $mb / 1024;
 251         return sprintf ("%.2fGB", $gb);
 252     }
 253 }
 254
 255 sub format_time {
 256     my $seconds = shift;
 257
 258     my $hours = int ($seconds/3600);
 259     $seconds = $seconds - $hours*3600;
 260     my $min = int ($seconds/60);
 261     $seconds = $seconds - $min*60;
 262
 263     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 264 }
 265
 266 sub encode8bit {
 267     my ($str) = @_;
 268
 269     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 270
 271     return $str;
 272 }
 273
 274 sub escape_html {
 275     my ($str) = @_;
 276
 277     $str =~ s/&/&amp;/g;
 278     $str =~ s/</&lt;/g;
 279     $str =~ s/>/&gt;/g;
 280
 281     return $str;
 282 }
 283
 284 sub check_bin {
 285     my ($bin)  = @_;
 286
 287     foreach my $p (split (/:/, $ENV{PATH})) {
 288         my $fn = "$p/$bin";
 289         if (-x $fn) {
 290             return $fn;
 291         }
 292     }
 293
 294     die "unable to find command '$bin'\n";
 295 }
 296
 297 sub check_vmids {
 298     my (@vmids) = @_;
 299
 300     my $res = [];
 301     foreach my $vmid (@vmids) {
 302         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 303         $vmid = int ($vmid); # remove leading zeros
 304         next if !$vmid;
 305         push @$res, $vmid;
 306     }
 307
 308     return $res;
 309 }
 310
 311
 312 sub read_vzdump_defaults {
 313
 314     my $fn = "/etc/vzdump.conf";
 315
 316     my $defaults = {
 317         map {
 318             my $default = $confdesc->{$_}->{default};
 319              defined($default) ? ($_ => $default) : ()
 320         } keys %$confdesc
 321     };
 322
 323     my $raw;
 324     eval { $raw = PVE::Tools::file_get_contents($fn); };
 325     return $defaults if $@;
 326
 327     my $conf_schema = { type => 'object', properties => $confdesc, };
 328     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 329     if (my $excludes = $res->{'exclude-path'}) {
 330         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 331     }
 332     if (defined($res->{mailto})) {
 333         my @mailto = PVE::Tools::split_list($res->{mailto});
 334         $res->{mailto} = [ @mailto ];
 335     }
 336
 337     foreach my $key (keys %$defaults) {
 338         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 339     }
 340
 341     return $res;
 342 }
 343
 344 sub sendmail {
 345     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 346
 347     my $opts = $self->{opts};
 348
 349     my $mailto = $opts->{mailto};
 350
 351     return if !($mailto && scalar(@$mailto));
 352
 353     my $cmdline = $self->{cmdline};
 354
 355     my $ecount = 0;
 356     foreach my $task (@$tasklist) {
 357         $ecount++ if $task->{state} ne 'ok';
 358         chomp $task->{msg} if $task->{msg};
 359         $task->{backuptime} = 0 if !$task->{backuptime};
 360         $task->{size} = 0 if !$task->{size};
 361         $task->{tarfile} = 'unknown' if !$task->{tarfile};
 362         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 363
 364         if ($task->{state} eq 'todo') {
 365             $task->{msg} = 'aborted';
 366         }
 367     }
 368
 369     my $notify = $opts->{mailnotification} || 'always';
 370     return if (!$ecount && !$err && ($notify eq 'failure'));
 371
 372     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 373     if ($err) {
 374         if ($err =~ /\n/) {
 375             $stat .= ": multiple problems";
 376         } else {
 377             $stat .= ": $err";
 378             $err = undef;
 379         }
 380     }
 381
 382     my $hostname = `hostname -f` || PVE::INotify::nodename();
 383     chomp $hostname;
 384
 385     # text part
 386     my $text = $err ? "$err\n\n" : '';
 387     $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", qw(VMID STATUS TIME SIZE FILENAME));
 388     foreach my $task (@$tasklist) {
 389         my $vmid = $task->{vmid};
 390         if  ($task->{state} eq 'ok') {
 391
 392             $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", $vmid,
 393                                 $task->{state},
 394                                 format_time($task->{backuptime}),
 395                                 format_size ($task->{size}),
 396                                 $task->{tarfile});
 397         } else {
 398             $text .= sprintf ("%-10s %-6s %10s %8.2fMB  %s\n", $vmid,
 399                                 $task->{state},
 400                                 format_time($task->{backuptime}),
 401                                 0, '-');
 402         }
 403     }
 404
 405     $text .= "\nDetailed backup logs:\n\n";
 406     $text .= "$cmdline\n\n";
 407
 408     $text .= $detail_pre . "\n" if defined($detail_pre);
 409     foreach my $task (@$tasklist) {
 410         my $vmid = $task->{vmid};
 411         my $log = $task->{tmplog};
 412         if (!$log) {
 413             $text .= "$vmid: no log available\n\n";
 414             next;
 415         }
 416         open (TMP, "$log");
 417         while (my $line = <TMP>) { $text .= encode8bit ("$vmid: $line"); }
 418         close (TMP);
 419         $text .= "\n";
 420     }
 421     $text .= $detail_post if defined($detail_post);
 422
 423     # html part
 424     my $html = "<html><body>\n";
 425     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 426     $html .= "<table border=1 cellpadding=3>\n";
 427     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 428
 429     my $ssize = 0;
 430
 431     foreach my $task (@$tasklist) {
 432         my $vmid = $task->{vmid};
 433         my $name = $task->{hostname};
 434
 435         if  ($task->{state} eq 'ok') {
 436
 437             $ssize += $task->{size};
 438
 439             $html .= sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 440                                 $vmid, $name,
 441                                 format_time($task->{backuptime}),
 442                                 format_size ($task->{size}),
 443                                 escape_html ($task->{tarfile}));
 444         } else {
 445             $html .= sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 446                                 $vmid, $name, format_time($task->{backuptime}),
 447                                 escape_html ($task->{msg}));
 448         }
 449     }
 450
 451     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 452  format_time ($totaltime), format_size ($ssize));
 453
 454     $html .= "</table><br><br>\n";
 455     $html .= "Detailed backup logs:<br /><br />\n";
 456     $html .= "<pre>\n";
 457     $html .= escape_html($cmdline) . "\n\n";
 458
 459     $html .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 460     foreach my $task (@$tasklist) {
 461         my $vmid = $task->{vmid};
 462         my $log = $task->{tmplog};
 463         if (!$log) {
 464             $html .= "$vmid: no log available\n\n";
 465             next;
 466         }
 467         open (TMP, "$log");
 468         while (my $line = <TMP>) {
 469             if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 470                 $html .= encode8bit ("$vmid: <font color=red>".
 471                                        escape_html ($line) . "</font>");
 472             } else {
 473                 $html .= encode8bit ("$vmid: " . escape_html ($line));
 474             }
 475         }
 476         close (TMP);
 477         $html .= "\n";
 478     }
 479     $html .= escape_html($detail_post) if defined($detail_post);
 480     $html .= "</pre></body></html>\n";
 481     # end html part
 482
 483     my $subject = "vzdump backup status ($hostname) : $stat";
 484
 485     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 486     my $mailfrom = $dcconf->{email_from} || "root";
 487
 488     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 489 };
 490
 491 sub new {
 492     my ($class, $cmdline, $opts, $skiplist) = @_;
 493
 494     mkpath $logdir;
 495
 496     check_bin ('cp');
 497     check_bin ('df');
 498     check_bin ('sendmail');
 499     check_bin ('rsync');
 500     check_bin ('tar');
 501     check_bin ('mount');
 502     check_bin ('umount');
 503     check_bin ('cstream');
 504     check_bin ('ionice');
 505
 506     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 507         check_bin ('lvcreate');
 508         check_bin ('lvs');
 509         check_bin ('lvremove');
 510     }
 511
 512     my $defaults = read_vzdump_defaults();
 513
 514     my $maxfiles = $opts->{maxfiles}; # save here, because we overwrite with default
 515
 516     $opts->{remove} = 1 if !defined($opts->{remove});
 517
 518     foreach my $k (keys %$defaults) {
 519         next if $k eq 'exclude-path'; # dealt with separately
 520         if ($k eq 'dumpdir' || $k eq 'storage') {
 521             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 522                 !defined ($opts->{storage});
 523         } else {
 524             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 525         }
 526     }
 527
 528     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 529     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 530
 531     $skiplist = [] if !$skiplist;
 532     my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist };
 533
 534     my $findexcl = $self->{findexcl} = [];
 535     if ($defaults->{'exclude-path'}) {
 536         push @$findexcl, @{$defaults->{'exclude-path'}};
 537     }
 538
 539     if ($opts->{'exclude-path'}) {
 540         push @$findexcl, @{$opts->{'exclude-path'}};
 541     }
 542
 543     if ($opts->{stdexcludes}) {
 544         push @$findexcl, '/tmp/?*',
 545                          '/var/tmp/?*',
 546                          '/var/run/?*.pid';
 547     }
 548
 549     foreach my $p (@plugins) {
 550
 551         my $pd = $p->new ($self);
 552
 553         push @{$self->{plugins}}, $pd;
 554     }
 555
 556     if (!$opts->{dumpdir} && !$opts->{storage}) {
 557         $opts->{storage} = 'local';
 558     }
 559
 560     my $errors = '';
 561
 562     if ($opts->{storage}) {
 563         my $info;
 564         eval {
 565             $info = storage_info ($opts->{storage});
 566         };
 567         $errors .= "could not get storage information for '$opts->{storage}': $@"
 568             if ($@);
 569         $opts->{dumpdir} = $info->{dumpdir};
 570         $maxfiles = $info->{maxfiles} if !defined($maxfiles) && defined($info->{maxfiles});
 571     } elsif ($opts->{dumpdir}) {
 572         $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
 573             if ! -d $opts->{dumpdir};
 574     } else {
 575         die "internal error";
 576     }
 577
 578     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 579         $errors .= "\n" if $errors;
 580         $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
 581     }
 582
 583     if ($errors) {
 584         eval { $self->sendmail([], 0, $errors); };
 585         debugmsg ('err', $@) if $@;
 586         die "$errors\n";
 587     }
 588
 589     $opts->{maxfiles} = $maxfiles if defined($maxfiles);
 590
 591     return $self;
 592
 593 }
 594
 595 sub get_mount_info {
 596     my ($dir) = @_;
 597
 598     # Note: df 'available' can be negative, and percentage set to '-'
 599
 600     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 601
 602     my $res;
 603
 604     my $parser = sub {
 605         my $line = shift;
 606         if (my ($fsid, $fstype, undef, $mp) = $line =~
 607             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 608             $res = {
 609                 device => $fsid,
 610                 fstype => $fstype,
 611                 mountpoint => $mp,
 612             };
 613         }
 614     };
 615
 616     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 617     warn $@ if $@;
 618
 619     return $res;
 620 }
 621
 622 sub getlock {
 623     my ($self, $upid) = @_;
 624
 625     my $fh;
 626
 627     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 628
 629     die "missimg UPID" if !$upid; # should not happen
 630
 631     if (!open (SERVER_FLCK, ">>$lockfile")) {
 632         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 633         die "can't open lock on file '$lockfile' - $!";
 634     }
 635
 636     if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 637
 638         if (!$maxwait) {
 639             debugmsg ('err', "can't aquire lock '$lockfile' (wait = 0)", undef, 1);
 640             die "can't aquire lock '$lockfile' (wait = 0)";
 641         }
 642
 643         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 644
 645         eval {
 646             alarm ($maxwait * 60);
 647
 648             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 649
 650             if (!flock (SERVER_FLCK, LOCK_EX)) {
 651                 my $err = $!;
 652                 close (SERVER_FLCK);
 653                 alarm (0);
 654                 die "$err\n";
 655             }
 656             alarm (0);
 657         };
 658         alarm (0);
 659
 660         my $err = $@;
 661
 662         if ($err) {
 663             debugmsg ('err', "can't aquire lock '$lockfile' - $err", undef, 1);
 664             die "can't aquire lock '$lockfile' - $err";
 665         }
 666
 667         debugmsg('info', "got global lock", undef, 1);
 668     }
 669
 670     PVE::Tools::file_set_contents($pidfile, $upid);
 671 }
 672
 673 sub run_hook_script {
 674     my ($self, $phase, $task, $logfd) = @_;
 675
 676     my $opts = $self->{opts};
 677
 678     my $script = $opts->{script};
 679
 680     return if !$script;
 681
 682     my $cmd = "$script $phase";
 683
 684     $cmd .= " $task->{mode} $task->{vmid}" if ($task);
 685
 686     local %ENV;
 687
 688     # set immutable opts directly (so they are available in all phases)
 689     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 690     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 691
 692     foreach my $ek (qw(vmtype hostname tarfile logfile)) {
 693         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 694     }
 695
 696     run_command ($logfd, $cmd);
 697 }
 698
 699 sub compressor_info {
 700     my ($opts) = @_;
 701     my $opt_compress = $opts->{compress};
 702
 703     if (!$opt_compress || $opt_compress eq '0') {
 704         return undef;
 705     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 706         return ('lzop', 'lzo');
 707     } elsif ($opt_compress eq 'gzip') {
 708         if ($opts->{pigz} > 0) {
 709             my $pigz_threads = $opts->{pigz};
 710             if ($pigz_threads == 1) {
 711                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 712                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 713             }
 714             return ("pigz -p ${pigz_threads}", 'gz');
 715         } else {
 716             return ('gzip', 'gz');
 717         }
 718     } else {
 719         die "internal error - unknown compression option '$opt_compress'";
 720     }
 721 }
 722
 723 sub get_backup_file_list {
 724     my ($dir, $bkname, $exclude_fn) = @_;
 725
 726     my $bklist = [];
 727     foreach my $fn (<$dir/${bkname}-*>) {
 728         next if $exclude_fn && $fn eq $exclude_fn;
 729         if ($fn =~ m!/(${bkname}-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?)))$!) {
 730             $fn = "$dir/$1"; # untaint
 731             my $t = timelocal ($7, $6, $5, $4, $3 - 1, $2);
 732             push @$bklist, [$fn, $t];
 733         }
 734     }
 735
 736     return $bklist;
 737 }
 738
 739 sub exec_backup_task {
 740     my ($self, $task) = @_;
 741
 742     my $opts = $self->{opts};
 743
 744     my $vmid = $task->{vmid};
 745     my $plugin = $task->{plugin};
 746
 747     my $vmstarttime = time ();
 748
 749     my $logfd;
 750
 751     my $cleanup = {};
 752
 753     my $log_vm_online_again = sub {
 754         return if !defined($task->{vmstoptime});
 755         $task->{vmconttime} //= time();
 756         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 757         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 758     };
 759
 760     eval {
 761         die "unable to find VM '$vmid'\n" if !$plugin;
 762
 763         # for now we deny backups of a running ha managed service in *stop* mode
 764         # as it interferes with the HA stack (started services should not stop).
 765         if ($opts->{mode} eq 'stop' &&
 766             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 767         {
 768             die "Cannot execute a backup with stop mode on a HA managed and".
 769                 " enabled Service. Use snapshot mode or disable the Service.\n";
 770         }
 771
 772         my $vmtype = $plugin->type();
 773
 774         my $tmplog = "$logdir/$vmtype-$vmid.log";
 775
 776         my $bkname = "vzdump-$vmtype-$vmid";
 777         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime());
 778
 779         my $maxfiles = $opts->{maxfiles};
 780
 781         if ($maxfiles && !$opts->{remove}) {
 782             my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 783             die "only $maxfiles backup(s) allowed - please consider to remove old backup files.\n"
 784                 if scalar(@$bklist) >= $maxfiles;
 785         }
 786
 787         my $logfile = $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 788
 789         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 790         my ($comp, $comp_ext) = compressor_info($opts);
 791         if ($comp && $comp_ext) {
 792             $ext .= ".${comp_ext}";
 793         }
 794
 795         if ($opts->{stdout}) {
 796             $task->{tarfile} = '-';
 797         } else {
 798             my $tarfile = $task->{tarfile} = "$opts->{dumpdir}/$basename$ext";
 799             $task->{tmptar} = $task->{tarfile};
 800             $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 801             unlink $task->{tmptar};
 802         }
 803
 804         $task->{vmtype} = $vmtype;
 805
 806         if ($opts->{tmpdir}) {
 807             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp$$";
 808         } else {
 809             # dumpdir is posix? then use it as temporary dir
 810             my $info = get_mount_info($opts->{dumpdir});
 811             if ($vmtype eq 'qemu' ||
 812                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 813                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 814             } else {
 815                 $task->{tmpdir} = "/var/tmp/vzdumptmp$$";
 816                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 817                           "using $task->{tmpdir} for temporary files", $logfd);
 818             }
 819         }
 820
 821         rmtree $task->{tmpdir};
 822         mkdir $task->{tmpdir};
 823         -d $task->{tmpdir} ||
 824             die "unable to create temporary directory '$task->{tmpdir}'";
 825
 826         $logfd = IO::File->new (">$tmplog") ||
 827             die "unable to create log file '$tmplog'";
 828
 829         $task->{dumpdir} = $opts->{dumpdir};
 830         $task->{storeid} = $opts->{storage};
 831         $task->{tmplog} = $tmplog;
 832
 833         unlink $logfile;
 834
 835         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 836         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
 837
 838         $plugin->set_logfd ($logfd);
 839
 840         # test is VM is running
 841         my ($running, $status_text) = $plugin->vm_status ($vmid);
 842
 843         debugmsg ('info', "status = ${status_text}", $logfd);
 844
 845         # lock VM (prevent config changes)
 846         $plugin->lock_vm ($vmid);
 847
 848         $cleanup->{unlock} = 1;
 849
 850         # prepare
 851
 852         my $mode = $running ? $task->{mode} : 'stop';
 853
 854         if ($mode eq 'snapshot') {
 855             my %saved_task = %$task;
 856             eval { $plugin->prepare ($task, $vmid, $mode); };
 857             if (my $err = $@) {
 858                 die $err if $err !~ m/^mode failure/;
 859                 debugmsg ('info',  $err, $logfd);
 860                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 861                 $mode = 'suspend'; # so prepare is called again below
 862                 %$task = %saved_task;
 863             }
 864         }
 865
 866         $cleanup->{prepared} = 1;
 867
 868         $task->{mode} = $mode;
 869
 870         debugmsg ('info', "backup mode: $mode", $logfd);
 871
 872         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
 873             if $opts->{bwlimit};
 874
 875         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 876
 877         if ($mode eq 'stop') {
 878
 879             $plugin->prepare ($task, $vmid, $mode);
 880
 881             $self->run_hook_script ('backup-start', $task, $logfd);
 882
 883             if ($running) {
 884                 debugmsg ('info', "stopping vm", $logfd);
 885                 $task->{vmstoptime} = time();
 886                 $self->run_hook_script ('pre-stop', $task, $logfd);
 887                 $plugin->stop_vm ($task, $vmid);
 888                 $cleanup->{restart} = 1;
 889             }
 890
 891
 892         } elsif ($mode eq 'suspend') {
 893
 894             $plugin->prepare ($task, $vmid, $mode);
 895
 896             $self->run_hook_script ('backup-start', $task, $logfd);
 897
 898             if ($vmtype eq 'lxc') {
 899                 # pre-suspend rsync
 900                 $plugin->copy_data_phase1($task, $vmid);
 901             }
 902
 903             debugmsg ('info', "suspend vm", $logfd);
 904             $task->{vmstoptime} = time ();
 905             $self->run_hook_script ('pre-stop', $task, $logfd);
 906             $plugin->suspend_vm ($task, $vmid);
 907             $cleanup->{resume} = 1;
 908
 909             if ($vmtype eq 'lxc') {
 910                 # post-suspend rsync
 911                 $plugin->copy_data_phase2($task, $vmid);
 912
 913                 debugmsg ('info', "resume vm", $logfd);
 914                 $cleanup->{resume} = 0;
 915                 $self->run_hook_script('pre-restart', $task, $logfd);
 916                 $plugin->resume_vm($task, $vmid);
 917                 $self->run_hook_script('post-restart', $task, $logfd);
 918                 $log_vm_online_again->();
 919             }
 920
 921         } elsif ($mode eq 'snapshot') {
 922
 923             $self->run_hook_script ('backup-start', $task, $logfd);
 924
 925             my $snapshot_count = $task->{snapshot_count} || 0;
 926
 927             $self->run_hook_script ('pre-stop', $task, $logfd);
 928
 929             if ($snapshot_count > 1) {
 930                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 931                 $task->{vmstoptime} = time ();
 932                 $plugin->suspend_vm ($task, $vmid);
 933                 $cleanup->{resume} = 1;
 934             }
 935
 936             $plugin->snapshot ($task, $vmid);
 937
 938             $self->run_hook_script ('pre-restart', $task, $logfd);
 939
 940             if ($snapshot_count > 1) {
 941                 debugmsg ('info', "resume vm", $logfd);
 942                 $cleanup->{resume} = 0;
 943                 $plugin->resume_vm ($task, $vmid);
 944                 $log_vm_online_again->();
 945             }
 946
 947             $self->run_hook_script ('post-restart', $task, $logfd);
 948
 949         } else {
 950             die "internal error - unknown mode '$mode'\n";
 951         }
 952
 953         # assemble archive image
 954         $plugin->assemble ($task, $vmid);
 955
 956         # produce archive
 957
 958         if ($opts->{stdout}) {
 959             debugmsg ('info', "sending archive to stdout", $logfd);
 960             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 961             $self->run_hook_script ('backup-end', $task, $logfd);
 962             return;
 963         }
 964
 965         debugmsg ('info', "creating archive '$task->{tarfile}'", $logfd);
 966         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 967
 968         rename ($task->{tmptar}, $task->{tarfile}) ||
 969             die "unable to rename '$task->{tmptar}' to '$task->{tarfile}'\n";
 970
 971         # determine size
 972         $task->{size} = (-s $task->{tarfile}) || 0;
 973         my $cs = format_size ($task->{size});
 974         debugmsg ('info', "archive file size: $cs", $logfd);
 975
 976         # purge older backup
 977
 978         if ($maxfiles && $opts->{remove}) {
 979             my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{tarfile});
 980             $bklist = [ sort { $b->[1] <=> $a->[1] } @$bklist ];
 981
 982             while (scalar (@$bklist) >= $maxfiles) {
 983                 my $d = pop @$bklist;
 984                 debugmsg ('info', "delete old backup '$d->[0]'", $logfd);
 985                 unlink $d->[0];
 986                 my $logfn = $d->[0];
 987                 $logfn =~ s/\.(tgz|((tar|vma)(\.(gz|lzo))?))$/\.log/;
 988                 unlink $logfn;
 989             }
 990         }
 991
 992         $self->run_hook_script ('backup-end', $task, $logfd);
 993     };
 994     my $err = $@;
 995
 996     if ($plugin) {
 997         # clean-up
 998
 999         if ($cleanup->{unlock}) {
1000             eval { $plugin->unlock_vm ($vmid); };
1001             warn $@ if $@;
1002         }
1003
1004         if ($cleanup->{prepared}) {
1005             # only call cleanup when necessary (when prepare was executed)
1006             eval { $plugin->cleanup ($task, $vmid) };
1007             warn $@ if $@;
1008         }
1009
1010         eval { $plugin->set_logfd (undef); };
1011         warn $@ if $@;
1012
1013         if ($cleanup->{resume} || $cleanup->{restart}) {
1014             eval {
1015                 $self->run_hook_script ('pre-restart', $task, $logfd);
1016                 if ($cleanup->{resume}) {
1017                     debugmsg ('info', "resume vm", $logfd);
1018                     $plugin->resume_vm ($task, $vmid);
1019                 } else {
1020                     my $running = $plugin->vm_status($vmid);
1021                     if (!$running) {
1022                         debugmsg ('info', "restarting vm", $logfd);
1023                         $plugin->start_vm ($task, $vmid);
1024                     }
1025                 }
1026                 $self->run_hook_script ('post-restart', $task, $logfd);
1027             };
1028             my $err = $@;
1029             if ($err) {
1030                 warn $err;
1031             } else {
1032                 $log_vm_online_again->();
1033             }
1034         }
1035     }
1036
1037     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
1038     warn $@ if $@;
1039
1040     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
1041     warn $@ if $@;
1042
1043     my $delay = $task->{backuptime} = time () - $vmstarttime;
1044
1045     if ($err) {
1046         $task->{state} = 'err';
1047         $task->{msg} = $err;
1048         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1049         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
1050
1051         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1052
1053     } else {
1054         $task->{state} = 'ok';
1055         my $tstr = format_time ($delay);
1056         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1057         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1058     }
1059
1060     close ($logfd) if $logfd;
1061
1062     if ($task->{tmplog} && $task->{logfile}) {
1063         system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1064     }
1065
1066     eval { $self->run_hook_script ('log-end', $task); };
1067
1068     die $err if $err && $err =~ m/^interrupted by signal$/;
1069 }
1070
1071 sub exec_backup {
1072     my ($self, $rpcenv, $authuser) = @_;
1073
1074     my $opts = $self->{opts};
1075
1076     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1077     debugmsg ('info', "skip external VMs: " . join(', ', @{$self->{skiplist}}))
1078         if scalar(@{$self->{skiplist}});
1079
1080     my $tasklist = [];
1081
1082     if ($opts->{all}) {
1083         foreach my $plugin (@{$self->{plugins}}) {
1084             my $vmlist = $plugin->vmlist();
1085             foreach my $vmid (sort @$vmlist) {
1086                 next if grep { $_ eq  $vmid } @{$opts->{exclude}};
1087                 next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], 1);
1088                 push @$tasklist, { vmid => $vmid,  state => 'todo', plugin => $plugin, mode => $opts->{mode} };
1089             }
1090         }
1091     } else {
1092         foreach my $vmid (sort @{$opts->{vmids}}) {
1093             my $plugin;
1094             foreach my $pg (@{$self->{plugins}}) {
1095                 my $vmlist = $pg->vmlist();
1096                 if (grep { $_ eq  $vmid } @$vmlist) {
1097                     $plugin = $pg;
1098                     last;
1099                 }
1100             }
1101             $rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ]);
1102             push @$tasklist, { vmid => $vmid,  state => 'todo', plugin => $plugin, mode => $opts->{mode} };
1103         }
1104     }
1105
1106     # Use in-memory files for the outer hook logs to pass them to sendmail.
1107     my $job_start_log = '';
1108     my $job_end_log = '';
1109     open my $job_start_fd, '>', \$job_start_log;
1110     open my $job_end_fd, '>', \$job_end_log;
1111
1112     my $starttime = time();
1113     my $errcount = 0;
1114     eval {
1115
1116         $self->run_hook_script ('job-start', undef, $job_start_fd);
1117
1118         foreach my $task (@$tasklist) {
1119             $self->exec_backup_task ($task);
1120             $errcount += 1 if $task->{state} ne 'ok';
1121         }
1122
1123         $self->run_hook_script ('job-end', undef, $job_end_fd);
1124     };
1125     my $err = $@;
1126
1127     $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1128
1129     if ($err) {
1130         debugmsg ('err', "Backup job failed - $err", undef, 1);
1131     } else {
1132         if ($errcount) {
1133             debugmsg ('info', "Backup job finished with errors", undef, 1);
1134         } else {
1135             debugmsg ('info', "Backup job finished successfully", undef, 1);
1136         }
1137     }
1138
1139     close $job_start_fd;
1140     close $job_end_fd;
1141
1142     my $totaltime = time() - $starttime;
1143
1144     eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); };
1145     debugmsg ('err', $@) if $@;
1146
1147     die $err if $err;
1148
1149     die "job errors\n" if $errcount;
1150
1151     unlink $pidfile;
1152 }
1153
1154
1155 sub option_exists {
1156     my $key = shift;
1157     return defined($confdesc->{$key});
1158 }
1159
1160 # add JSON properties for create and set function
1161 sub json_config_properties {
1162     my $prop = shift;
1163
1164     foreach my $opt (keys %$confdesc) {
1165         $prop->{$opt} = $confdesc->{$opt};
1166     }
1167
1168     return $prop;
1169 }
1170
1171 sub verify_vzdump_parameters {
1172     my ($param, $check_missing) = @_;
1173
1174     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1175         if $param->{all} && $param->{vmid};
1176
1177     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1178         if $param->{exclude} && $param->{vmid};
1179
1180     $param->{all} = 1 if defined($param->{exclude});
1181
1182     warn "option 'size' is deprecated and will be removed in a future " .
1183          "release, please update your script/configuration!\n"
1184         if defined($param->{size});
1185
1186     return if !$check_missing;
1187
1188     raise_param_exc({ vmid => "property is missing"})
1189         if !($param->{all} || $param->{stop}) && !$param->{vmid};
1190
1191 }
1192
1193 sub stop_running_backups {
1194     my($self) = @_;
1195
1196     my $upid = PVE::Tools::file_read_firstline($pidfile);
1197     return if !$upid;
1198
1199     my $task = PVE::Tools::upid_decode($upid);
1200
1201     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1202         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1203         kill(15, $task->{pid});
1204         # wait max 15 seconds to shut down (else, do nothing for now)
1205         my $i;
1206         for ($i = 15; $i > 0; $i--) {
1207             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1208             sleep (1);
1209         }
1210         die "stoping backup process $task->{pid} failed\n" if $i == 0;
1211     }
1212 }
1213
1214 sub command_line {
1215     my ($param) = @_;
1216
1217     my $cmd = "vzdump";
1218
1219     if ($param->{vmid}) {
1220         $cmd .= " " . join(' ', PVE::Tools::split_list($param->{vmid}));
1221     }
1222
1223     foreach my $p (keys %$param) {
1224         next if $p eq 'id' || $p eq 'vmid' || $p eq 'starttime' ||
1225                 $p eq 'dow' || $p eq 'stdout' || $p eq 'enabled';
1226         my $v = $param->{$p};
1227         my $pd = $confdesc->{$p} || die "no such vzdump option '$p'\n";
1228         if ($p eq 'exclude-path') {
1229             foreach my $path (split(/\0/, $v || '')) {
1230                 $cmd .= " --$p " . PVE::Tools::shellquote($path);
1231             }
1232         } else {
1233             $cmd .= " --$p " . PVE::Tools::shellquote($v) if defined($v) && $v ne '';
1234         }
1235     }
1236
1237     return $cmd;
1238 }
1239
1240 # bash completion helpers
1241 sub complete_backup_storage {
1242
1243     my $cfg = PVE::Storage::config();
1244     my $ids = $cfg->{ids};
1245
1246     my $nodename = PVE::INotify::nodename();
1247
1248     my $res = [];
1249     foreach my $sid (keys %$ids) {
1250         my $scfg = $ids->{$sid};
1251         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $nodename, 1);
1252         next if !$scfg->{content}->{backup};
1253         push @$res, $sid;
1254     }
1255
1256     return $res;
1257 }
1258
1259 1;