PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5 use Fcntl ':flock';
   6 use PVE::Exception qw(raise_param_exc);
   7 use PVE::SafeSyslog;
   8 use IO::File;
   9 use IO::Select;
  10 use IPC::Open3;
  11 use POSIX qw(strftime);
  12 use File::Path;
  13 use PVE::RPCEnvironment;
  14 use PVE::Storage;
  15 use PVE::Cluster qw(cfs_read_file);
  16 use Time::localtime;
  17 use Time::Local;
  18 use PVE::JSONSchema qw(get_standard_option);
  19 use PVE::HA::Env::PVE2;
  20 use PVE::HA::Config;
  21
  22 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  23
  24 my $lockfile = '/var/run/vzdump.lock';
  25
  26 my $pidfile = '/var/run/vzdump.pid';
  27
  28 my $logdir = '/var/log/vzdump';
  29
  30 my @plugins = qw();
  31
  32 my $confdesc = {
  33     vmid => {
  34         type => 'string', format => 'pve-vmid-list',
  35         description => "The ID of the guest system you want to backup.",
  36         completion => \&PVE::Cluster::complete_local_vmid,
  37         optional => 1,
  38     },
  39     node => get_standard_option('pve-node', {
  40         description => "Only run if executed on this node.",
  41         completion => \&PVE::Cluster::get_nodelist,
  42         optional => 1,
  43     }),
  44     all => {
  45         type => 'boolean',
  46         description => "Backup all known guest systems on this host.",
  47         optional => 1,
  48         default => 0,
  49     },
  50     stdexcludes => {
  51         type => 'boolean',
  52         description => "Exclude temporary files and logs.",
  53         optional => 1,
  54         default => 1,
  55     },
  56     compress => {
  57         type => 'string',
  58         description => "Compress dump file.",
  59         optional => 1,
  60         enum => ['0', '1', 'gzip', 'lzo'],
  61         default => '0',
  62     },
  63     pigz=> {
  64         type => "integer",
  65         description => "Use pigz instead of gzip when N>0.".
  66             " N=1 uses half of cores, N>1 uses N as thread count.",
  67         optional => 1,
  68         default => 0,
  69     },
  70     quiet => {
  71         type => 'boolean',
  72         description => "Be quiet.",
  73         optional => 1,
  74         default => 0,
  75     },
  76     mode => {
  77         type => 'string',
  78         description => "Backup mode.",
  79         optional => 1,
  80         default => 'snapshot',
  81         enum => [ 'snapshot', 'suspend', 'stop' ],
  82     },
  83     exclude => {
  84         type => 'string', format => 'pve-vmid-list',
  85         description => "Exclude specified guest systems (assumes --all)",
  86         optional => 1,
  87     },
  88     'exclude-path' => {
  89         type => 'string', format => 'string-alist',
  90         description => "Exclude certain files/directories (shell globs).",
  91         optional => 1,
  92     },
  93     mailto => {
  94         type => 'string', format => 'string-list',
  95         description => "",
  96         optional => 1,
  97     },
  98     mailnotification => {
  99         type => 'string',
 100         description => "Specify when to send an email",
 101         optional => 1,
 102         enum => [ 'always', 'failure' ],
 103         default => 'always',
 104     },
 105     tmpdir => {
 106         type => 'string',
 107         description => "Store temporary files to specified directory.",
 108         optional => 1,
 109     },
 110     dumpdir => {
 111         type => 'string',
 112         description => "Store resulting files to specified directory.",
 113         optional => 1,
 114     },
 115     script => {
 116         type => 'string',
 117         description => "Use specified hook script.",
 118         optional => 1,
 119     },
 120     storage => get_standard_option('pve-storage-id', {
 121         description => "Store resulting file to this storage.",
 122         completion => \&complete_backup_storage,
 123         optional => 1,
 124     }),
 125     stop => {
 126         type => 'boolean',
 127         description => "Stop runnig backup jobs on this host.",
 128         optional => 1,
 129         default => 0,
 130     },
 131     size => {
 132         type => 'integer',
 133         description => "Unused, will be removed in a future release.",
 134         optional => 1,
 135         minimum => 500,
 136         default => 1024,
 137     },
 138     bwlimit => {
 139         type => 'integer',
 140         description => "Limit I/O bandwidth (KBytes per second).",
 141         optional => 1,
 142         minimum => 0,
 143         default => 0,
 144     },
 145     ionice => {
 146         type => 'integer',
 147         description => "Set CFQ ionice priority.",
 148         optional => 1,
 149         minimum => 0,
 150         maximum => 8,
 151         default => 7,
 152     },
 153     lockwait => {
 154         type => 'integer',
 155         description => "Maximal time to wait for the global lock (minutes).",
 156         optional => 1,
 157         minimum => 0,
 158         default => 3*60, # 3 hours
 159     },
 160     stopwait => {
 161         type => 'integer',
 162         description => "Maximal time to wait until a guest system is stopped (minutes).",
 163         optional => 1,
 164         minimum => 0,
 165         default => 10, # 10 minutes
 166     },
 167     maxfiles => {
 168         type => 'integer',
 169         description => "Maximal number of backup files per guest system.",
 170         optional => 1,
 171         minimum => 1,
 172         default => 1,
 173     },
 174     remove => {
 175         type => 'boolean',
 176         description => "Remove old backup files if there are more than 'maxfiles' backup files.",
 177         optional => 1,
 178         default => 1,
 179     },
 180 };
 181
 182 # Load available plugins
 183 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
 184 foreach my $plug (@pve_vzdump_classes) {
 185     my $filename = "/usr/share/perl5/$plug.pm";
 186     $filename =~ s!::!/!g;
 187     if (-f $filename) {
 188         eval { require $filename; };
 189         if (!$@) {
 190             $plug->import ();
 191             push @plugins, $plug;
 192         } else {
 193             warn $@;
 194         }
 195     }
 196 }
 197
 198 # helper functions
 199
 200 my $debugstattxt = {
 201     err =>  'ERROR:',
 202     info => 'INFO:',
 203     warn => 'WARN:',
 204 };
 205
 206 sub debugmsg {
 207     my ($mtype, $msg, $logfd, $syslog) = @_;
 208
 209     chomp $msg;
 210
 211     return if !$msg;
 212
 213     my $pre = $debugstattxt->{$mtype} || $debugstattxt->{'err'};
 214
 215     my $timestr = strftime ("%b %d %H:%M:%S", CORE::localtime);
 216
 217     syslog ($mtype eq 'info' ? 'info' : 'err', "$pre $msg") if $syslog;
 218
 219     foreach my $line (split (/\n/, $msg)) {
 220         print STDERR "$pre $line\n";
 221         print $logfd "$timestr $pre $line\n" if $logfd;
 222     }
 223 }
 224
 225 sub run_command {
 226     my ($logfd, $cmdstr, %param) = @_;
 227
 228     my $logfunc = sub {
 229         my $line = shift;
 230         debugmsg ('info', $line, $logfd);
 231     };
 232
 233     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
 234 }
 235
 236 sub storage_info {
 237     my $storage = shift;
 238
 239     my $cfg = PVE::Storage::config();
 240     my $scfg = PVE::Storage::storage_config($cfg, $storage);
 241     my $type = $scfg->{type};
 242
 243     die "can't use storage type '$type' for backup\n"
 244         if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'));
 245     die "can't use storage '$storage' for backups - wrong content type\n"
 246         if (!$scfg->{content}->{backup});
 247
 248     PVE::Storage::activate_storage($cfg, $storage);
 249
 250     return {
 251         dumpdir => PVE::Storage::get_backup_dir($cfg, $storage),
 252         maxfiles => $scfg->{maxfiles},
 253     };
 254 }
 255
 256 sub format_size {
 257     my $size = shift;
 258
 259     my $kb = $size / 1024;
 260
 261     if ($kb < 1024) {
 262         return int ($kb) . "KB";
 263     }
 264
 265     my $mb = $size / (1024*1024);
 266
 267     if ($mb < 1024) {
 268         return int ($mb) . "MB";
 269     } else {
 270         my $gb = $mb / 1024;
 271         return sprintf ("%.2fGB", $gb);
 272     }
 273 }
 274
 275 sub format_time {
 276     my $seconds = shift;
 277
 278     my $hours = int ($seconds/3600);
 279     $seconds = $seconds - $hours*3600;
 280     my $min = int ($seconds/60);
 281     $seconds = $seconds - $min*60;
 282
 283     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 284 }
 285
 286 sub encode8bit {
 287     my ($str) = @_;
 288
 289     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 290
 291     return $str;
 292 }
 293
 294 sub escape_html {
 295     my ($str) = @_;
 296
 297     $str =~ s/&/&amp;/g;
 298     $str =~ s/</&lt;/g;
 299     $str =~ s/>/&gt;/g;
 300
 301     return $str;
 302 }
 303
 304 sub check_bin {
 305     my ($bin)  = @_;
 306
 307     foreach my $p (split (/:/, $ENV{PATH})) {
 308         my $fn = "$p/$bin";
 309         if (-x $fn) {
 310             return $fn;
 311         }
 312     }
 313
 314     die "unable to find command '$bin'\n";
 315 }
 316
 317 sub check_vmids {
 318     my (@vmids) = @_;
 319
 320     my $res = [];
 321     foreach my $vmid (@vmids) {
 322         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 323         $vmid = int ($vmid); # remove leading zeros
 324         next if !$vmid;
 325         push @$res, $vmid;
 326     }
 327
 328     return $res;
 329 }
 330
 331
 332 sub read_vzdump_defaults {
 333
 334     my $fn = "/etc/vzdump.conf";
 335
 336     my $defaults = {
 337         map {
 338             my $default = $confdesc->{$_}->{default};
 339              defined($default) ? ($_ => $default) : ()
 340         } keys %$confdesc
 341     };
 342
 343     my $raw;
 344     eval { $raw = PVE::Tools::file_get_contents($fn); };
 345     return $defaults if $@;
 346
 347     my $conf_schema = { type => 'object', properties => $confdesc, };
 348     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 349     if (my $excludes = $res->{'exclude-path'}) {
 350         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 351     }
 352
 353     foreach my $key (keys %$defaults) {
 354         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 355     }
 356
 357     return $res;
 358 }
 359
 360 sub sendmail {
 361     my ($self, $tasklist, $totaltime, $err) = @_;
 362
 363     my $opts = $self->{opts};
 364
 365     my $mailto = $opts->{mailto};
 366
 367     return if !($mailto && scalar(@$mailto));
 368
 369     my $cmdline = $self->{cmdline};
 370
 371     my $ecount = 0;
 372     foreach my $task (@$tasklist) {
 373         $ecount++ if $task->{state} ne 'ok';
 374         chomp $task->{msg} if $task->{msg};
 375         $task->{backuptime} = 0 if !$task->{backuptime};
 376         $task->{size} = 0 if !$task->{size};
 377         $task->{tarfile} = 'unknown' if !$task->{tarfile};
 378         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 379
 380         if ($task->{state} eq 'todo') {
 381             $task->{msg} = 'aborted';
 382         }
 383     }
 384
 385     my $notify = $opts->{mailnotification} || 'always';
 386     return if (!$ecount && !$err && ($notify eq 'failure'));
 387
 388     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 389     $stat .= ": $err" if $err;
 390
 391     my $hostname = `hostname -f` || PVE::INotify::nodename();
 392     chomp $hostname;
 393
 394     # text part
 395     my $text = sprintf ("%-10s %-6s %10s %10s  %s\n", qw(VMID STATUS TIME SIZE FILENAME));
 396     foreach my $task (@$tasklist) {
 397         my $vmid = $task->{vmid};
 398         if  ($task->{state} eq 'ok') {
 399
 400             $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", $vmid,
 401                                 $task->{state},
 402                                 format_time($task->{backuptime}),
 403                                 format_size ($task->{size}),
 404                                 $task->{tarfile});
 405         } else {
 406             $text .= sprintf ("%-10s %-6s %10s %8.2fMB  %s\n", $vmid,
 407                                 $task->{state},
 408                                 format_time($task->{backuptime}),
 409                                 0, '-');
 410         }
 411     }
 412
 413     $text .= "Detailed backup logs:\n\n";
 414     $text .= "$cmdline\n\n";
 415
 416     foreach my $task (@$tasklist) {
 417         my $vmid = $task->{vmid};
 418         my $log = $task->{tmplog};
 419         if (!$log) {
 420             $text .= "$vmid: no log available\n\n";
 421             next;
 422         }
 423         open (TMP, "$log");
 424         while (my $line = <TMP>) { $text .= encode8bit ("$vmid: $line"); }
 425         close (TMP);
 426         $text .= "\n";
 427     }
 428
 429     # html part
 430     my $html = "<html><body>\n";
 431     $html .= "<table border=1 cellpadding=3>\n";
 432     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 433
 434     my $ssize = 0;
 435
 436     foreach my $task (@$tasklist) {
 437         my $vmid = $task->{vmid};
 438         my $name = $task->{hostname};
 439
 440         if  ($task->{state} eq 'ok') {
 441
 442             $ssize += $task->{size};
 443
 444             $html .= sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 445                                 $vmid, $name,
 446                                 format_time($task->{backuptime}),
 447                                 format_size ($task->{size}),
 448                                 escape_html ($task->{tarfile}));
 449         } else {
 450             $html .= sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 451                                 $vmid, $name, format_time($task->{backuptime}),
 452                                 escape_html ($task->{msg}));
 453         }
 454     }
 455
 456     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 457  format_time ($totaltime), format_size ($ssize));
 458
 459     $html .= "</table><br><br>\n";
 460     $html .= "Detailed backup logs:<br /><br />\n";
 461     $html .= "<pre>\n";
 462     $html .= escape_html($cmdline) . "\n\n";
 463
 464     foreach my $task (@$tasklist) {
 465         my $vmid = $task->{vmid};
 466         my $log = $task->{tmplog};
 467         if (!$log) {
 468             $html .= "$vmid: no log available\n\n";
 469             next;
 470         }
 471         open (TMP, "$log");
 472         while (my $line = <TMP>) {
 473             if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 474                 $html .= encode8bit ("$vmid: <font color=red>".
 475                                        escape_html ($line) . "</font>");
 476             } else {
 477                 $html .= encode8bit ("$vmid: " . escape_html ($line));
 478             }
 479         }
 480         close (TMP);
 481         $html .= "\n";
 482     }
 483     $html .= "</pre></body></html>\n";
 484     # end html part
 485
 486     my $subject = "vzdump backup status ($hostname) : $stat";
 487
 488     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 489     my $mailfrom = $dcconf->{email_from} || "root";
 490
 491     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 492 };
 493
 494 sub new {
 495     my ($class, $cmdline, $opts, $skiplist) = @_;
 496
 497     mkpath $logdir;
 498
 499     check_bin ('cp');
 500     check_bin ('df');
 501     check_bin ('sendmail');
 502     check_bin ('rsync');
 503     check_bin ('tar');
 504     check_bin ('mount');
 505     check_bin ('umount');
 506     check_bin ('cstream');
 507     check_bin ('ionice');
 508
 509     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 510         check_bin ('lvcreate');
 511         check_bin ('lvs');
 512         check_bin ('lvremove');
 513     }
 514
 515     my $defaults = read_vzdump_defaults();
 516
 517     my $maxfiles = $opts->{maxfiles}; # save here, because we overwrite with default
 518
 519     $opts->{remove} = 1 if !defined($opts->{remove});
 520
 521     foreach my $k (keys %$defaults) {
 522         next if $k eq 'exclude-path'; # dealt with separately
 523         if ($k eq 'dumpdir' || $k eq 'storage') {
 524             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 525                 !defined ($opts->{storage});
 526         } else {
 527             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 528         }
 529     }
 530
 531     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 532     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 533
 534     $skiplist = [] if !$skiplist;
 535     my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist };
 536
 537     my $findexcl = $self->{findexcl} = [];
 538     if ($defaults->{'exclude-path'}) {
 539         push @$findexcl, @{$defaults->{'exclude-path'}};
 540     }
 541
 542     if ($opts->{'exclude-path'}) {
 543         push @$findexcl, @{$opts->{'exclude-path'}};
 544     }
 545
 546     if ($opts->{stdexcludes}) {
 547         push @$findexcl, '/var/log/?*',
 548                          '/tmp/?*',
 549                          '/var/tmp/?*',
 550                          '/var/run/?*.pid';
 551     }
 552
 553     foreach my $p (@plugins) {
 554
 555         my $pd = $p->new ($self);
 556
 557         push @{$self->{plugins}}, $pd;
 558     }
 559
 560     if (!$opts->{dumpdir} && !$opts->{storage}) {
 561         $opts->{storage} = 'local';
 562     }
 563
 564     if ($opts->{storage}) {
 565         my $info = storage_info ($opts->{storage});
 566         $opts->{dumpdir} = $info->{dumpdir};
 567         $maxfiles = $info->{maxfiles} if !defined($maxfiles) && defined($info->{maxfiles});
 568     } elsif ($opts->{dumpdir}) {
 569         die "dumpdir '$opts->{dumpdir}' does not exist\n"
 570             if ! -d $opts->{dumpdir};
 571     } else {
 572         die "internal error";
 573     }
 574
 575     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 576         die "tmpdir '$opts->{tmpdir}' does not exist\n";
 577     }
 578
 579     $opts->{maxfiles} = $maxfiles if defined($maxfiles);
 580
 581     return $self;
 582
 583 }
 584
 585 sub get_mount_info {
 586     my ($dir) = @_;
 587
 588     # Note: df 'available' can be negative, and percentage set to '-'
 589
 590     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 591
 592     my $res;
 593
 594     my $parser = sub {
 595         my $line = shift;
 596         if (my ($fsid, $fstype, undef, $mp) = $line =~
 597             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 598             $res = {
 599                 device => $fsid,
 600                 fstype => $fstype,
 601                 mountpoint => $mp,
 602             };
 603         }
 604     };
 605
 606     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 607     warn $@ if $@;
 608
 609     return $res;
 610 }
 611
 612 sub getlock {
 613     my ($self, $upid) = @_;
 614
 615     my $fh;
 616
 617     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 618
 619     die "missimg UPID" if !$upid; # should not happen
 620
 621     if (!open (SERVER_FLCK, ">>$lockfile")) {
 622         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 623         die "can't open lock on file '$lockfile' - $!";
 624     }
 625
 626     if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 627
 628         if (!$maxwait) {
 629             debugmsg ('err', "can't aquire lock '$lockfile' (wait = 0)", undef, 1);
 630             die "can't aquire lock '$lockfile' (wait = 0)";
 631         }
 632
 633         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 634
 635         eval {
 636             alarm ($maxwait * 60);
 637
 638             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 639
 640             if (!flock (SERVER_FLCK, LOCK_EX)) {
 641                 my $err = $!;
 642                 close (SERVER_FLCK);
 643                 alarm (0);
 644                 die "$err\n";
 645             }
 646             alarm (0);
 647         };
 648         alarm (0);
 649
 650         my $err = $@;
 651
 652         if ($err) {
 653             debugmsg ('err', "can't aquire lock '$lockfile' - $err", undef, 1);
 654             die "can't aquire lock '$lockfile' - $err";
 655         }
 656
 657         debugmsg('info', "got global lock", undef, 1);
 658     }
 659
 660     PVE::Tools::file_set_contents($pidfile, $upid);
 661 }
 662
 663 sub run_hook_script {
 664     my ($self, $phase, $task, $logfd) = @_;
 665
 666     my $opts = $self->{opts};
 667
 668     my $script = $opts->{script};
 669
 670     return if !$script;
 671
 672     my $cmd = "$script $phase";
 673
 674     $cmd .= " $task->{mode} $task->{vmid}" if ($task);
 675
 676     local %ENV;
 677
 678     # set immutable opts directly (so they are available in all phases)
 679     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 680     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 681
 682     foreach my $ek (qw(vmtype hostname tarfile logfile)) {
 683         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 684     }
 685
 686     run_command ($logfd, $cmd);
 687 }
 688
 689 sub compressor_info {
 690     my ($opts) = @_;
 691     my $opt_compress = $opts->{compress};
 692
 693     if (!$opt_compress || $opt_compress eq '0') {
 694         return undef;
 695     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 696         return ('lzop', 'lzo');
 697     } elsif ($opt_compress eq 'gzip') {
 698         if ($opts->{pigz} > 0) {
 699             # As default use int((#cores + 1)/2), we need #cores+1 for the case that #cores = 1
 700             my $cores = POSIX::sysconf(84);
 701             my $pigz_threads = ($opts->{pigz} > 1) ? $opts->{pigz} : int(($cores + 1)/2);
 702             return ("pigz -p ${pigz_threads}", 'gz');
 703         } else {
 704             return ('gzip', 'gz');
 705         }
 706     } else {
 707         die "internal error - unknown compression option '$opt_compress'";
 708     }
 709 }
 710
 711 sub get_backup_file_list {
 712     my ($dir, $bkname, $exclude_fn) = @_;
 713
 714     my $bklist = [];
 715     foreach my $fn (<$dir/${bkname}-*>) {
 716         next if $exclude_fn && $fn eq $exclude_fn;
 717         if ($fn =~ m!/(${bkname}-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?)))$!) {
 718             $fn = "$dir/$1"; # untaint
 719             my $t = timelocal ($7, $6, $5, $4, $3 - 1, $2 - 1900);
 720             push @$bklist, [$fn, $t];
 721         }
 722     }
 723
 724     return $bklist;
 725 }
 726
 727 sub exec_backup_task {
 728     my ($self, $task) = @_;
 729
 730     my $opts = $self->{opts};
 731
 732     my $vmid = $task->{vmid};
 733     my $plugin = $task->{plugin};
 734
 735     my $vmstarttime = time ();
 736
 737     my $logfd;
 738
 739     my $cleanup = {};
 740
 741     my $vmstoptime = 0;
 742
 743     eval {
 744         die "unable to find VM '$vmid'\n" if !$plugin;
 745
 746         # for now we deny backups of a running ha managed service in *stop* mode
 747         # as it interferes with the HA stack (enabled services should not stop).
 748         if ($opts->{mode} eq 'stop' &&
 749             PVE::HA::Config::vm_is_ha_managed($vmid, 'enabled'))
 750         {
 751             die "Cannot execute a backup with stop mode on a HA managed and".
 752                 " enabled Service. Use snapshot mode or disable the Service.\n";
 753         }
 754
 755         my $vmtype = $plugin->type();
 756
 757         my $tmplog = "$logdir/$vmtype-$vmid.log";
 758
 759         my $lt = localtime();
 760
 761         my $bkname = "vzdump-$vmtype-$vmid";
 762         my $basename = sprintf "${bkname}-%04d_%02d_%02d-%02d_%02d_%02d",
 763         $lt->year + 1900, $lt->mon + 1, $lt->mday,
 764         $lt->hour, $lt->min, $lt->sec;
 765
 766         my $maxfiles = $opts->{maxfiles};
 767
 768         if ($maxfiles && !$opts->{remove}) {
 769             my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 770             die "only $maxfiles backup(s) allowed - please consider to remove old backup files.\n"
 771                 if scalar(@$bklist) >= $maxfiles;
 772         }
 773
 774         my $logfile = $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 775
 776         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 777         my ($comp, $comp_ext) = compressor_info($opts);
 778         if ($comp && $comp_ext) {
 779             $ext .= ".${comp_ext}";
 780         }
 781
 782         if ($opts->{stdout}) {
 783             $task->{tarfile} = '-';
 784         } else {
 785             my $tarfile = $task->{tarfile} = "$opts->{dumpdir}/$basename$ext";
 786             $task->{tmptar} = $task->{tarfile};
 787             $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 788             unlink $task->{tmptar};
 789         }
 790
 791         $task->{vmtype} = $vmtype;
 792
 793         if ($opts->{tmpdir}) {
 794             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp$$";
 795         } else {
 796             # dumpdir is posix? then use it as temporary dir
 797             my $info = get_mount_info($opts->{dumpdir});
 798             if ($vmtype eq 'qemu' ||
 799                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 800                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 801             } else {
 802                 $task->{tmpdir} = "/var/tmp/vzdumptmp$$";
 803                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 804                           "using $task->{tmpdir} for temporary files", $logfd);
 805             }
 806         }
 807
 808         rmtree $task->{tmpdir};
 809         mkdir $task->{tmpdir};
 810         -d $task->{tmpdir} ||
 811             die "unable to create temporary directory '$task->{tmpdir}'";
 812
 813         $logfd = IO::File->new (">$tmplog") ||
 814             die "unable to create log file '$tmplog'";
 815
 816         $task->{dumpdir} = $opts->{dumpdir};
 817         $task->{storeid} = $opts->{storage};
 818         $task->{tmplog} = $tmplog;
 819
 820         unlink $logfile;
 821
 822         debugmsg ('info',  "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 823
 824         $plugin->set_logfd ($logfd);
 825
 826         # test is VM is running
 827         my ($running, $status_text) = $plugin->vm_status ($vmid);
 828
 829         debugmsg ('info', "status = ${status_text}", $logfd);
 830
 831         # lock VM (prevent config changes)
 832         $plugin->lock_vm ($vmid);
 833
 834         $cleanup->{unlock} = 1;
 835
 836         # prepare
 837
 838         my $mode = $running ? $opts->{mode} : 'stop';
 839
 840         if ($mode eq 'snapshot') {
 841             my %saved_task = %$task;
 842             eval { $plugin->prepare ($task, $vmid, $mode); };
 843             if (my $err = $@) {
 844                 die $err if $err !~ m/^mode failure/;
 845                 debugmsg ('info',  $err, $logfd);
 846                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 847                 $mode = 'suspend'; # so prepare is called again below
 848                 %$task = %saved_task;
 849             }
 850         }
 851
 852         $task->{mode} = $mode;
 853
 854         debugmsg ('info', "backup mode: $mode", $logfd);
 855
 856         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
 857             if $opts->{bwlimit};
 858
 859         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 860
 861         if ($mode eq 'stop') {
 862
 863             $plugin->prepare ($task, $vmid, $mode);
 864
 865             $self->run_hook_script ('backup-start', $task, $logfd);
 866
 867             if ($running) {
 868                 debugmsg ('info', "stopping vm", $logfd);
 869                 $vmstoptime = time ();
 870                 $self->run_hook_script ('pre-stop', $task, $logfd);
 871                 $plugin->stop_vm ($task, $vmid);
 872                 $cleanup->{restart} = 1;
 873             }
 874
 875
 876         } elsif ($mode eq 'suspend') {
 877
 878             $plugin->prepare ($task, $vmid, $mode);
 879
 880             $self->run_hook_script ('backup-start', $task, $logfd);
 881
 882             if ($vmtype eq 'lxc') {
 883                 # pre-suspend rsync
 884                 $plugin->copy_data_phase1($task, $vmid);
 885             }
 886
 887             debugmsg ('info', "suspend vm", $logfd);
 888             $vmstoptime = time ();
 889             $self->run_hook_script ('pre-stop', $task, $logfd);
 890             $plugin->suspend_vm ($task, $vmid);
 891             $cleanup->{resume} = 1;
 892
 893             if ($vmtype eq 'lxc') {
 894                 # post-suspend rsync
 895                 $plugin->copy_data_phase2($task, $vmid);
 896
 897                 debugmsg ('info', "resume vm", $logfd);
 898                 $cleanup->{resume} = 0;
 899                 $self->run_hook_script('pre-restart', $task, $logfd);
 900                 $plugin->resume_vm($task, $vmid);
 901                 my $delay = time () - $vmstoptime;
 902                 debugmsg('info', "vm is online again after $delay seconds", $logfd);
 903             }
 904
 905         } elsif ($mode eq 'snapshot') {
 906
 907             $self->run_hook_script ('backup-start', $task, $logfd);
 908
 909             my $snapshot_count = $task->{snapshot_count} || 0;
 910
 911             $self->run_hook_script ('pre-stop', $task, $logfd);
 912
 913             if ($snapshot_count > 1) {
 914                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 915                 $vmstoptime = time ();
 916                 $plugin->suspend_vm ($task, $vmid);
 917                 $cleanup->{resume} = 1;
 918             }
 919
 920             $plugin->snapshot ($task, $vmid);
 921
 922             $self->run_hook_script ('pre-restart', $task, $logfd);
 923
 924             if ($snapshot_count > 1) {
 925                 debugmsg ('info', "resume vm", $logfd);
 926                 $cleanup->{resume} = 0;
 927                 $plugin->resume_vm ($task, $vmid);
 928                 my $delay = time () - $vmstoptime;
 929                 debugmsg ('info', "vm is online again after $delay seconds", $logfd);
 930             }
 931
 932         } else {
 933             die "internal error - unknown mode '$mode'\n";
 934         }
 935
 936         # assemble archive image
 937         $plugin->assemble ($task, $vmid);
 938
 939         # produce archive
 940
 941         if ($opts->{stdout}) {
 942             debugmsg ('info', "sending archive to stdout", $logfd);
 943             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 944             $self->run_hook_script ('backup-end', $task, $logfd);
 945             return;
 946         }
 947
 948         debugmsg ('info', "creating archive '$task->{tarfile}'", $logfd);
 949         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 950
 951         rename ($task->{tmptar}, $task->{tarfile}) ||
 952             die "unable to rename '$task->{tmptar}' to '$task->{tarfile}'\n";
 953
 954         # determine size
 955         $task->{size} = (-s $task->{tarfile}) || 0;
 956         my $cs = format_size ($task->{size});
 957         debugmsg ('info', "archive file size: $cs", $logfd);
 958
 959         # purge older backup
 960
 961         if ($maxfiles && $opts->{remove}) {
 962             my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{tarfile});
 963             $bklist = [ sort { $b->[1] <=> $a->[1] } @$bklist ];
 964
 965             while (scalar (@$bklist) >= $maxfiles) {
 966                 my $d = pop @$bklist;
 967                 debugmsg ('info', "delete old backup '$d->[0]'", $logfd);
 968                 unlink $d->[0];
 969                 my $logfn = $d->[0];
 970                 $logfn =~ s/\.(tgz|((tar|vma)(\.(gz|lzo))?))$/\.log/;
 971                 unlink $logfn;
 972             }
 973         }
 974
 975         $self->run_hook_script ('backup-end', $task, $logfd);
 976     };
 977     my $err = $@;
 978
 979     if ($plugin) {
 980         # clean-up
 981
 982         if ($cleanup->{unlock}) {
 983             eval { $plugin->unlock_vm ($vmid); };
 984             warn $@ if $@;
 985         }
 986
 987         if (defined($task->{mode})) {
 988             # only call cleanup when necessary (when prepare was executed)
 989             eval { $plugin->cleanup ($task, $vmid) };
 990             warn $@ if $@;
 991         }
 992
 993         eval { $plugin->set_logfd (undef); };
 994         warn $@ if $@;
 995
 996         if ($cleanup->{resume} || $cleanup->{restart}) {
 997             eval {
 998                 $self->run_hook_script ('pre-restart', $task, $logfd);
 999                 if ($cleanup->{resume}) {
1000                     debugmsg ('info', "resume vm", $logfd);
1001                     $plugin->resume_vm ($task, $vmid);
1002                 } else {
1003                     my $running = $plugin->vm_status($vmid);
1004                     if (!$running) {
1005                         debugmsg ('info', "restarting vm", $logfd);
1006                         $plugin->start_vm ($task, $vmid);
1007                     }
1008                 }
1009             };
1010             my $err = $@;
1011             if ($err) {
1012                 warn $err;
1013             } else {
1014                 my $delay = time () - $vmstoptime;
1015                 debugmsg ('info', "vm is online again after $delay seconds", $logfd);
1016             }
1017         }
1018     }
1019
1020     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
1021     warn $@ if $@;
1022
1023     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
1024     warn $@ if $@;
1025
1026     my $delay = $task->{backuptime} = time () - $vmstarttime;
1027
1028     if ($err) {
1029         $task->{state} = 'err';
1030         $task->{msg} = $err;
1031         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1032
1033         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1034
1035     } else {
1036         $task->{state} = 'ok';
1037         my $tstr = format_time ($delay);
1038         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1039     }
1040
1041     close ($logfd) if $logfd;
1042
1043     if ($task->{tmplog} && $task->{logfile}) {
1044         system ("cp '$task->{tmplog}' '$task->{logfile}'");
1045     }
1046
1047     eval { $self->run_hook_script ('log-end', $task); };
1048
1049     die $err if $err && $err =~ m/^interrupted by signal$/;
1050 }
1051
1052 sub exec_backup {
1053     my ($self, $rpcenv, $authuser) = @_;
1054
1055     my $opts = $self->{opts};
1056
1057     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1058     debugmsg ('info', "skip external VMs: " . join(', ', @{$self->{skiplist}}))
1059         if scalar(@{$self->{skiplist}});
1060
1061     my $tasklist = [];
1062
1063     if ($opts->{all}) {
1064         foreach my $plugin (@{$self->{plugins}}) {
1065             my $vmlist = $plugin->vmlist();
1066             foreach my $vmid (sort @$vmlist) {
1067                 next if grep { $_ eq  $vmid } @{$opts->{exclude}};
1068                 next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], 1);
1069                 push @$tasklist, { vmid => $vmid,  state => 'todo', plugin => $plugin };
1070             }
1071         }
1072     } else {
1073         foreach my $vmid (sort @{$opts->{vmids}}) {
1074             my $plugin;
1075             foreach my $pg (@{$self->{plugins}}) {
1076                 my $vmlist = $pg->vmlist();
1077                 if (grep { $_ eq  $vmid } @$vmlist) {
1078                     $plugin = $pg;
1079                     last;
1080                 }
1081             }
1082             $rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ]);
1083             push @$tasklist, { vmid => $vmid,  state => 'todo', plugin => $plugin };
1084         }
1085     }
1086
1087     my $starttime = time();
1088     my $errcount = 0;
1089     eval {
1090
1091         $self->run_hook_script ('job-start');
1092
1093         foreach my $task (@$tasklist) {
1094             $self->exec_backup_task ($task);
1095             $errcount += 1 if $task->{state} ne 'ok';
1096         }
1097
1098         $self->run_hook_script ('job-end');
1099     };
1100     my $err = $@;
1101
1102     $self->run_hook_script ('job-abort') if $err;
1103
1104     if ($err) {
1105         debugmsg ('err', "Backup job failed - $err", undef, 1);
1106     } else {
1107         if ($errcount) {
1108             debugmsg ('info', "Backup job finished with errors", undef, 1);
1109         } else {
1110             debugmsg ('info', "Backup job finished successfully", undef, 1);
1111         }
1112     }
1113
1114     my $totaltime = time() - $starttime;
1115
1116     eval { $self->sendmail ($tasklist, $totaltime); };
1117     debugmsg ('err', $@) if $@;
1118
1119     die $err if $err;
1120
1121     die "job errors\n" if $errcount;
1122
1123     unlink $pidfile;
1124 }
1125
1126
1127 sub option_exists {
1128     my $key = shift;
1129     return defined($confdesc->{$key});
1130 }
1131
1132 # add JSON properties for create and set function
1133 sub json_config_properties {
1134     my $prop = shift;
1135
1136     foreach my $opt (keys %$confdesc) {
1137         $prop->{$opt} = $confdesc->{$opt};
1138     }
1139
1140     return $prop;
1141 }
1142
1143 sub verify_vzdump_parameters {
1144     my ($param, $check_missing) = @_;
1145
1146     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1147         if $param->{all} && $param->{vmid};
1148
1149     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1150         if $param->{exclude} && $param->{vmid};
1151
1152     $param->{all} = 1 if defined($param->{exclude});
1153
1154     warn "option 'size' is deprecated and will be removed in a future " .
1155          "release, please update your script/configuration!\n"
1156         if defined($param->{size});
1157
1158     return if !$check_missing;
1159
1160     raise_param_exc({ vmid => "property is missing"})
1161         if !($param->{all} || $param->{stop}) && !$param->{vmid};
1162
1163 }
1164
1165 sub stop_running_backups {
1166     my($self) = @_;
1167
1168     my $upid = PVE::Tools::file_read_firstline($pidfile);
1169     return if !$upid;
1170
1171     my $task = PVE::Tools::upid_decode($upid);
1172
1173     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1174         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1175         kill(15, $task->{pid});
1176         # wait max 15 seconds to shut down (else, do nothing for now)
1177         my $i;
1178         for ($i = 15; $i > 0; $i--) {
1179             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1180             sleep (1);
1181         }
1182         die "stoping backup process $task->{pid} failed\n" if $i == 0;
1183     }
1184 }
1185
1186 sub command_line {
1187     my ($param) = @_;
1188
1189     my $cmd = "vzdump";
1190
1191     if ($param->{vmid}) {
1192         $cmd .= " " . join(' ', PVE::Tools::split_list($param->{vmid}));
1193     }
1194
1195     foreach my $p (keys %$param) {
1196         next if $p eq 'id' || $p eq 'vmid' || $p eq 'starttime' ||
1197                 $p eq 'dow' || $p eq 'stdout' || $p eq 'enabled';
1198         my $v = $param->{$p};
1199         my $pd = $confdesc->{$p} || die "no such vzdump option '$p'\n";
1200         if ($p eq 'exclude-path') {
1201             foreach my $path (split(/\0/, $v || '')) {
1202                 $cmd .= " --$p " . PVE::Tools::shellquote($path);
1203             }
1204         } else {
1205             $cmd .= " --$p " . PVE::Tools::shellquote($v) if defined($v) && $v ne '';
1206         }
1207     }
1208
1209     return $cmd;
1210 }
1211
1212 # bash completion helpers
1213 sub complete_backup_storage {
1214
1215     my $cfg = PVE::Storage::config();
1216     my $ids = $cfg->{ids};
1217
1218     my $nodename = PVE::INotify::nodename();
1219
1220     my $res = [];
1221     foreach my $sid (keys %$ids) {
1222         my $scfg = $ids->{$sid};
1223         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $nodename, 1);
1224         next if !$scfg->{content}->{backup};
1225         push @$res, $sid;
1226     }
1227
1228     return $res;
1229 }
1230
1231 1;