PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5 use Fcntl ':flock';
   6 use PVE::Exception qw(raise_param_exc);
   7 use PVE::SafeSyslog;
   8 use IO::File;
   9 use IO::Select;
  10 use IPC::Open3;
  11 use POSIX qw(strftime);
  12 use File::Path;
  13 use PVE::RPCEnvironment;
  14 use PVE::Storage;
  15 use PVE::Cluster qw(cfs_read_file);
  16 use Time::localtime;
  17 use Time::Local;
  18 use PVE::JSONSchema qw(get_standard_option);
  19 use PVE::HA::Env::PVE2;
  20 use PVE::HA::Config;
  21
  22 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  23
  24 my $lockfile = '/var/run/vzdump.lock';
  25
  26 my $pidfile = '/var/run/vzdump.pid';
  27
  28 my $logdir = '/var/log/vzdump';
  29
  30 my @plugins = qw();
  31
  32 my $confdesc = {
  33     vmid => {
  34         type => 'string', format => 'pve-vmid-list',
  35         description => "The ID of the VM you want to backup.",
  36         completion => \&PVE::Cluster::complete_local_vmid,
  37         optional => 1,
  38     },
  39     node => get_standard_option('pve-node', {
  40         description => "Only run if executed on this node.",
  41         completion => \&PVE::Cluster::get_nodelist,
  42         optional => 1,
  43     }),
  44     all => {
  45         type => 'boolean',
  46         description => "Backup all known VMs on this host.",
  47         optional => 1,
  48         default => 0,
  49     },
  50     stdexcludes => {
  51         type => 'boolean',
  52         description => "Exclude temorary files and logs.",
  53         optional => 1,
  54         default => 1,
  55     },
  56     compress => {
  57         type => 'string',
  58         description => "Compress dump file.",
  59         optional => 1,
  60         enum => ['0', '1', 'gzip', 'lzo'],
  61         default => '0',
  62     },
  63     pigz=> {
  64         type => "integer",
  65         description => "Uses pigz instead of gzip when N>0.".
  66             " N=1 uses half of cores, N>1 uses N as thread count.",
  67         optional => 1,
  68         default => 0,
  69     },
  70     quiet => {
  71         type => 'boolean',
  72         description => "Be quiet.",
  73         optional => 1,
  74         default => 0,
  75     },
  76     mode => {
  77         type => 'string',
  78         description => "Backup mode.",
  79         optional => 1,
  80         default => 'snapshot',
  81         enum => [ 'snapshot', 'suspend', 'stop' ],
  82     },
  83     exclude => {
  84         type => 'string', format => 'pve-vmid-list',
  85         description => "exclude specified VMs (assumes --all)",
  86         optional => 1,
  87     },
  88     'exclude-path' => {
  89         type => 'string', format => 'string-alist',
  90         description => "exclude certain files/directories (shell globs).",
  91         optional => 1,
  92     },
  93     mailto => {
  94         type => 'string', format => 'string-list',
  95         description => "",
  96         optional => 1,
  97     },
  98     mailnotification => {
  99         type => 'string',
 100         description => "Specify when to send an email",
 101         optional => 1,
 102         enum => [ 'always', 'failure' ],
 103         default => 'always',
 104     },
 105     tmpdir => {
 106         type => 'string',
 107         description => "Store temporary files to specified directory.",
 108         optional => 1,
 109     },
 110     dumpdir => {
 111         type => 'string',
 112         description => "Store resulting files to specified directory.",
 113         optional => 1,
 114     },
 115     script => {
 116         type => 'string',
 117         description => "Use specified hook script.",
 118         optional => 1,
 119     },
 120     storage => get_standard_option('pve-storage-id', {
 121         description => "Store resulting file to this storage.",
 122         completion => \&complete_backup_storage,
 123         optional => 1,
 124     }),
 125     stop => {
 126         type => 'boolean',
 127         description => "Stop runnig backup jobs on this host.",
 128         optional => 1,
 129         default => 0,
 130     },
 131     size => {
 132         type => 'integer',
 133         description => "LVM snapshot size in MB.",
 134         optional => 1,
 135         minimum => 500,
 136         default => 1024,
 137     },
 138     bwlimit => {
 139         type => 'integer',
 140         description => "Limit I/O bandwidth (KBytes per second).",
 141         optional => 1,
 142         minimum => 0,
 143         default => 0,
 144     },
 145     ionice => {
 146         type => 'integer',
 147         description => "Set CFQ ionice priority.",
 148         optional => 1,
 149         minimum => 0,
 150         maximum => 8,
 151         default => 7,
 152     },
 153     lockwait => {
 154         type => 'integer',
 155         description => "Maximal time to wait for the global lock (minutes).",
 156         optional => 1,
 157         minimum => 0,
 158         default => 3*60, # 3 hours
 159     },
 160     stopwait => {
 161         type => 'integer',
 162         description => "Maximal time to wait until a VM is stopped (minutes).",
 163         optional => 1,
 164         minimum => 0,
 165         default => 10, # 10 minutes
 166     },
 167     maxfiles => {
 168         type => 'integer',
 169         description => "Maximal number of backup files per VM.",
 170         optional => 1,
 171         minimum => 1,
 172         default => 1,
 173     },
 174     remove => {
 175         type => 'boolean',
 176         description => "Remove old backup files if there are more than 'maxfiles' backup files.",
 177         optional => 1,
 178         default => 1,
 179     },
 180 };
 181
 182 # Load available plugins
 183 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
 184 foreach my $plug (@pve_vzdump_classes) {
 185     my $filename = "/usr/share/perl5/$plug.pm";
 186     $filename =~ s!::!/!g;
 187     if (-f $filename) {
 188         eval { require $filename; };
 189         if (!$@) {
 190             $plug->import ();
 191             push @plugins, $plug;
 192         } else {
 193             warn $@;
 194         }
 195     }
 196 }
 197
 198 # helper functions
 199
 200 my $debugstattxt = {
 201     err =>  'ERROR:',
 202     info => 'INFO:',
 203     warn => 'WARN:',
 204 };
 205
 206 sub debugmsg {
 207     my ($mtype, $msg, $logfd, $syslog) = @_;
 208
 209     chomp $msg;
 210
 211     return if !$msg;
 212
 213     my $pre = $debugstattxt->{$mtype} || $debugstattxt->{'err'};
 214
 215     my $timestr = strftime ("%b %d %H:%M:%S", CORE::localtime);
 216
 217     syslog ($mtype eq 'info' ? 'info' : 'err', "$pre $msg") if $syslog;
 218
 219     foreach my $line (split (/\n/, $msg)) {
 220         print STDERR "$pre $line\n";
 221         print $logfd "$timestr $pre $line\n" if $logfd;
 222     }
 223 }
 224
 225 sub run_command {
 226     my ($logfd, $cmdstr, %param) = @_;
 227
 228     my $logfunc = sub {
 229         my $line = shift;
 230         debugmsg ('info', $line, $logfd);
 231     };
 232
 233     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
 234 }
 235
 236 sub storage_info {
 237     my $storage = shift;
 238
 239     my $cfg = PVE::Storage::config();
 240     my $scfg = PVE::Storage::storage_config($cfg, $storage);
 241     my $type = $scfg->{type};
 242
 243     die "can't use storage type '$type' for backup\n"
 244         if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'));
 245     die "can't use storage '$storage' for backups - wrong content type\n"
 246         if (!$scfg->{content}->{backup});
 247
 248     PVE::Storage::activate_storage($cfg, $storage);
 249
 250     return {
 251         dumpdir => PVE::Storage::get_backup_dir($cfg, $storage),
 252         maxfiles => $scfg->{maxfiles},
 253     };
 254 }
 255
 256 sub format_size {
 257     my $size = shift;
 258
 259     my $kb = $size / 1024;
 260
 261     if ($kb < 1024) {
 262         return int ($kb) . "KB";
 263     }
 264
 265     my $mb = $size / (1024*1024);
 266
 267     if ($mb < 1024) {
 268         return int ($mb) . "MB";
 269     } else {
 270         my $gb = $mb / 1024;
 271         return sprintf ("%.2fGB", $gb);
 272     }
 273 }
 274
 275 sub format_time {
 276     my $seconds = shift;
 277
 278     my $hours = int ($seconds/3600);
 279     $seconds = $seconds - $hours*3600;
 280     my $min = int ($seconds/60);
 281     $seconds = $seconds - $min*60;
 282
 283     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 284 }
 285
 286 sub encode8bit {
 287     my ($str) = @_;
 288
 289     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 290
 291     return $str;
 292 }
 293
 294 sub escape_html {
 295     my ($str) = @_;
 296
 297     $str =~ s/&/&amp;/g;
 298     $str =~ s/</&lt;/g;
 299     $str =~ s/>/&gt;/g;
 300
 301     return $str;
 302 }
 303
 304 sub check_bin {
 305     my ($bin)  = @_;
 306
 307     foreach my $p (split (/:/, $ENV{PATH})) {
 308         my $fn = "$p/$bin";
 309         if (-x $fn) {
 310             return $fn;
 311         }
 312     }
 313
 314     die "unable to find command '$bin'\n";
 315 }
 316
 317 sub check_vmids {
 318     my (@vmids) = @_;
 319
 320     my $res = [];
 321     foreach my $vmid (@vmids) {
 322         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 323         $vmid = int ($vmid); # remove leading zeros
 324         next if !$vmid;
 325         push @$res, $vmid;
 326     }
 327
 328     return $res;
 329 }
 330
 331
 332 sub read_vzdump_defaults {
 333
 334     my $fn = "/etc/vzdump.conf";
 335
 336     my $defaults = {
 337         map {
 338             my $default = $confdesc->{$_}->{default};
 339              defined($default) ? ($_ => $default) : ()
 340         } keys %$confdesc
 341     };
 342
 343     my $raw;
 344     eval { $raw = PVE::Tools::file_get_contents($fn); };
 345     return $defaults if $@;
 346
 347     my $conf_schema = { type => 'object', properties => $confdesc, };
 348     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 349     if (my $excludes = $res->{'exclude-path'}) {
 350         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 351     }
 352
 353     foreach my $key (keys %$defaults) {
 354         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 355     }
 356
 357     return $res;
 358 }
 359
 360 sub sendmail {
 361     my ($self, $tasklist, $totaltime, $err) = @_;
 362
 363     my $opts = $self->{opts};
 364
 365     my $mailto = $opts->{mailto};
 366
 367     return if !($mailto && scalar(@$mailto));
 368
 369     my $cmdline = $self->{cmdline};
 370
 371     my $ecount = 0;
 372     foreach my $task (@$tasklist) {
 373         $ecount++ if $task->{state} ne 'ok';
 374         chomp $task->{msg} if $task->{msg};
 375         $task->{backuptime} = 0 if !$task->{backuptime};
 376         $task->{size} = 0 if !$task->{size};
 377         $task->{tarfile} = 'unknown' if !$task->{tarfile};
 378         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 379
 380         if ($task->{state} eq 'todo') {
 381             $task->{msg} = 'aborted';
 382         }
 383     }
 384
 385     my $notify = $opts->{mailnotification} || 'always';
 386     return if (!$ecount && !$err && ($notify eq 'failure'));
 387
 388     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 389     $stat .= ": $err" if $err;
 390
 391     my $hostname = `hostname -f` || PVE::INotify::nodename();
 392     chomp $hostname;
 393
 394     # text part
 395     my $text = sprintf ("%-10s %-6s %10s %10s  %s\n", qw(VMID STATUS TIME SIZE FILENAME));
 396     foreach my $task (@$tasklist) {
 397         my $vmid = $task->{vmid};
 398         if  ($task->{state} eq 'ok') {
 399
 400             $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", $vmid,
 401                                 $task->{state},
 402                                 format_time($task->{backuptime}),
 403                                 format_size ($task->{size}),
 404                                 $task->{tarfile});
 405         } else {
 406             $text .= sprintf ("%-10s %-6s %10s %8.2fMB  %s\n", $vmid,
 407                                 $task->{state},
 408                                 format_time($task->{backuptime}),
 409                                 0, '-');
 410         }
 411     }
 412
 413     $text .= "Detailed backup logs:\n\n";
 414     $text .= "$cmdline\n\n";
 415
 416     foreach my $task (@$tasklist) {
 417         my $vmid = $task->{vmid};
 418         my $log = $task->{tmplog};
 419         if (!$log) {
 420             $text .= "$vmid: no log available\n\n";
 421             next;
 422         }
 423         open (TMP, "$log");
 424         while (my $line = <TMP>) { $text .= encode8bit ("$vmid: $line"); }
 425         close (TMP);
 426         $text .= "\n";
 427     }
 428
 429     # html part
 430     my $html = "<html><body>\n";
 431     $html .= "<table border=1 cellpadding=3>\n";
 432     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 433
 434     my $ssize = 0;
 435
 436     foreach my $task (@$tasklist) {
 437         my $vmid = $task->{vmid};
 438         my $name = $task->{hostname};
 439
 440         if  ($task->{state} eq 'ok') {
 441
 442             $ssize += $task->{size};
 443
 444             $html .= sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 445                                 $vmid, $name,
 446                                 format_time($task->{backuptime}),
 447                                 format_size ($task->{size}),
 448                                 escape_html ($task->{tarfile}));
 449         } else {
 450             $html .= sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 451                                 $vmid, $name, format_time($task->{backuptime}),
 452                                 escape_html ($task->{msg}));
 453         }
 454     }
 455
 456     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 457  format_time ($totaltime), format_size ($ssize));
 458
 459     $html .= "</table><br><br>\n";
 460     $html .= "Detailed backup logs:<br /><br />\n";
 461     $html .= "<pre>\n";
 462     $html .= escape_html($cmdline) . "\n\n";
 463
 464     foreach my $task (@$tasklist) {
 465         my $vmid = $task->{vmid};
 466         my $log = $task->{tmplog};
 467         if (!$log) {
 468             $html .= "$vmid: no log available\n\n";
 469             next;
 470         }
 471         open (TMP, "$log");
 472         while (my $line = <TMP>) {
 473             if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 474                 $html .= encode8bit ("$vmid: <font color=red>".
 475                                        escape_html ($line) . "</font>");
 476             } else {
 477                 $html .= encode8bit ("$vmid: " . escape_html ($line));
 478             }
 479         }
 480         close (TMP);
 481         $html .= "\n";
 482     }
 483     $html .= "</pre></body></html>\n";
 484     # end html part
 485
 486     my $subject = "vzdump backup status ($hostname) : $stat";
 487
 488     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 489     my $mailfrom = $dcconf->{email_from} || "root";
 490
 491     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 492 };
 493
 494 sub new {
 495     my ($class, $cmdline, $opts, $skiplist) = @_;
 496
 497     mkpath $logdir;
 498
 499     check_bin ('cp');
 500     check_bin ('df');
 501     check_bin ('sendmail');
 502     check_bin ('rsync');
 503     check_bin ('tar');
 504     check_bin ('mount');
 505     check_bin ('umount');
 506     check_bin ('cstream');
 507     check_bin ('ionice');
 508
 509     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 510         check_bin ('lvcreate');
 511         check_bin ('lvs');
 512         check_bin ('lvremove');
 513     }
 514
 515     my $defaults = read_vzdump_defaults();
 516
 517     my $maxfiles = $opts->{maxfiles}; # save here, because we overwrite with default
 518
 519     $opts->{remove} = 1 if !defined($opts->{remove});
 520
 521     foreach my $k (keys %$defaults) {
 522         next if $k eq 'exclude-path'; # dealt with separately
 523         if ($k eq 'dumpdir' || $k eq 'storage') {
 524             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 525                 !defined ($opts->{storage});
 526         } else {
 527             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 528         }
 529     }
 530
 531     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 532     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 533
 534     $skiplist = [] if !$skiplist;
 535     my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist };
 536
 537     my $findexcl = $self->{findexcl} = [];
 538     if ($defaults->{'exclude-path'}) {
 539         push @$findexcl, @{$defaults->{'exclude-path'}};
 540     }
 541
 542     if ($opts->{'exclude-path'}) {
 543         push @$findexcl, @{$opts->{'exclude-path'}};
 544     }
 545
 546     if ($opts->{stdexcludes}) {
 547         push @$findexcl, '/var/log/?*',
 548                          '/tmp/?*',
 549                          '/var/tmp/?*',
 550                          '/var/run/?*.pid';
 551     }
 552
 553     foreach my $p (@plugins) {
 554
 555         my $pd = $p->new ($self);
 556
 557         push @{$self->{plugins}}, $pd;
 558     }
 559
 560     if (!$opts->{dumpdir} && !$opts->{storage}) {
 561         $opts->{storage} = 'local';
 562     }
 563
 564     if ($opts->{storage}) {
 565         my $info = storage_info ($opts->{storage});
 566         $opts->{dumpdir} = $info->{dumpdir};
 567         $maxfiles = $info->{maxfiles} if !defined($maxfiles) && defined($info->{maxfiles});
 568     } elsif ($opts->{dumpdir}) {
 569         die "dumpdir '$opts->{dumpdir}' does not exist\n"
 570             if ! -d $opts->{dumpdir};
 571     } else {
 572         die "internal error";
 573     }
 574
 575     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 576         die "tmpdir '$opts->{tmpdir}' does not exist\n";
 577     }
 578
 579     $opts->{maxfiles} = $maxfiles if defined($maxfiles);
 580
 581     return $self;
 582
 583 }
 584
 585 sub get_lvm_mapping {
 586
 587     my $devmapper;
 588
 589     my $cmd = ['lvs', '--units', 'm', '--separator', ':', '--noheadings',
 590                '-o', 'vg_name,lv_name,lv_size' ];
 591
 592     my $parser = sub {
 593         my $line = shift;
 594         if ($line =~ m|^\s*(\S+):(\S+):(\d+(\.\d+))[Mm]$|) {
 595             my $vg = $1;
 596             my $lv = $2;
 597             $devmapper->{"/dev/$vg/$lv"} = [$vg, $lv];
 598             my $qlv = $lv;
 599             $qlv =~ s/-/--/g;
 600             my $qvg = $vg;
 601             $qvg =~ s/-/--/g;
 602             $devmapper->{"/dev/mapper/$qvg-$qlv"} = [$vg, $lv];
 603         }
 604     };
 605
 606     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 607     warn $@ if $@;
 608
 609     return $devmapper;
 610 }
 611
 612 sub get_mount_info {
 613     my ($dir) = @_;
 614
 615     # Note: df 'available' can be negative, and percentage set to '-'
 616
 617     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 618
 619     my $res;
 620
 621     my $parser = sub {
 622         my $line = shift;
 623         if (my ($fsid, $fstype, undef, $mp) = $line =~
 624             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 625             $res = {
 626                 device => $fsid,
 627                 fstype => $fstype,
 628                 mountpoint => $mp,
 629             };
 630         }
 631     };
 632
 633     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 634     warn $@ if $@;
 635
 636     return $res;
 637 }
 638
 639 sub get_lvm_device {
 640     my ($dir, $mapping) = @_;
 641
 642     my $info = get_mount_info($dir);
 643
 644     return undef if !$info;
 645
 646     my $dev = $info->{device};
 647
 648     my ($vg, $lv);
 649
 650     ($vg, $lv) = @{$mapping->{$dev}} if defined $mapping->{$dev};
 651
 652     return wantarray ? ($dev, $info->{mountpoint}, $vg, $lv, $info->{fstype}) : $dev;
 653 }
 654
 655 sub getlock {
 656     my ($self, $upid) = @_;
 657
 658     my $fh;
 659
 660     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 661
 662     die "missimg UPID" if !$upid; # should not happen
 663
 664     if (!open (SERVER_FLCK, ">>$lockfile")) {
 665         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 666         die "can't open lock on file '$lockfile' - $!";
 667     }
 668
 669     if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 670
 671         if (!$maxwait) {
 672             debugmsg ('err', "can't aquire lock '$lockfile' (wait = 0)", undef, 1);
 673             die "can't aquire lock '$lockfile' (wait = 0)";
 674         }
 675
 676         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 677
 678         eval {
 679             alarm ($maxwait * 60);
 680
 681             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 682
 683             if (!flock (SERVER_FLCK, LOCK_EX)) {
 684                 my $err = $!;
 685                 close (SERVER_FLCK);
 686                 alarm (0);
 687                 die "$err\n";
 688             }
 689             alarm (0);
 690         };
 691         alarm (0);
 692
 693         my $err = $@;
 694
 695         if ($err) {
 696             debugmsg ('err', "can't aquire lock '$lockfile' - $err", undef, 1);
 697             die "can't aquire lock '$lockfile' - $err";
 698         }
 699
 700         debugmsg('info', "got global lock", undef, 1);
 701     }
 702
 703     PVE::Tools::file_set_contents($pidfile, $upid);
 704 }
 705
 706 sub run_hook_script {
 707     my ($self, $phase, $task, $logfd) = @_;
 708
 709     my $opts = $self->{opts};
 710
 711     my $script = $opts->{script};
 712
 713     return if !$script;
 714
 715     my $cmd = "$script $phase";
 716
 717     $cmd .= " $task->{mode} $task->{vmid}" if ($task);
 718
 719     local %ENV;
 720
 721     # set immutable opts directly (so they are available in all phases)
 722     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 723     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 724
 725     foreach my $ek (qw(vmtype hostname tarfile logfile)) {
 726         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 727     }
 728
 729     run_command ($logfd, $cmd);
 730 }
 731
 732 sub compressor_info {
 733     my ($opts) = @_;
 734     my $opt_compress = $opts->{compress};
 735
 736     if (!$opt_compress || $opt_compress eq '0') {
 737         return undef;
 738     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 739         return ('lzop', 'lzo');
 740     } elsif ($opt_compress eq 'gzip') {
 741         if ($opts->{pigz} > 0) {
 742             # As default use int((#cores + 1)/2), we need #cores+1 for the case that #cores = 1
 743             my $cores = POSIX::sysconf(84);
 744             my $pigz_threads = ($opts->{pigz} > 1) ? $opts->{pigz} : int(($cores + 1)/2);
 745             return ("pigz -p ${pigz_threads}", 'gz');
 746         } else {
 747             return ('gzip', 'gz');
 748         }
 749     } else {
 750         die "internal error - unknown compression option '$opt_compress'";
 751     }
 752 }
 753
 754 sub get_backup_file_list {
 755     my ($dir, $bkname, $exclude_fn) = @_;
 756
 757     my $bklist = [];
 758     foreach my $fn (<$dir/${bkname}-*>) {
 759         next if $exclude_fn && $fn eq $exclude_fn;
 760         if ($fn =~ m!/(${bkname}-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?)))$!) {
 761             $fn = "$dir/$1"; # untaint
 762             my $t = timelocal ($7, $6, $5, $4, $3 - 1, $2 - 1900);
 763             push @$bklist, [$fn, $t];
 764         }
 765     }
 766
 767     return $bklist;
 768 }
 769
 770 sub exec_backup_task {
 771     my ($self, $task) = @_;
 772
 773     my $opts = $self->{opts};
 774
 775     my $vmid = $task->{vmid};
 776     my $plugin = $task->{plugin};
 777
 778     my $vmstarttime = time ();
 779
 780     my $logfd;
 781
 782     my $cleanup = {};
 783
 784     my $vmstoptime = 0;
 785
 786     eval {
 787         die "unable to find VM '$vmid'\n" if !$plugin;
 788
 789         # for now we deny backups of a running ha managed service in *stop* mode
 790         # as it interferes with the HA stack (enabled services should not stop).
 791         if ($opts->{mode} eq 'stop' &&
 792             PVE::HA::Config::vm_is_ha_managed($vmid, 'enabled'))
 793         {
 794             die "Cannot execute a backup with stop mode on a HA managed and".
 795                 " enabled Service. Use snapshot mode or disable the Service.\n";
 796         }
 797
 798         my $vmtype = $plugin->type();
 799
 800         my $tmplog = "$logdir/$vmtype-$vmid.log";
 801
 802         my $lt = localtime();
 803
 804         my $bkname = "vzdump-$vmtype-$vmid";
 805         my $basename = sprintf "${bkname}-%04d_%02d_%02d-%02d_%02d_%02d",
 806         $lt->year + 1900, $lt->mon + 1, $lt->mday,
 807         $lt->hour, $lt->min, $lt->sec;
 808
 809         my $maxfiles = $opts->{maxfiles};
 810
 811         if ($maxfiles && !$opts->{remove}) {
 812             my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 813             die "only $maxfiles backup(s) allowed - please consider to remove old backup files.\n"
 814                 if scalar(@$bklist) >= $maxfiles;
 815         }
 816
 817         my $logfile = $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 818
 819         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 820         my ($comp, $comp_ext) = compressor_info($opts);
 821         if ($comp && $comp_ext) {
 822             $ext .= ".${comp_ext}";
 823         }
 824
 825         if ($opts->{stdout}) {
 826             $task->{tarfile} = '-';
 827         } else {
 828             my $tarfile = $task->{tarfile} = "$opts->{dumpdir}/$basename$ext";
 829             $task->{tmptar} = $task->{tarfile};
 830             $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 831             unlink $task->{tmptar};
 832         }
 833
 834         $task->{vmtype} = $vmtype;
 835
 836         if ($opts->{tmpdir}) {
 837             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp$$";
 838         } else {
 839             # dumpdir is posix? then use it as temporary dir
 840             my $info = get_mount_info($opts->{dumpdir});
 841             if ($vmtype eq 'qemu' ||
 842                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 843                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 844             } else {
 845                 $task->{tmpdir} = "/var/tmp/vzdumptmp$$";
 846                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 847                           "using $task->{tmpdir} for temporary files", $logfd);
 848             }
 849         }
 850
 851         rmtree $task->{tmpdir};
 852         mkdir $task->{tmpdir};
 853         -d $task->{tmpdir} ||
 854             die "unable to create temporary directory '$task->{tmpdir}'";
 855
 856         $logfd = IO::File->new (">$tmplog") ||
 857             die "unable to create log file '$tmplog'";
 858
 859         $task->{dumpdir} = $opts->{dumpdir};
 860         $task->{storeid} = $opts->{storage};
 861         $task->{tmplog} = $tmplog;
 862
 863         unlink $logfile;
 864
 865         debugmsg ('info',  "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 866
 867         $plugin->set_logfd ($logfd);
 868
 869         # test is VM is running
 870         my ($running, $status_text) = $plugin->vm_status ($vmid);
 871
 872         debugmsg ('info', "status = ${status_text}", $logfd);
 873
 874         # lock VM (prevent config changes)
 875         $plugin->lock_vm ($vmid);
 876
 877         $cleanup->{unlock} = 1;
 878
 879         # prepare
 880
 881         my $mode = $running ? $opts->{mode} : 'stop';
 882
 883         if ($mode eq 'snapshot') {
 884             my %saved_task = %$task;
 885             eval { $plugin->prepare ($task, $vmid, $mode); };
 886             if (my $err = $@) {
 887                 die $err if $err !~ m/^mode failure/;
 888                 debugmsg ('info',  $err, $logfd);
 889                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 890                 $mode = 'suspend'; # so prepare is called again below
 891                 %$task = %saved_task;
 892             }
 893         }
 894
 895         $task->{mode} = $mode;
 896
 897         debugmsg ('info', "backup mode: $mode", $logfd);
 898
 899         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
 900             if $opts->{bwlimit};
 901
 902         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 903
 904         if ($mode eq 'stop') {
 905
 906             $plugin->prepare ($task, $vmid, $mode);
 907
 908             $self->run_hook_script ('backup-start', $task, $logfd);
 909
 910             if ($running) {
 911                 debugmsg ('info', "stopping vm", $logfd);
 912                 $vmstoptime = time ();
 913                 $self->run_hook_script ('pre-stop', $task, $logfd);
 914                 $plugin->stop_vm ($task, $vmid);
 915                 $cleanup->{restart} = 1;
 916             }
 917
 918
 919         } elsif ($mode eq 'suspend') {
 920
 921             $plugin->prepare ($task, $vmid, $mode);
 922
 923             $self->run_hook_script ('backup-start', $task, $logfd);
 924
 925             if ($vmtype eq 'lxc') {
 926                 # pre-suspend rsync
 927                 $plugin->copy_data_phase1($task, $vmid);
 928             }
 929
 930             debugmsg ('info', "suspend vm", $logfd);
 931             $vmstoptime = time ();
 932             $self->run_hook_script ('pre-stop', $task, $logfd);
 933             $plugin->suspend_vm ($task, $vmid);
 934             $cleanup->{resume} = 1;
 935
 936             if ($vmtype eq 'lxc') {
 937                 # post-suspend rsync
 938                 $plugin->copy_data_phase2($task, $vmid);
 939
 940                 debugmsg ('info', "resume vm", $logfd);
 941                 $cleanup->{resume} = 0;
 942                 $self->run_hook_script('pre-restart', $task, $logfd);
 943                 $plugin->resume_vm($task, $vmid);
 944                 my $delay = time () - $vmstoptime;
 945                 debugmsg('info', "vm is online again after $delay seconds", $logfd);
 946             }
 947
 948         } elsif ($mode eq 'snapshot') {
 949
 950             $self->run_hook_script ('backup-start', $task, $logfd);
 951
 952             my $snapshot_count = $task->{snapshot_count} || 0;
 953
 954             $self->run_hook_script ('pre-stop', $task, $logfd);
 955
 956             if ($snapshot_count > 1) {
 957                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 958                 $vmstoptime = time ();
 959                 $plugin->suspend_vm ($task, $vmid);
 960                 $cleanup->{resume} = 1;
 961             }
 962
 963             $plugin->snapshot ($task, $vmid);
 964
 965             $self->run_hook_script ('pre-restart', $task, $logfd);
 966
 967             if ($snapshot_count > 1) {
 968                 debugmsg ('info', "resume vm", $logfd);
 969                 $cleanup->{resume} = 0;
 970                 $plugin->resume_vm ($task, $vmid);
 971                 my $delay = time () - $vmstoptime;
 972                 debugmsg ('info', "vm is online again after $delay seconds", $logfd);
 973             }
 974
 975         } else {
 976             die "internal error - unknown mode '$mode'\n";
 977         }
 978
 979         # assemble archive image
 980         $plugin->assemble ($task, $vmid);
 981
 982         # produce archive
 983
 984         if ($opts->{stdout}) {
 985             debugmsg ('info', "sending archive to stdout", $logfd);
 986             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 987             $self->run_hook_script ('backup-end', $task, $logfd);
 988             return;
 989         }
 990
 991         debugmsg ('info', "creating archive '$task->{tarfile}'", $logfd);
 992         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 993
 994         rename ($task->{tmptar}, $task->{tarfile}) ||
 995             die "unable to rename '$task->{tmptar}' to '$task->{tarfile}'\n";
 996
 997         # determine size
 998         $task->{size} = (-s $task->{tarfile}) || 0;
 999         my $cs = format_size ($task->{size});
1000         debugmsg ('info', "archive file size: $cs", $logfd);
1001
1002         # purge older backup
1003
1004         if ($maxfiles && $opts->{remove}) {
1005             my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{tarfile});
1006             $bklist = [ sort { $b->[1] <=> $a->[1] } @$bklist ];
1007
1008             while (scalar (@$bklist) >= $maxfiles) {
1009                 my $d = pop @$bklist;
1010                 debugmsg ('info', "delete old backup '$d->[0]'", $logfd);
1011                 unlink $d->[0];
1012                 my $logfn = $d->[0];
1013                 $logfn =~ s/\.(tgz|((tar|vma)(\.(gz|lzo))?))$/\.log/;
1014                 unlink $logfn;
1015             }
1016         }
1017
1018         $self->run_hook_script ('backup-end', $task, $logfd);
1019     };
1020     my $err = $@;
1021
1022     if ($plugin) {
1023         # clean-up
1024
1025         if ($cleanup->{unlock}) {
1026             eval { $plugin->unlock_vm ($vmid); };
1027             warn $@ if $@;
1028         }
1029
1030         if (defined($task->{mode})) {
1031             # only call cleanup when necessary (when prepare was executed)
1032             eval { $plugin->cleanup ($task, $vmid) };
1033             warn $@ if $@;
1034         }
1035
1036         eval { $plugin->set_logfd (undef); };
1037         warn $@ if $@;
1038
1039         if ($cleanup->{resume} || $cleanup->{restart}) {
1040             eval {
1041                 $self->run_hook_script ('pre-restart', $task, $logfd);
1042                 if ($cleanup->{resume}) {
1043                     debugmsg ('info', "resume vm", $logfd);
1044                     $plugin->resume_vm ($task, $vmid);
1045                 } else {
1046                     my $running = $plugin->vm_status($vmid);
1047                     if (!$running) {
1048                         debugmsg ('info', "restarting vm", $logfd);
1049                         $plugin->start_vm ($task, $vmid);
1050                     }
1051                 }
1052             };
1053             my $err = $@;
1054             if ($err) {
1055                 warn $err;
1056             } else {
1057                 my $delay = time () - $vmstoptime;
1058                 debugmsg ('info', "vm is online again after $delay seconds", $logfd);
1059             }
1060         }
1061     }
1062
1063     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
1064     warn $@ if $@;
1065
1066     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
1067     warn $@ if $@;
1068
1069     my $delay = $task->{backuptime} = time () - $vmstarttime;
1070
1071     if ($err) {
1072         $task->{state} = 'err';
1073         $task->{msg} = $err;
1074         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1075
1076         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1077
1078     } else {
1079         $task->{state} = 'ok';
1080         my $tstr = format_time ($delay);
1081         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1082     }
1083
1084     close ($logfd) if $logfd;
1085
1086     if ($task->{tmplog} && $task->{logfile}) {
1087         system ("cp '$task->{tmplog}' '$task->{logfile}'");
1088     }
1089
1090     eval { $self->run_hook_script ('log-end', $task); };
1091
1092     die $err if $err && $err =~ m/^interrupted by signal$/;
1093 }
1094
1095 sub exec_backup {
1096     my ($self, $rpcenv, $authuser) = @_;
1097
1098     my $opts = $self->{opts};
1099
1100     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1101     debugmsg ('info', "skip external VMs: " . join(', ', @{$self->{skiplist}}))
1102         if scalar(@{$self->{skiplist}});
1103
1104     my $tasklist = [];
1105
1106     if ($opts->{all}) {
1107         foreach my $plugin (@{$self->{plugins}}) {
1108             my $vmlist = $plugin->vmlist();
1109             foreach my $vmid (sort @$vmlist) {
1110                 next if grep { $_ eq  $vmid } @{$opts->{exclude}};
1111                 next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], 1);
1112                 push @$tasklist, { vmid => $vmid,  state => 'todo', plugin => $plugin };
1113             }
1114         }
1115     } else {
1116         foreach my $vmid (sort @{$opts->{vmids}}) {
1117             my $plugin;
1118             foreach my $pg (@{$self->{plugins}}) {
1119                 my $vmlist = $pg->vmlist();
1120                 if (grep { $_ eq  $vmid } @$vmlist) {
1121                     $plugin = $pg;
1122                     last;
1123                 }
1124             }
1125             $rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ]);
1126             push @$tasklist, { vmid => $vmid,  state => 'todo', plugin => $plugin };
1127         }
1128     }
1129
1130     my $starttime = time();
1131     my $errcount = 0;
1132     eval {
1133
1134         $self->run_hook_script ('job-start');
1135
1136         foreach my $task (@$tasklist) {
1137             $self->exec_backup_task ($task);
1138             $errcount += 1 if $task->{state} ne 'ok';
1139         }
1140
1141         $self->run_hook_script ('job-end');
1142     };
1143     my $err = $@;
1144
1145     $self->run_hook_script ('job-abort') if $err;
1146
1147     if ($err) {
1148         debugmsg ('err', "Backup job failed - $err", undef, 1);
1149     } else {
1150         if ($errcount) {
1151             debugmsg ('info', "Backup job finished with errors", undef, 1);
1152         } else {
1153             debugmsg ('info', "Backup job finished successfully", undef, 1);
1154         }
1155     }
1156
1157     my $totaltime = time() - $starttime;
1158
1159     eval { $self->sendmail ($tasklist, $totaltime); };
1160     debugmsg ('err', $@) if $@;
1161
1162     die $err if $err;
1163
1164     die "job errors\n" if $errcount;
1165
1166     unlink $pidfile;
1167 }
1168
1169
1170 sub option_exists {
1171     my $key = shift;
1172     return defined($confdesc->{$key});
1173 }
1174
1175 # add JSON properties for create and set function
1176 sub json_config_properties {
1177     my $prop = shift;
1178
1179     foreach my $opt (keys %$confdesc) {
1180         $prop->{$opt} = $confdesc->{$opt};
1181     }
1182
1183     return $prop;
1184 }
1185
1186 sub verify_vzdump_parameters {
1187     my ($param, $check_missing) = @_;
1188
1189     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1190         if $param->{all} && $param->{vmid};
1191
1192     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1193         if $param->{exclude} && $param->{vmid};
1194
1195     $param->{all} = 1 if defined($param->{exclude});
1196
1197     return if !$check_missing;
1198
1199     raise_param_exc({ vmid => "property is missing"})
1200         if !($param->{all} || $param->{stop}) && !$param->{vmid};
1201
1202 }
1203
1204 sub stop_running_backups {
1205     my($self) = @_;
1206
1207     my $upid = PVE::Tools::file_read_firstline($pidfile);
1208     return if !$upid;
1209
1210     my $task = PVE::Tools::upid_decode($upid);
1211
1212     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1213         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1214         kill(15, $task->{pid});
1215         # wait max 15 seconds to shut down (else, do nothing for now)
1216         my $i;
1217         for ($i = 15; $i > 0; $i--) {
1218             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1219             sleep (1);
1220         }
1221         die "stoping backup process $task->{pid} failed\n" if $i == 0;
1222     }
1223 }
1224
1225 sub command_line {
1226     my ($param) = @_;
1227
1228     my $cmd = "vzdump";
1229
1230     if ($param->{vmid}) {
1231         $cmd .= " " . join(' ', PVE::Tools::split_list($param->{vmid}));
1232     }
1233
1234     foreach my $p (keys %$param) {
1235         next if $p eq 'id' || $p eq 'vmid' || $p eq 'starttime' ||
1236                 $p eq 'dow' || $p eq 'stdout' || $p eq 'enabled';
1237         my $v = $param->{$p};
1238         my $pd = $confdesc->{$p} || die "no such vzdump option '$p'\n";
1239         if ($p eq 'exclude-path') {
1240             foreach my $path (split(/\0/, $v || '')) {
1241                 $cmd .= " --$p " . PVE::Tools::shellquote($path);
1242             }
1243         } else {
1244             $cmd .= " --$p " . PVE::Tools::shellquote($v) if defined($v) && $v ne '';
1245         }
1246     }
1247
1248     return $cmd;
1249 }
1250
1251 # bash completion helpers
1252 sub complete_backup_storage {
1253
1254     my $cfg = PVE::Storage::config();
1255     my $ids = $cfg->{ids};
1256
1257     my $nodename = PVE::INotify::nodename();
1258
1259     my $res = [];
1260     foreach my $sid (keys %$ids) {
1261         my $scfg = $ids->{$sid};
1262         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $nodename, 1);
1263         next if !$scfg->{content}->{backup};
1264         push @$res, $sid;
1265     }
1266
1267     return $res;
1268 }
1269
1270 1;