PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5 use Fcntl ':flock';
   6 use PVE::Exception qw(raise_param_exc);
   7 use PVE::SafeSyslog;
   8 use IO::File;
   9 use IO::Select;
  10 use IPC::Open3;
  11 use POSIX qw(strftime);
  12 use File::Path;
  13 use PVE::RPCEnvironment;
  14 use PVE::Storage;
  15 use PVE::Cluster qw(cfs_read_file);
  16 use Time::localtime;
  17 use Time::Local;
  18 use PVE::JSONSchema qw(get_standard_option);
  19 use PVE::HA::Env::PVE2;
  20 use PVE::HA::Config;
  21
  22 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  23
  24 my $lockfile = '/var/run/vzdump.lock';
  25
  26 my $pidfile = '/var/run/vzdump.pid';
  27
  28 my $logdir = '/var/log/vzdump';
  29
  30 my @plugins = qw();
  31
  32 my $confdesc = {
  33     vmid => {
  34         type => 'string', format => 'pve-vmid-list',
  35         description => "The ID of the guest system you want to backup.",
  36         completion => \&PVE::Cluster::complete_local_vmid,
  37         optional => 1,
  38     },
  39     node => get_standard_option('pve-node', {
  40         description => "Only run if executed on this node.",
  41         completion => \&PVE::Cluster::get_nodelist,
  42         optional => 1,
  43     }),
  44     all => {
  45         type => 'boolean',
  46         description => "Backup all known guest systems on this host.",
  47         optional => 1,
  48         default => 0,
  49     },
  50     stdexcludes => {
  51         type => 'boolean',
  52         description => "Exclude temporary files and logs.",
  53         optional => 1,
  54         default => 1,
  55     },
  56     compress => {
  57         type => 'string',
  58         description => "Compress dump file.",
  59         optional => 1,
  60         enum => ['0', '1', 'gzip', 'lzo'],
  61         default => '0',
  62     },
  63     pigz=> {
  64         type => "integer",
  65         description => "Use pigz instead of gzip when N>0.".
  66             " N=1 uses half of cores, N>1 uses N as thread count.",
  67         optional => 1,
  68         default => 0,
  69     },
  70     quiet => {
  71         type => 'boolean',
  72         description => "Be quiet.",
  73         optional => 1,
  74         default => 0,
  75     },
  76     mode => {
  77         type => 'string',
  78         description => "Backup mode.",
  79         optional => 1,
  80         default => 'snapshot',
  81         enum => [ 'snapshot', 'suspend', 'stop' ],
  82     },
  83     exclude => {
  84         type => 'string', format => 'pve-vmid-list',
  85         description => "Exclude specified guest systems (assumes --all)",
  86         optional => 1,
  87     },
  88     'exclude-path' => {
  89         type => 'string', format => 'string-alist',
  90         description => "Exclude certain files/directories (shell globs).",
  91         optional => 1,
  92     },
  93     mailto => {
  94         type => 'string', format => 'string-list',
  95         description => "Comma-separated list of email addresses that should" .
  96             " receive email notifications.",
  97         optional => 1,
  98     },
  99     mailnotification => {
 100         type => 'string',
 101         description => "Specify when to send an email",
 102         optional => 1,
 103         enum => [ 'always', 'failure' ],
 104         default => 'always',
 105     },
 106     tmpdir => {
 107         type => 'string',
 108         description => "Store temporary files to specified directory.",
 109         optional => 1,
 110     },
 111     dumpdir => {
 112         type => 'string',
 113         description => "Store resulting files to specified directory.",
 114         optional => 1,
 115     },
 116     script => {
 117         type => 'string',
 118         description => "Use specified hook script.",
 119         optional => 1,
 120     },
 121     storage => get_standard_option('pve-storage-id', {
 122         description => "Store resulting file to this storage.",
 123         completion => \&complete_backup_storage,
 124         optional => 1,
 125     }),
 126     stop => {
 127         type => 'boolean',
 128         description => "Stop runnig backup jobs on this host.",
 129         optional => 1,
 130         default => 0,
 131     },
 132     size => {
 133         type => 'integer',
 134         description => "Unused, will be removed in a future release.",
 135         optional => 1,
 136         minimum => 500,
 137         default => 1024,
 138     },
 139     bwlimit => {
 140         type => 'integer',
 141         description => "Limit I/O bandwidth (KBytes per second).",
 142         optional => 1,
 143         minimum => 0,
 144         default => 0,
 145     },
 146     ionice => {
 147         type => 'integer',
 148         description => "Set CFQ ionice priority.",
 149         optional => 1,
 150         minimum => 0,
 151         maximum => 8,
 152         default => 7,
 153     },
 154     lockwait => {
 155         type => 'integer',
 156         description => "Maximal time to wait for the global lock (minutes).",
 157         optional => 1,
 158         minimum => 0,
 159         default => 3*60, # 3 hours
 160     },
 161     stopwait => {
 162         type => 'integer',
 163         description => "Maximal time to wait until a guest system is stopped (minutes).",
 164         optional => 1,
 165         minimum => 0,
 166         default => 10, # 10 minutes
 167     },
 168     maxfiles => {
 169         type => 'integer',
 170         description => "Maximal number of backup files per guest system.",
 171         optional => 1,
 172         minimum => 1,
 173         default => 1,
 174     },
 175     remove => {
 176         type => 'boolean',
 177         description => "Remove old backup files if there are more than 'maxfiles' backup files.",
 178         optional => 1,
 179         default => 1,
 180     },
 181 };
 182
 183 # Load available plugins
 184 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
 185 foreach my $plug (@pve_vzdump_classes) {
 186     my $filename = "/usr/share/perl5/$plug.pm";
 187     $filename =~ s!::!/!g;
 188     if (-f $filename) {
 189         eval { require $filename; };
 190         if (!$@) {
 191             $plug->import ();
 192             push @plugins, $plug;
 193         } else {
 194             die $@;
 195         }
 196     }
 197 }
 198
 199 # helper functions
 200
 201 my $debugstattxt = {
 202     err =>  'ERROR:',
 203     info => 'INFO:',
 204     warn => 'WARN:',
 205 };
 206
 207 sub debugmsg {
 208     my ($mtype, $msg, $logfd, $syslog) = @_;
 209
 210     chomp $msg;
 211
 212     return if !$msg;
 213
 214     my $pre = $debugstattxt->{$mtype} || $debugstattxt->{'err'};
 215
 216     my $timestr = strftime ("%F %H:%M:%S", CORE::localtime);
 217
 218     syslog ($mtype eq 'info' ? 'info' : 'err', "$pre $msg") if $syslog;
 219
 220     foreach my $line (split (/\n/, $msg)) {
 221         print STDERR "$pre $line\n";
 222         print $logfd "$timestr $pre $line\n" if $logfd;
 223     }
 224 }
 225
 226 sub run_command {
 227     my ($logfd, $cmdstr, %param) = @_;
 228
 229     my $logfunc = sub {
 230         my $line = shift;
 231         debugmsg ('info', $line, $logfd);
 232     };
 233
 234     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
 235 }
 236
 237 sub storage_info {
 238     my $storage = shift;
 239
 240     my $cfg = PVE::Storage::config();
 241     my $scfg = PVE::Storage::storage_config($cfg, $storage);
 242     my $type = $scfg->{type};
 243
 244     die "can't use storage type '$type' for backup\n"
 245         if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'));
 246     die "can't use storage '$storage' for backups - wrong content type\n"
 247         if (!$scfg->{content}->{backup});
 248
 249     PVE::Storage::activate_storage($cfg, $storage);
 250
 251     return {
 252         dumpdir => PVE::Storage::get_backup_dir($cfg, $storage),
 253         maxfiles => $scfg->{maxfiles},
 254     };
 255 }
 256
 257 sub format_size {
 258     my $size = shift;
 259
 260     my $kb = $size / 1024;
 261
 262     if ($kb < 1024) {
 263         return int ($kb) . "KB";
 264     }
 265
 266     my $mb = $size / (1024*1024);
 267
 268     if ($mb < 1024) {
 269         return int ($mb) . "MB";
 270     } else {
 271         my $gb = $mb / 1024;
 272         return sprintf ("%.2fGB", $gb);
 273     }
 274 }
 275
 276 sub format_time {
 277     my $seconds = shift;
 278
 279     my $hours = int ($seconds/3600);
 280     $seconds = $seconds - $hours*3600;
 281     my $min = int ($seconds/60);
 282     $seconds = $seconds - $min*60;
 283
 284     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 285 }
 286
 287 sub encode8bit {
 288     my ($str) = @_;
 289
 290     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 291
 292     return $str;
 293 }
 294
 295 sub escape_html {
 296     my ($str) = @_;
 297
 298     $str =~ s/&/&amp;/g;
 299     $str =~ s/</&lt;/g;
 300     $str =~ s/>/&gt;/g;
 301
 302     return $str;
 303 }
 304
 305 sub check_bin {
 306     my ($bin)  = @_;
 307
 308     foreach my $p (split (/:/, $ENV{PATH})) {
 309         my $fn = "$p/$bin";
 310         if (-x $fn) {
 311             return $fn;
 312         }
 313     }
 314
 315     die "unable to find command '$bin'\n";
 316 }
 317
 318 sub check_vmids {
 319     my (@vmids) = @_;
 320
 321     my $res = [];
 322     foreach my $vmid (@vmids) {
 323         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 324         $vmid = int ($vmid); # remove leading zeros
 325         next if !$vmid;
 326         push @$res, $vmid;
 327     }
 328
 329     return $res;
 330 }
 331
 332
 333 sub read_vzdump_defaults {
 334
 335     my $fn = "/etc/vzdump.conf";
 336
 337     my $defaults = {
 338         map {
 339             my $default = $confdesc->{$_}->{default};
 340              defined($default) ? ($_ => $default) : ()
 341         } keys %$confdesc
 342     };
 343
 344     my $raw;
 345     eval { $raw = PVE::Tools::file_get_contents($fn); };
 346     return $defaults if $@;
 347
 348     my $conf_schema = { type => 'object', properties => $confdesc, };
 349     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 350     if (my $excludes = $res->{'exclude-path'}) {
 351         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 352     }
 353     if (defined($res->{mailto})) {
 354         my @mailto = PVE::Tools::split_list($res->{mailto});
 355         $res->{mailto} = [ @mailto ];
 356     }
 357
 358     foreach my $key (keys %$defaults) {
 359         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 360     }
 361
 362     return $res;
 363 }
 364
 365 sub sendmail {
 366     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 367
 368     my $opts = $self->{opts};
 369
 370     my $mailto = $opts->{mailto};
 371
 372     return if !($mailto && scalar(@$mailto));
 373
 374     my $cmdline = $self->{cmdline};
 375
 376     my $ecount = 0;
 377     foreach my $task (@$tasklist) {
 378         $ecount++ if $task->{state} ne 'ok';
 379         chomp $task->{msg} if $task->{msg};
 380         $task->{backuptime} = 0 if !$task->{backuptime};
 381         $task->{size} = 0 if !$task->{size};
 382         $task->{tarfile} = 'unknown' if !$task->{tarfile};
 383         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 384
 385         if ($task->{state} eq 'todo') {
 386             $task->{msg} = 'aborted';
 387         }
 388     }
 389
 390     my $notify = $opts->{mailnotification} || 'always';
 391     return if (!$ecount && !$err && ($notify eq 'failure'));
 392
 393     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 394     if ($err) {
 395         if ($err =~ /\n/) {
 396             $stat .= ": multiple problems";
 397         } else {
 398             $stat .= ": $err";
 399             $err = undef;
 400         }
 401     }
 402
 403     my $hostname = `hostname -f` || PVE::INotify::nodename();
 404     chomp $hostname;
 405
 406     # text part
 407     my $text = $err ? "$err\n\n" : '';
 408     $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", qw(VMID STATUS TIME SIZE FILENAME));
 409     foreach my $task (@$tasklist) {
 410         my $vmid = $task->{vmid};
 411         if  ($task->{state} eq 'ok') {
 412
 413             $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", $vmid,
 414                                 $task->{state},
 415                                 format_time($task->{backuptime}),
 416                                 format_size ($task->{size}),
 417                                 $task->{tarfile});
 418         } else {
 419             $text .= sprintf ("%-10s %-6s %10s %8.2fMB  %s\n", $vmid,
 420                                 $task->{state},
 421                                 format_time($task->{backuptime}),
 422                                 0, '-');
 423         }
 424     }
 425
 426     $text .= "\nDetailed backup logs:\n\n";
 427     $text .= "$cmdline\n\n";
 428
 429     $text .= $detail_pre . "\n" if defined($detail_pre);
 430     foreach my $task (@$tasklist) {
 431         my $vmid = $task->{vmid};
 432         my $log = $task->{tmplog};
 433         if (!$log) {
 434             $text .= "$vmid: no log available\n\n";
 435             next;
 436         }
 437         open (TMP, "$log");
 438         while (my $line = <TMP>) { $text .= encode8bit ("$vmid: $line"); }
 439         close (TMP);
 440         $text .= "\n";
 441     }
 442     $text .= $detail_post if defined($detail_post);
 443
 444     # html part
 445     my $html = "<html><body>\n";
 446     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 447     $html .= "<table border=1 cellpadding=3>\n";
 448     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 449
 450     my $ssize = 0;
 451
 452     foreach my $task (@$tasklist) {
 453         my $vmid = $task->{vmid};
 454         my $name = $task->{hostname};
 455
 456         if  ($task->{state} eq 'ok') {
 457
 458             $ssize += $task->{size};
 459
 460             $html .= sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 461                                 $vmid, $name,
 462                                 format_time($task->{backuptime}),
 463                                 format_size ($task->{size}),
 464                                 escape_html ($task->{tarfile}));
 465         } else {
 466             $html .= sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 467                                 $vmid, $name, format_time($task->{backuptime}),
 468                                 escape_html ($task->{msg}));
 469         }
 470     }
 471
 472     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 473  format_time ($totaltime), format_size ($ssize));
 474
 475     $html .= "</table><br><br>\n";
 476     $html .= "Detailed backup logs:<br /><br />\n";
 477     $html .= "<pre>\n";
 478     $html .= escape_html($cmdline) . "\n\n";
 479
 480     $html .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 481     foreach my $task (@$tasklist) {
 482         my $vmid = $task->{vmid};
 483         my $log = $task->{tmplog};
 484         if (!$log) {
 485             $html .= "$vmid: no log available\n\n";
 486             next;
 487         }
 488         open (TMP, "$log");
 489         while (my $line = <TMP>) {
 490             if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 491                 $html .= encode8bit ("$vmid: <font color=red>".
 492                                        escape_html ($line) . "</font>");
 493             } else {
 494                 $html .= encode8bit ("$vmid: " . escape_html ($line));
 495             }
 496         }
 497         close (TMP);
 498         $html .= "\n";
 499     }
 500     $html .= escape_html($detail_post) if defined($detail_post);
 501     $html .= "</pre></body></html>\n";
 502     # end html part
 503
 504     my $subject = "vzdump backup status ($hostname) : $stat";
 505
 506     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 507     my $mailfrom = $dcconf->{email_from} || "root";
 508
 509     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 510 };
 511
 512 sub new {
 513     my ($class, $cmdline, $opts, $skiplist) = @_;
 514
 515     mkpath $logdir;
 516
 517     check_bin ('cp');
 518     check_bin ('df');
 519     check_bin ('sendmail');
 520     check_bin ('rsync');
 521     check_bin ('tar');
 522     check_bin ('mount');
 523     check_bin ('umount');
 524     check_bin ('cstream');
 525     check_bin ('ionice');
 526
 527     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 528         check_bin ('lvcreate');
 529         check_bin ('lvs');
 530         check_bin ('lvremove');
 531     }
 532
 533     my $defaults = read_vzdump_defaults();
 534
 535     my $maxfiles = $opts->{maxfiles}; # save here, because we overwrite with default
 536
 537     $opts->{remove} = 1 if !defined($opts->{remove});
 538
 539     foreach my $k (keys %$defaults) {
 540         next if $k eq 'exclude-path'; # dealt with separately
 541         if ($k eq 'dumpdir' || $k eq 'storage') {
 542             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 543                 !defined ($opts->{storage});
 544         } else {
 545             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 546         }
 547     }
 548
 549     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 550     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 551
 552     $skiplist = [] if !$skiplist;
 553     my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist };
 554
 555     my $findexcl = $self->{findexcl} = [];
 556     if ($defaults->{'exclude-path'}) {
 557         push @$findexcl, @{$defaults->{'exclude-path'}};
 558     }
 559
 560     if ($opts->{'exclude-path'}) {
 561         push @$findexcl, @{$opts->{'exclude-path'}};
 562     }
 563
 564     if ($opts->{stdexcludes}) {
 565         push @$findexcl, '/tmp/?*',
 566                          '/var/tmp/?*',
 567                          '/var/run/?*.pid';
 568     }
 569
 570     foreach my $p (@plugins) {
 571
 572         my $pd = $p->new ($self);
 573
 574         push @{$self->{plugins}}, $pd;
 575     }
 576
 577     if (!$opts->{dumpdir} && !$opts->{storage}) {
 578         $opts->{storage} = 'local';
 579     }
 580
 581     my $errors = '';
 582
 583     if ($opts->{storage}) {
 584         my $info = storage_info ($opts->{storage});
 585         $opts->{dumpdir} = $info->{dumpdir};
 586         $maxfiles = $info->{maxfiles} if !defined($maxfiles) && defined($info->{maxfiles});
 587     } elsif ($opts->{dumpdir}) {
 588         $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
 589             if ! -d $opts->{dumpdir};
 590     } else {
 591         die "internal error";
 592     }
 593
 594     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 595         $errors .= "\n" if $errors;
 596         $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
 597     }
 598
 599     if ($errors) {
 600         eval { $self->sendmail([], 0, $errors); };
 601         debugmsg ('err', $@) if $@;
 602         die "$errors\n";
 603     }
 604
 605     $opts->{maxfiles} = $maxfiles if defined($maxfiles);
 606
 607     return $self;
 608
 609 }
 610
 611 sub get_mount_info {
 612     my ($dir) = @_;
 613
 614     # Note: df 'available' can be negative, and percentage set to '-'
 615
 616     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 617
 618     my $res;
 619
 620     my $parser = sub {
 621         my $line = shift;
 622         if (my ($fsid, $fstype, undef, $mp) = $line =~
 623             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 624             $res = {
 625                 device => $fsid,
 626                 fstype => $fstype,
 627                 mountpoint => $mp,
 628             };
 629         }
 630     };
 631
 632     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 633     warn $@ if $@;
 634
 635     return $res;
 636 }
 637
 638 sub getlock {
 639     my ($self, $upid) = @_;
 640
 641     my $fh;
 642
 643     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 644
 645     die "missimg UPID" if !$upid; # should not happen
 646
 647     if (!open (SERVER_FLCK, ">>$lockfile")) {
 648         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 649         die "can't open lock on file '$lockfile' - $!";
 650     }
 651
 652     if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 653
 654         if (!$maxwait) {
 655             debugmsg ('err', "can't aquire lock '$lockfile' (wait = 0)", undef, 1);
 656             die "can't aquire lock '$lockfile' (wait = 0)";
 657         }
 658
 659         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 660
 661         eval {
 662             alarm ($maxwait * 60);
 663
 664             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 665
 666             if (!flock (SERVER_FLCK, LOCK_EX)) {
 667                 my $err = $!;
 668                 close (SERVER_FLCK);
 669                 alarm (0);
 670                 die "$err\n";
 671             }
 672             alarm (0);
 673         };
 674         alarm (0);
 675
 676         my $err = $@;
 677
 678         if ($err) {
 679             debugmsg ('err', "can't aquire lock '$lockfile' - $err", undef, 1);
 680             die "can't aquire lock '$lockfile' - $err";
 681         }
 682
 683         debugmsg('info', "got global lock", undef, 1);
 684     }
 685
 686     PVE::Tools::file_set_contents($pidfile, $upid);
 687 }
 688
 689 sub run_hook_script {
 690     my ($self, $phase, $task, $logfd) = @_;
 691
 692     my $opts = $self->{opts};
 693
 694     my $script = $opts->{script};
 695
 696     return if !$script;
 697
 698     my $cmd = "$script $phase";
 699
 700     $cmd .= " $task->{mode} $task->{vmid}" if ($task);
 701
 702     local %ENV;
 703
 704     # set immutable opts directly (so they are available in all phases)
 705     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 706     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 707
 708     foreach my $ek (qw(vmtype hostname tarfile logfile)) {
 709         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 710     }
 711
 712     run_command ($logfd, $cmd);
 713 }
 714
 715 sub compressor_info {
 716     my ($opts) = @_;
 717     my $opt_compress = $opts->{compress};
 718
 719     if (!$opt_compress || $opt_compress eq '0') {
 720         return undef;
 721     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 722         return ('lzop', 'lzo');
 723     } elsif ($opt_compress eq 'gzip') {
 724         if ($opts->{pigz} > 0) {
 725             # As default use int((#cores + 1)/2), we need #cores+1 for the case that #cores = 1
 726             my $cores = POSIX::sysconf(84);
 727             my $pigz_threads = ($opts->{pigz} > 1) ? $opts->{pigz} : int(($cores + 1)/2);
 728             return ("pigz -p ${pigz_threads}", 'gz');
 729         } else {
 730             return ('gzip', 'gz');
 731         }
 732     } else {
 733         die "internal error - unknown compression option '$opt_compress'";
 734     }
 735 }
 736
 737 sub get_backup_file_list {
 738     my ($dir, $bkname, $exclude_fn) = @_;
 739
 740     my $bklist = [];
 741     foreach my $fn (<$dir/${bkname}-*>) {
 742         next if $exclude_fn && $fn eq $exclude_fn;
 743         if ($fn =~ m!/(${bkname}-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?)))$!) {
 744             $fn = "$dir/$1"; # untaint
 745             my $t = timelocal ($7, $6, $5, $4, $3 - 1, $2 - 1900);
 746             push @$bklist, [$fn, $t];
 747         }
 748     }
 749
 750     return $bklist;
 751 }
 752
 753 sub exec_backup_task {
 754     my ($self, $task) = @_;
 755
 756     my $opts = $self->{opts};
 757
 758     my $vmid = $task->{vmid};
 759     my $plugin = $task->{plugin};
 760
 761     my $vmstarttime = time ();
 762
 763     my $logfd;
 764
 765     my $cleanup = {};
 766
 767     my $vmstoptime = 0;
 768
 769     eval {
 770         die "unable to find VM '$vmid'\n" if !$plugin;
 771
 772         # for now we deny backups of a running ha managed service in *stop* mode
 773         # as it interferes with the HA stack (started services should not stop).
 774         if ($opts->{mode} eq 'stop' &&
 775             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 776         {
 777             die "Cannot execute a backup with stop mode on a HA managed and".
 778                 " enabled Service. Use snapshot mode or disable the Service.\n";
 779         }
 780
 781         my $vmtype = $plugin->type();
 782
 783         my $tmplog = "$logdir/$vmtype-$vmid.log";
 784
 785         my $lt = localtime();
 786
 787         my $bkname = "vzdump-$vmtype-$vmid";
 788         my $basename = sprintf "${bkname}-%04d_%02d_%02d-%02d_%02d_%02d",
 789         $lt->year + 1900, $lt->mon + 1, $lt->mday,
 790         $lt->hour, $lt->min, $lt->sec;
 791
 792         my $maxfiles = $opts->{maxfiles};
 793
 794         if ($maxfiles && !$opts->{remove}) {
 795             my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 796             die "only $maxfiles backup(s) allowed - please consider to remove old backup files.\n"
 797                 if scalar(@$bklist) >= $maxfiles;
 798         }
 799
 800         my $logfile = $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 801
 802         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 803         my ($comp, $comp_ext) = compressor_info($opts);
 804         if ($comp && $comp_ext) {
 805             $ext .= ".${comp_ext}";
 806         }
 807
 808         if ($opts->{stdout}) {
 809             $task->{tarfile} = '-';
 810         } else {
 811             my $tarfile = $task->{tarfile} = "$opts->{dumpdir}/$basename$ext";
 812             $task->{tmptar} = $task->{tarfile};
 813             $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 814             unlink $task->{tmptar};
 815         }
 816
 817         $task->{vmtype} = $vmtype;
 818
 819         if ($opts->{tmpdir}) {
 820             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp$$";
 821         } else {
 822             # dumpdir is posix? then use it as temporary dir
 823             my $info = get_mount_info($opts->{dumpdir});
 824             if ($vmtype eq 'qemu' ||
 825                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 826                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 827             } else {
 828                 $task->{tmpdir} = "/var/tmp/vzdumptmp$$";
 829                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 830                           "using $task->{tmpdir} for temporary files", $logfd);
 831             }
 832         }
 833
 834         rmtree $task->{tmpdir};
 835         mkdir $task->{tmpdir};
 836         -d $task->{tmpdir} ||
 837             die "unable to create temporary directory '$task->{tmpdir}'";
 838
 839         $logfd = IO::File->new (">$tmplog") ||
 840             die "unable to create log file '$tmplog'";
 841
 842         $task->{dumpdir} = $opts->{dumpdir};
 843         $task->{storeid} = $opts->{storage};
 844         $task->{tmplog} = $tmplog;
 845
 846         unlink $logfile;
 847
 848         debugmsg ('info',  "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 849
 850         $plugin->set_logfd ($logfd);
 851
 852         # test is VM is running
 853         my ($running, $status_text) = $plugin->vm_status ($vmid);
 854
 855         debugmsg ('info', "status = ${status_text}", $logfd);
 856
 857         # lock VM (prevent config changes)
 858         $plugin->lock_vm ($vmid);
 859
 860         $cleanup->{unlock} = 1;
 861
 862         # prepare
 863
 864         my $mode = $running ? $task->{mode} : 'stop';
 865
 866         if ($mode eq 'snapshot') {
 867             my %saved_task = %$task;
 868             eval { $plugin->prepare ($task, $vmid, $mode); };
 869             if (my $err = $@) {
 870                 die $err if $err !~ m/^mode failure/;
 871                 debugmsg ('info',  $err, $logfd);
 872                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 873                 $mode = 'suspend'; # so prepare is called again below
 874                 %$task = %saved_task;
 875             }
 876         }
 877
 878         $cleanup->{prepared} = 1;
 879
 880         $task->{mode} = $mode;
 881
 882         debugmsg ('info', "backup mode: $mode", $logfd);
 883
 884         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
 885             if $opts->{bwlimit};
 886
 887         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 888
 889         if ($mode eq 'stop') {
 890
 891             $plugin->prepare ($task, $vmid, $mode);
 892
 893             $self->run_hook_script ('backup-start', $task, $logfd);
 894
 895             if ($running) {
 896                 debugmsg ('info', "stopping vm", $logfd);
 897                 $vmstoptime = time ();
 898                 $self->run_hook_script ('pre-stop', $task, $logfd);
 899                 $plugin->stop_vm ($task, $vmid);
 900                 $cleanup->{restart} = 1;
 901             }
 902
 903
 904         } elsif ($mode eq 'suspend') {
 905
 906             $plugin->prepare ($task, $vmid, $mode);
 907
 908             $self->run_hook_script ('backup-start', $task, $logfd);
 909
 910             if ($vmtype eq 'lxc') {
 911                 # pre-suspend rsync
 912                 $plugin->copy_data_phase1($task, $vmid);
 913             }
 914
 915             debugmsg ('info', "suspend vm", $logfd);
 916             $vmstoptime = time ();
 917             $self->run_hook_script ('pre-stop', $task, $logfd);
 918             $plugin->suspend_vm ($task, $vmid);
 919             $cleanup->{resume} = 1;
 920
 921             if ($vmtype eq 'lxc') {
 922                 # post-suspend rsync
 923                 $plugin->copy_data_phase2($task, $vmid);
 924
 925                 debugmsg ('info', "resume vm", $logfd);
 926                 $cleanup->{resume} = 0;
 927                 $self->run_hook_script('pre-restart', $task, $logfd);
 928                 $plugin->resume_vm($task, $vmid);
 929                 $self->run_hook_script('post-restart', $task, $logfd);
 930                 my $delay = time () - $vmstoptime;
 931                 debugmsg('info', "vm is online again after $delay seconds", $logfd);
 932             }
 933
 934         } elsif ($mode eq 'snapshot') {
 935
 936             $self->run_hook_script ('backup-start', $task, $logfd);
 937
 938             my $snapshot_count = $task->{snapshot_count} || 0;
 939
 940             $self->run_hook_script ('pre-stop', $task, $logfd);
 941
 942             if ($snapshot_count > 1) {
 943                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 944                 $vmstoptime = time ();
 945                 $plugin->suspend_vm ($task, $vmid);
 946                 $cleanup->{resume} = 1;
 947             }
 948
 949             $plugin->snapshot ($task, $vmid);
 950
 951             $self->run_hook_script ('pre-restart', $task, $logfd);
 952
 953             if ($snapshot_count > 1) {
 954                 debugmsg ('info', "resume vm", $logfd);
 955                 $cleanup->{resume} = 0;
 956                 $plugin->resume_vm ($task, $vmid);
 957                 my $delay = time () - $vmstoptime;
 958                 debugmsg ('info', "vm is online again after $delay seconds", $logfd);
 959             }
 960
 961             $self->run_hook_script ('post-restart', $task, $logfd);
 962
 963         } else {
 964             die "internal error - unknown mode '$mode'\n";
 965         }
 966
 967         # assemble archive image
 968         $plugin->assemble ($task, $vmid);
 969
 970         # produce archive
 971
 972         if ($opts->{stdout}) {
 973             debugmsg ('info', "sending archive to stdout", $logfd);
 974             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 975             $self->run_hook_script ('backup-end', $task, $logfd);
 976             return;
 977         }
 978
 979         debugmsg ('info', "creating archive '$task->{tarfile}'", $logfd);
 980         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 981
 982         rename ($task->{tmptar}, $task->{tarfile}) ||
 983             die "unable to rename '$task->{tmptar}' to '$task->{tarfile}'\n";
 984
 985         # determine size
 986         $task->{size} = (-s $task->{tarfile}) || 0;
 987         my $cs = format_size ($task->{size});
 988         debugmsg ('info', "archive file size: $cs", $logfd);
 989
 990         # purge older backup
 991
 992         if ($maxfiles && $opts->{remove}) {
 993             my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{tarfile});
 994             $bklist = [ sort { $b->[1] <=> $a->[1] } @$bklist ];
 995
 996             while (scalar (@$bklist) >= $maxfiles) {
 997                 my $d = pop @$bklist;
 998                 debugmsg ('info', "delete old backup '$d->[0]'", $logfd);
 999                 unlink $d->[0];
1000                 my $logfn = $d->[0];
1001                 $logfn =~ s/\.(tgz|((tar|vma)(\.(gz|lzo))?))$/\.log/;
1002                 unlink $logfn;
1003             }
1004         }
1005
1006         $self->run_hook_script ('backup-end', $task, $logfd);
1007     };
1008     my $err = $@;
1009
1010     if ($plugin) {
1011         # clean-up
1012
1013         if ($cleanup->{unlock}) {
1014             eval { $plugin->unlock_vm ($vmid); };
1015             warn $@ if $@;
1016         }
1017
1018         if ($cleanup->{prepared}) {
1019             # only call cleanup when necessary (when prepare was executed)
1020             eval { $plugin->cleanup ($task, $vmid) };
1021             warn $@ if $@;
1022         }
1023
1024         eval { $plugin->set_logfd (undef); };
1025         warn $@ if $@;
1026
1027         if ($cleanup->{resume} || $cleanup->{restart}) {
1028             eval {
1029                 $self->run_hook_script ('pre-restart', $task, $logfd);
1030                 if ($cleanup->{resume}) {
1031                     debugmsg ('info', "resume vm", $logfd);
1032                     $plugin->resume_vm ($task, $vmid);
1033                 } else {
1034                     my $running = $plugin->vm_status($vmid);
1035                     if (!$running) {
1036                         debugmsg ('info', "restarting vm", $logfd);
1037                         $plugin->start_vm ($task, $vmid);
1038                     }
1039                 }
1040                 $self->run_hook_script ('post-restart', $task, $logfd);
1041             };
1042             my $err = $@;
1043             if ($err) {
1044                 warn $err;
1045             } else {
1046                 my $delay = time () - $vmstoptime;
1047                 debugmsg ('info', "vm is online again after $delay seconds", $logfd);
1048             }
1049         }
1050     }
1051
1052     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
1053     warn $@ if $@;
1054
1055     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
1056     warn $@ if $@;
1057
1058     my $delay = $task->{backuptime} = time () - $vmstarttime;
1059
1060     if ($err) {
1061         $task->{state} = 'err';
1062         $task->{msg} = $err;
1063         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1064
1065         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1066
1067     } else {
1068         $task->{state} = 'ok';
1069         my $tstr = format_time ($delay);
1070         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1071     }
1072
1073     close ($logfd) if $logfd;
1074
1075     if ($task->{tmplog} && $task->{logfile}) {
1076         system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1077     }
1078
1079     eval { $self->run_hook_script ('log-end', $task); };
1080
1081     die $err if $err && $err =~ m/^interrupted by signal$/;
1082 }
1083
1084 sub exec_backup {
1085     my ($self, $rpcenv, $authuser) = @_;
1086
1087     my $opts = $self->{opts};
1088
1089     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1090     debugmsg ('info', "skip external VMs: " . join(', ', @{$self->{skiplist}}))
1091         if scalar(@{$self->{skiplist}});
1092
1093     my $tasklist = [];
1094
1095     if ($opts->{all}) {
1096         foreach my $plugin (@{$self->{plugins}}) {
1097             my $vmlist = $plugin->vmlist();
1098             foreach my $vmid (sort @$vmlist) {
1099                 next if grep { $_ eq  $vmid } @{$opts->{exclude}};
1100                 next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], 1);
1101                 push @$tasklist, { vmid => $vmid,  state => 'todo', plugin => $plugin, mode => $opts->{mode} };
1102             }
1103         }
1104     } else {
1105         foreach my $vmid (sort @{$opts->{vmids}}) {
1106             my $plugin;
1107             foreach my $pg (@{$self->{plugins}}) {
1108                 my $vmlist = $pg->vmlist();
1109                 if (grep { $_ eq  $vmid } @$vmlist) {
1110                     $plugin = $pg;
1111                     last;
1112                 }
1113             }
1114             $rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ]);
1115             push @$tasklist, { vmid => $vmid,  state => 'todo', plugin => $plugin, mode => $opts->{mode} };
1116         }
1117     }
1118
1119     # Use in-memory files for the outer hook logs to pass them to sendmail.
1120     my $job_start_log = '';
1121     my $job_end_log = '';
1122     open my $job_start_fd, '>', \$job_start_log;
1123     open my $job_end_fd, '>', \$job_end_log;
1124
1125     my $starttime = time();
1126     my $errcount = 0;
1127     eval {
1128
1129         $self->run_hook_script ('job-start', undef, $job_start_fd);
1130
1131         foreach my $task (@$tasklist) {
1132             $self->exec_backup_task ($task);
1133             $errcount += 1 if $task->{state} ne 'ok';
1134         }
1135
1136         $self->run_hook_script ('job-end', undef, $job_end_fd);
1137     };
1138     my $err = $@;
1139
1140     $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1141
1142     if ($err) {
1143         debugmsg ('err', "Backup job failed - $err", undef, 1);
1144     } else {
1145         if ($errcount) {
1146             debugmsg ('info', "Backup job finished with errors", undef, 1);
1147         } else {
1148             debugmsg ('info', "Backup job finished successfully", undef, 1);
1149         }
1150     }
1151
1152     close $job_start_fd;
1153     close $job_end_fd;
1154
1155     my $totaltime = time() - $starttime;
1156
1157     eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); };
1158     debugmsg ('err', $@) if $@;
1159
1160     die $err if $err;
1161
1162     die "job errors\n" if $errcount;
1163
1164     unlink $pidfile;
1165 }
1166
1167
1168 sub option_exists {
1169     my $key = shift;
1170     return defined($confdesc->{$key});
1171 }
1172
1173 # add JSON properties for create and set function
1174 sub json_config_properties {
1175     my $prop = shift;
1176
1177     foreach my $opt (keys %$confdesc) {
1178         $prop->{$opt} = $confdesc->{$opt};
1179     }
1180
1181     return $prop;
1182 }
1183
1184 sub verify_vzdump_parameters {
1185     my ($param, $check_missing) = @_;
1186
1187     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1188         if $param->{all} && $param->{vmid};
1189
1190     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1191         if $param->{exclude} && $param->{vmid};
1192
1193     $param->{all} = 1 if defined($param->{exclude});
1194
1195     warn "option 'size' is deprecated and will be removed in a future " .
1196          "release, please update your script/configuration!\n"
1197         if defined($param->{size});
1198
1199     return if !$check_missing;
1200
1201     raise_param_exc({ vmid => "property is missing"})
1202         if !($param->{all} || $param->{stop}) && !$param->{vmid};
1203
1204 }
1205
1206 sub stop_running_backups {
1207     my($self) = @_;
1208
1209     my $upid = PVE::Tools::file_read_firstline($pidfile);
1210     return if !$upid;
1211
1212     my $task = PVE::Tools::upid_decode($upid);
1213
1214     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1215         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1216         kill(15, $task->{pid});
1217         # wait max 15 seconds to shut down (else, do nothing for now)
1218         my $i;
1219         for ($i = 15; $i > 0; $i--) {
1220             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1221             sleep (1);
1222         }
1223         die "stoping backup process $task->{pid} failed\n" if $i == 0;
1224     }
1225 }
1226
1227 sub command_line {
1228     my ($param) = @_;
1229
1230     my $cmd = "vzdump";
1231
1232     if ($param->{vmid}) {
1233         $cmd .= " " . join(' ', PVE::Tools::split_list($param->{vmid}));
1234     }
1235
1236     foreach my $p (keys %$param) {
1237         next if $p eq 'id' || $p eq 'vmid' || $p eq 'starttime' ||
1238                 $p eq 'dow' || $p eq 'stdout' || $p eq 'enabled';
1239         my $v = $param->{$p};
1240         my $pd = $confdesc->{$p} || die "no such vzdump option '$p'\n";
1241         if ($p eq 'exclude-path') {
1242             foreach my $path (split(/\0/, $v || '')) {
1243                 $cmd .= " --$p " . PVE::Tools::shellquote($path);
1244             }
1245         } else {
1246             $cmd .= " --$p " . PVE::Tools::shellquote($v) if defined($v) && $v ne '';
1247         }
1248     }
1249
1250     return $cmd;
1251 }
1252
1253 # bash completion helpers
1254 sub complete_backup_storage {
1255
1256     my $cfg = PVE::Storage::config();
1257     my $ids = $cfg->{ids};
1258
1259     my $nodename = PVE::INotify::nodename();
1260
1261     my $res = [];
1262     foreach my $sid (keys %$ids) {
1263         my $scfg = $ids->{$sid};
1264         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $nodename, 1);
1265         next if !$scfg->{content}->{backup};
1266         push @$res, $sid;
1267     }
1268
1269     return $res;
1270 }
1271
1272 1;