PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Path;
   8 use IO::File;
   9 use IO::Select;
  10 use IPC::Open3;
  11 use POSIX qw(strftime);
  12 use Time::Local;
  13
  14 use PVE::Cluster qw(cfs_read_file);
  15 use PVE::DataCenterConfig;
  16 use PVE::Exception qw(raise_param_exc);
  17 use PVE::HA::Config;
  18 use PVE::HA::Env::PVE2;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::RPCEnvironment;
  21 use PVE::Storage;
  22 use PVE::VZDump::Common;
  23 use PVE::VZDump::Plugin;
  24 use PVE::Tools qw(extract_param split_list);
  25 use PVE::API2Tools;
  26
  27 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  28
  29 my $lockfile = '/var/run/vzdump.lock';
  30 my $pidfile = '/var/run/vzdump.pid';
  31 my $logdir = '/var/log/vzdump';
  32
  33 my @plugins = qw();
  34
  35 my $confdesc = PVE::VZDump::Common::get_confdesc();
  36
  37 # Load available plugins
  38 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
  39 foreach my $plug (@pve_vzdump_classes) {
  40     my $filename = "/usr/share/perl5/$plug.pm";
  41     $filename =~ s!::!/!g;
  42     if (-f $filename) {
  43         eval { require $filename; };
  44         if (!$@) {
  45             $plug->import ();
  46             push @plugins, $plug;
  47         } else {
  48             die $@;
  49         }
  50     }
  51 }
  52
  53 # helper functions
  54
  55 sub debugmsg {
  56     my ($mtype, $msg, $logfd, $syslog) = @_;
  57
  58     PVE::VZDump::Plugin::debugmsg(@_);
  59 }
  60
  61 sub run_command {
  62     my ($logfd, $cmdstr, %param) = @_;
  63
  64     my $logfunc = sub {
  65         my $line = shift;
  66         debugmsg ('info', $line, $logfd);
  67     };
  68
  69     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
  70 }
  71
  72 sub storage_info {
  73     my $storage = shift;
  74
  75     my $cfg = PVE::Storage::config();
  76     my $scfg = PVE::Storage::storage_config($cfg, $storage);
  77     my $type = $scfg->{type};
  78
  79     die "can't use storage type '$type' for backup\n"
  80         if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'
  81               || $type eq 'cifs' || $type eq 'cephfs' || $type eq 'pbs'));
  82     die "can't use storage '$storage' for backups - wrong content type\n"
  83         if (!$scfg->{content}->{backup});
  84
  85     PVE::Storage::activate_storage($cfg, $storage);
  86
  87     my $info = {
  88         scfg => $scfg,
  89         maxfiles => $scfg->{maxfiles},
  90     };
  91
  92     $info->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'})
  93         if defined($scfg->{'prune-backups'});
  94
  95     die "cannot have 'maxfiles' and 'prune-backups' configured at the same time\n"
  96         if defined($info->{'prune-backups'}) && defined($info->{maxfiles});
  97
  98     if ($type eq 'pbs') {
  99         $info->{pbs} = 1;
 100     } else {
 101         $info->{dumpdir} = PVE::Storage::get_backup_dir($cfg, $storage);
 102     }
 103
 104     return $info;
 105 }
 106
 107 sub format_size {
 108     my $size = shift;
 109
 110     my $kb = $size / 1024;
 111
 112     if ($kb < 1024) {
 113         return int ($kb) . "KB";
 114     }
 115
 116     my $mb = $size / (1024*1024);
 117     if ($mb < 1024) {
 118         return int ($mb) . "MB";
 119     }
 120     my $gb = $mb / 1024;
 121     if ($gb < 1024) {
 122         return sprintf ("%.2fGB", $gb);
 123     }
 124     my $tb = $gb / 1024;
 125     return sprintf ("%.2fTB", $tb);
 126 }
 127
 128 sub format_time {
 129     my $seconds = shift;
 130
 131     my $hours = int ($seconds/3600);
 132     $seconds = $seconds - $hours*3600;
 133     my $min = int ($seconds/60);
 134     $seconds = $seconds - $min*60;
 135
 136     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 137 }
 138
 139 sub encode8bit {
 140     my ($str) = @_;
 141
 142     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 143
 144     return $str;
 145 }
 146
 147 sub escape_html {
 148     my ($str) = @_;
 149
 150     $str =~ s/&/&amp;/g;
 151     $str =~ s/</&lt;/g;
 152     $str =~ s/>/&gt;/g;
 153
 154     return $str;
 155 }
 156
 157 sub check_bin {
 158     my ($bin)  = @_;
 159
 160     foreach my $p (split (/:/, $ENV{PATH})) {
 161         my $fn = "$p/$bin";
 162         if (-x $fn) {
 163             return $fn;
 164         }
 165     }
 166
 167     die "unable to find command '$bin'\n";
 168 }
 169
 170 sub check_vmids {
 171     my (@vmids) = @_;
 172
 173     my $res = [];
 174     for my $vmid (sort {$a <=> $b} @vmids) {
 175         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 176         $vmid = int ($vmid); # remove leading zeros
 177         next if !$vmid;
 178         push @$res, $vmid;
 179     }
 180
 181     return $res;
 182 }
 183
 184
 185 sub read_vzdump_defaults {
 186
 187     my $fn = "/etc/vzdump.conf";
 188
 189     my $defaults = {
 190         map {
 191             my $default = $confdesc->{$_}->{default};
 192              defined($default) ? ($_ => $default) : ()
 193         } keys %$confdesc
 194     };
 195
 196     my $raw;
 197     eval { $raw = PVE::Tools::file_get_contents($fn); };
 198     return $defaults if $@;
 199
 200     my $conf_schema = { type => 'object', properties => $confdesc, };
 201     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 202     if (my $excludes = $res->{'exclude-path'}) {
 203         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 204     }
 205     if (defined($res->{mailto})) {
 206         my @mailto = split_list($res->{mailto});
 207         $res->{mailto} = [ @mailto ];
 208     }
 209
 210     foreach my $key (keys %$defaults) {
 211         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 212     }
 213
 214     return $res;
 215 }
 216
 217 use constant MAX_MAIL_SIZE => 1024*1024;
 218 sub sendmail {
 219     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 220
 221     my $opts = $self->{opts};
 222
 223     my $mailto = $opts->{mailto};
 224
 225     return if !($mailto && scalar(@$mailto));
 226
 227     my $cmdline = $self->{cmdline};
 228
 229     my $ecount = 0;
 230     foreach my $task (@$tasklist) {
 231         $ecount++ if $task->{state} ne 'ok';
 232         chomp $task->{msg} if $task->{msg};
 233         $task->{backuptime} = 0 if !$task->{backuptime};
 234         $task->{size} = 0 if !$task->{size};
 235         $task->{target} = 'unknown' if !$task->{target};
 236         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 237
 238         if ($task->{state} eq 'todo') {
 239             $task->{msg} = 'aborted';
 240         }
 241     }
 242
 243     my $notify = $opts->{mailnotification} || 'always';
 244     return if (!$ecount && !$err && ($notify eq 'failure'));
 245
 246     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 247     if ($err) {
 248         if ($err =~ /\n/) {
 249             $stat .= ": multiple problems";
 250         } else {
 251             $stat .= ": $err";
 252             $err = undef;
 253         }
 254     }
 255
 256     my $hostname = `hostname -f` || PVE::INotify::nodename();
 257     chomp $hostname;
 258
 259     # text part
 260     my $text = $err ? "$err\n\n" : '';
 261     $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", qw(VMID STATUS TIME SIZE FILENAME));
 262     foreach my $task (@$tasklist) {
 263         my $vmid = $task->{vmid};
 264         if  ($task->{state} eq 'ok') {
 265
 266             $text .= sprintf ("%-10s %-6s %10s %10s  %s\n", $vmid,
 267                                 $task->{state},
 268                                 format_time($task->{backuptime}),
 269                                 format_size ($task->{size}),
 270                                 $task->{target});
 271         } else {
 272             $text .= sprintf ("%-10s %-6s %10s %8.2fMB  %s\n", $vmid,
 273                                 $task->{state},
 274                                 format_time($task->{backuptime}),
 275                                 0, '-');
 276         }
 277     }
 278
 279     my $text_log_part;
 280     $text_log_part .= "\nDetailed backup logs:\n\n";
 281     $text_log_part .= "$cmdline\n\n";
 282
 283     $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
 284     foreach my $task (@$tasklist) {
 285         my $vmid = $task->{vmid};
 286         my $log = $task->{tmplog};
 287         if (!$log) {
 288             $text_log_part .= "$vmid: no log available\n\n";
 289             next;
 290         }
 291         if (open (TMP, "$log")) {
 292             while (my $line = <TMP>) {
 293                 next if $line =~ /^status: \d+/; # not useful in mails
 294                 $text_log_part .= encode8bit ("$vmid: $line");
 295             }
 296         } else {
 297             $text_log_part .= "$vmid: Could not open log file\n\n";
 298         }
 299         close (TMP);
 300         $text_log_part .= "\n";
 301     }
 302     $text_log_part .= $detail_post if defined($detail_post);
 303
 304     # html part
 305     my $html = "<html><body>\n";
 306     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 307     $html .= "<table border=1 cellpadding=3>\n";
 308     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 309
 310     my $ssize = 0;
 311
 312     foreach my $task (@$tasklist) {
 313         my $vmid = $task->{vmid};
 314         my $name = $task->{hostname};
 315
 316         if  ($task->{state} eq 'ok') {
 317
 318             $ssize += $task->{size};
 319
 320             $html .= sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 321                                 $vmid, $name,
 322                                 format_time($task->{backuptime}),
 323                                 format_size ($task->{size}),
 324                                 escape_html ($task->{target}));
 325         } else {
 326             $html .= sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 327                                 $vmid, $name, format_time($task->{backuptime}),
 328                                 escape_html ($task->{msg}));
 329         }
 330     }
 331
 332     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 333  format_time ($totaltime), format_size ($ssize));
 334
 335     $html .= "\n</table><br><br>\n";
 336     my $html_log_part;
 337     $html_log_part .= "Detailed backup logs:<br /><br />\n";
 338     $html_log_part .= "<pre>\n";
 339     $html_log_part .= escape_html($cmdline) . "\n\n";
 340
 341     $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 342     foreach my $task (@$tasklist) {
 343         my $vmid = $task->{vmid};
 344         my $log = $task->{tmplog};
 345         if (!$log) {
 346             $html_log_part .= "$vmid: no log available\n\n";
 347             next;
 348         }
 349         if (open (TMP, "$log")) {
 350             while (my $line = <TMP>) {
 351                 next if $line =~ /^status: \d+/; # not useful in mails
 352                 if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 353                     $html_log_part .= encode8bit ("$vmid: <font color=red>".
 354                         escape_html ($line) . "</font>");
 355                 } else {
 356                     $html_log_part .= encode8bit ("$vmid: " . escape_html ($line));
 357                 }
 358             }
 359         } else {
 360             $html_log_part .= "$vmid: Could not open log file\n\n";
 361         }
 362         close (TMP);
 363         $html_log_part .= "\n";
 364     }
 365     $html_log_part .= escape_html($detail_post) if defined($detail_post);
 366     $html_log_part .= "</pre>";
 367     my $html_end .= "\n</body></html>\n";
 368     # end html part
 369
 370     if (length($text) + length($text_log_part) +
 371         length($html) + length($html_log_part) < MAX_MAIL_SIZE)
 372     {
 373         $html .= $html_log_part;
 374         $text .= $text_log_part;
 375     } else {
 376         my $msg = "Log output was too long to be sent by mail. ".
 377             "See Task History for details!\n";
 378         $text .= $msg;
 379         $html .= "<p>$msg</p>";
 380         $html .= $html_end;
 381     }
 382
 383     my $subject = "vzdump backup status ($hostname) : $stat";
 384
 385     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 386     my $mailfrom = $dcconf->{email_from} || "root";
 387
 388     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 389 };
 390
 391 sub new {
 392     my ($class, $cmdline, $opts, $skiplist) = @_;
 393
 394     mkpath $logdir;
 395
 396     check_bin ('cp');
 397     check_bin ('df');
 398     check_bin ('sendmail');
 399     check_bin ('rsync');
 400     check_bin ('tar');
 401     check_bin ('mount');
 402     check_bin ('umount');
 403     check_bin ('cstream');
 404     check_bin ('ionice');
 405
 406     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 407         check_bin ('lvcreate');
 408         check_bin ('lvs');
 409         check_bin ('lvremove');
 410     }
 411
 412     my $defaults = read_vzdump_defaults();
 413
 414     $opts->{remove} = 1 if !defined($opts->{remove});
 415
 416     foreach my $k (keys %$defaults) {
 417         next if $k eq 'exclude-path' || $k eq 'maxfiles'; # dealt with separately
 418         if ($k eq 'dumpdir' || $k eq 'storage') {
 419             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 420                 !defined ($opts->{storage});
 421         } else {
 422             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 423         }
 424     }
 425
 426     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 427     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 428
 429     $skiplist = [] if !$skiplist;
 430     my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist };
 431
 432     my $findexcl = $self->{findexcl} = [];
 433     if ($defaults->{'exclude-path'}) {
 434         push @$findexcl, @{$defaults->{'exclude-path'}};
 435     }
 436
 437     if ($opts->{'exclude-path'}) {
 438         push @$findexcl, @{$opts->{'exclude-path'}};
 439     }
 440
 441     if ($opts->{stdexcludes}) {
 442         push @$findexcl, '/tmp/?*',
 443                          '/var/tmp/?*',
 444                          '/var/run/?*.pid';
 445     }
 446
 447     foreach my $p (@plugins) {
 448
 449         my $pd = $p->new ($self);
 450
 451         push @{$self->{plugins}}, $pd;
 452     }
 453
 454     if (defined($opts->{storage}) && $opts->{stdout}) {
 455         die "cannot use options 'storage' and 'stdout' at the same time\n";
 456     } elsif (defined($opts->{storage}) && defined($opts->{dumpdir})) {
 457         die "cannot use options 'storage' and 'dumpdir' at the same time\n";
 458     }
 459
 460     if (!$opts->{dumpdir} && !$opts->{storage}) {
 461         $opts->{storage} = 'local';
 462     }
 463
 464     my $errors = '';
 465
 466     if ($opts->{storage}) {
 467         my $info = eval { storage_info ($opts->{storage}) };
 468         if (my $err = $@) {
 469             $errors .= "could not get storage information for '$opts->{storage}': $err";
 470         } else {
 471             $opts->{dumpdir} = $info->{dumpdir};
 472             $opts->{scfg} = $info->{scfg};
 473             $opts->{pbs} = $info->{pbs};
 474
 475             if (!defined($opts->{'prune-backups'}) && !defined($opts->{maxfiles})) {
 476                 $opts->{'prune-backups'} = $info->{'prune-backups'};
 477                 $opts->{maxfiles} = $info->{maxfiles};
 478             }
 479         }
 480     } elsif ($opts->{dumpdir}) {
 481         $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
 482             if ! -d $opts->{dumpdir};
 483     } else {
 484         die "internal error";
 485     }
 486
 487     if (!defined($opts->{'prune-backups'}) && !defined($opts->{maxfiles})) {
 488         $opts->{maxfiles} = $defaults->{maxfiles};
 489     }
 490
 491     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 492         $errors .= "\n" if $errors;
 493         $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
 494     }
 495
 496     if ($errors) {
 497         eval { $self->sendmail([], 0, $errors); };
 498         debugmsg ('err', $@) if $@;
 499         die "$errors\n";
 500     }
 501
 502     return $self;
 503 }
 504
 505 sub get_mount_info {
 506     my ($dir) = @_;
 507
 508     # Note: df 'available' can be negative, and percentage set to '-'
 509
 510     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 511
 512     my $res;
 513
 514     my $parser = sub {
 515         my $line = shift;
 516         if (my ($fsid, $fstype, undef, $mp) = $line =~
 517             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 518             $res = {
 519                 device => $fsid,
 520                 fstype => $fstype,
 521                 mountpoint => $mp,
 522             };
 523         }
 524     };
 525
 526     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 527     warn $@ if $@;
 528
 529     return $res;
 530 }
 531
 532 sub getlock {
 533     my ($self, $upid) = @_;
 534
 535     my $fh;
 536
 537     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 538
 539     die "missing UPID" if !$upid; # should not happen
 540
 541     if (!open (SERVER_FLCK, ">>$lockfile")) {
 542         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 543         die "can't open lock on file '$lockfile' - $!";
 544     }
 545
 546     if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 547
 548         if (!$maxwait) {
 549             debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
 550             die "can't acquire lock '$lockfile' (wait = 0)";
 551         }
 552
 553         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 554
 555         eval {
 556             alarm ($maxwait * 60);
 557
 558             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 559
 560             if (!flock (SERVER_FLCK, LOCK_EX)) {
 561                 my $err = $!;
 562                 close (SERVER_FLCK);
 563                 alarm (0);
 564                 die "$err\n";
 565             }
 566             alarm (0);
 567         };
 568         alarm (0);
 569
 570         my $err = $@;
 571
 572         if ($err) {
 573             debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
 574             die "can't acquire lock '$lockfile' - $err";
 575         }
 576
 577         debugmsg('info', "got global lock", undef, 1);
 578     }
 579
 580     PVE::Tools::file_set_contents($pidfile, $upid);
 581 }
 582
 583 sub run_hook_script {
 584     my ($self, $phase, $task, $logfd) = @_;
 585
 586     my $opts = $self->{opts};
 587
 588     my $script = $opts->{script};
 589     return if !$script;
 590
 591     if (!-x $script) {
 592         die "The hook script '$script' is not executable.\n";
 593     }
 594
 595     my $cmd = "$script $phase";
 596
 597     $cmd .= " $task->{mode} $task->{vmid}" if ($task);
 598
 599     local %ENV;
 600     # set immutable opts directly (so they are available in all phases)
 601     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 602     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 603
 604     foreach my $ek (qw(vmtype hostname target logfile)) {
 605         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 606     }
 607     # FIXME: for backwards compatibility - drop with PVE 7.0
 608     $ENV{TARFILE} = $task->{target} if $task->{target};
 609
 610     run_command ($logfd, $cmd);
 611 }
 612
 613 sub compressor_info {
 614     my ($opts) = @_;
 615     my $opt_compress = $opts->{compress};
 616
 617     if (!$opt_compress || $opt_compress eq '0') {
 618         return undef;
 619     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 620         return ('lzop', 'lzo');
 621     } elsif ($opt_compress eq 'gzip') {
 622         if ($opts->{pigz} > 0) {
 623             my $pigz_threads = $opts->{pigz};
 624             if ($pigz_threads == 1) {
 625                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 626                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 627             }
 628             return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
 629         } else {
 630             return ('gzip --rsyncable', 'gz');
 631         }
 632     } elsif ($opt_compress eq 'zstd') {
 633         my $zstd_threads = $opts->{zstd} // 1;
 634         if ($zstd_threads == 0) {
 635             my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 636             $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
 637         }
 638         return ("zstd --rsyncable --threads=${zstd_threads}", 'zst');
 639     } else {
 640         die "internal error - unknown compression option '$opt_compress'";
 641     }
 642 }
 643
 644 sub get_backup_file_list {
 645     my ($dir, $bkname, $exclude_fn) = @_;
 646
 647     my $bklist = [];
 648     foreach my $fn (<$dir/${bkname}-*>) {
 649         next if $exclude_fn && $fn eq $exclude_fn;
 650
 651         my $archive_info = eval { PVE::Storage::archive_info($fn) } // {};
 652         if ($archive_info->{is_std_name}) {
 653             my $filename = $archive_info->{filename};
 654             my $backup = {
 655                 'path' => "$dir/$filename",
 656                 'ctime' => $archive_info->{ctime},
 657             };
 658             push @{$bklist}, $backup;
 659         }
 660     }
 661
 662     return $bklist;
 663 }
 664
 665 sub exec_backup_task {
 666     my ($self, $task) = @_;
 667
 668     my $opts = $self->{opts};
 669
 670     my $cfg = PVE::Storage::config();
 671     my $vmid = $task->{vmid};
 672     my $plugin = $task->{plugin};
 673     my $vmtype = $plugin->type();
 674
 675     $task->{backup_time} = time();
 676
 677     my $pbs_group_name;
 678     my $pbs_snapshot_name;
 679
 680     if ($self->{opts}->{pbs}) {
 681         if ($vmtype eq 'lxc') {
 682             $pbs_group_name = "ct/$vmid";
 683         } elsif  ($vmtype eq 'qemu') {
 684             $pbs_group_name = "vm/$vmid";
 685         } else {
 686             die "pbs backup not implemented for plugin type '$vmtype'\n";
 687         }
 688         my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
 689         $pbs_snapshot_name = "$pbs_group_name/$btime";
 690     }
 691
 692     my $vmstarttime = time ();
 693
 694     my $logfd;
 695
 696     my $cleanup = {};
 697
 698     my $log_vm_online_again = sub {
 699         return if !defined($task->{vmstoptime});
 700         $task->{vmconttime} //= time();
 701         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 702         $delay = '<1' if $delay < 1;
 703         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 704     };
 705
 706     eval {
 707         die "unable to find VM '$vmid'\n" if !$plugin;
 708
 709         # for now we deny backups of a running ha managed service in *stop* mode
 710         # as it interferes with the HA stack (started services should not stop).
 711         if ($opts->{mode} eq 'stop' &&
 712             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 713         {
 714             die "Cannot execute a backup with stop mode on a HA managed and".
 715                 " enabled Service. Use snapshot mode or disable the Service.\n";
 716         }
 717
 718         my $tmplog = "$logdir/$vmtype-$vmid.log";
 719
 720         my $bkname = "vzdump-$vmtype-$vmid";
 721         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
 722
 723         my $maxfiles = $opts->{maxfiles};
 724         my $prune_options = $opts->{'prune-backups'};
 725
 726         my $backup_limit = 0;
 727         if (defined($maxfiles)) {
 728             $backup_limit = $maxfiles;
 729         } elsif (defined($prune_options)) {
 730             foreach my $keep (values %{$prune_options}) {
 731                 $backup_limit += $keep;
 732             }
 733         }
 734
 735         if ($backup_limit && !$opts->{remove}) {
 736             my $count;
 737             if ($self->{opts}->{pbs}) {
 738                 my $res = PVE::Storage::PBSPlugin::run_client_cmd($opts->{scfg}, $opts->{storage}, 'snapshots', $pbs_group_name);
 739                 $count = scalar(@$res);
 740             } else {
 741                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 742                 $count = scalar(@$bklist);
 743             }
 744             die "There is a max backup limit of $backup_limit enforced by the".
 745                 " target storage or the vzdump parameters.".
 746                 " Either increase the limit or delete old backup(s).\n"
 747                 if $count >= $backup_limit;
 748         }
 749
 750         if (!$self->{opts}->{pbs}) {
 751             $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 752         }
 753
 754         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 755         my ($comp, $comp_ext) = compressor_info($opts);
 756         if ($comp && $comp_ext) {
 757             $ext .= ".${comp_ext}";
 758         }
 759
 760         if ($self->{opts}->{pbs}) {
 761             die "unable to pipe backup to stdout\n" if $opts->{stdout};
 762             $task->{target} = $pbs_snapshot_name;
 763         } else {
 764             if ($opts->{stdout}) {
 765                 $task->{target} = '-';
 766             } else {
 767                 $task->{target} = $task->{tmptar} = "$opts->{dumpdir}/$basename$ext";
 768                 $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 769                 unlink $task->{tmptar};
 770             }
 771         }
 772
 773         $task->{vmtype} = $vmtype;
 774
 775         if ($opts->{tmpdir}) {
 776             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp$$";
 777         } elsif ($self->{opts}->{pbs}) {
 778             $task->{tmpdir} = "/var/tmp/vzdumptmp$$"; #fixme
 779         } else {
 780             # dumpdir is posix? then use it as temporary dir
 781             my $info = get_mount_info($opts->{dumpdir});
 782             if ($vmtype eq 'qemu' ||
 783                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 784                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 785             } else {
 786                 $task->{tmpdir} = "/var/tmp/vzdumptmp$$";
 787                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 788                           "using $task->{tmpdir} for temporary files", $logfd);
 789             }
 790         }
 791
 792         rmtree $task->{tmpdir};
 793         mkdir $task->{tmpdir};
 794         -d $task->{tmpdir} ||
 795             die "unable to create temporary directory '$task->{tmpdir}'";
 796
 797         $logfd = IO::File->new (">$tmplog") ||
 798             die "unable to create log file '$tmplog'";
 799
 800         $task->{dumpdir} = $opts->{dumpdir};
 801         $task->{storeid} = $opts->{storage};
 802         $task->{scfg} = $opts->{scfg};
 803         $task->{tmplog} = $tmplog;
 804
 805         unlink $task->{logfile} if defined($task->{logfile});
 806
 807         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 808         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
 809
 810         $plugin->set_logfd ($logfd);
 811
 812         # test is VM is running
 813         my ($running, $status_text) = $plugin->vm_status ($vmid);
 814
 815         debugmsg ('info', "status = ${status_text}", $logfd);
 816
 817         # lock VM (prevent config changes)
 818         $plugin->lock_vm ($vmid);
 819
 820         $cleanup->{unlock} = 1;
 821
 822         # prepare
 823
 824         my $mode = $running ? $task->{mode} : 'stop';
 825
 826         if ($mode eq 'snapshot') {
 827             my %saved_task = %$task;
 828             eval { $plugin->prepare ($task, $vmid, $mode); };
 829             if (my $err = $@) {
 830                 die $err if $err !~ m/^mode failure/;
 831                 debugmsg ('info',  $err, $logfd);
 832                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 833                 $mode = 'suspend'; # so prepare is called again below
 834                 %$task = %saved_task;
 835             }
 836         }
 837
 838         $cleanup->{prepared} = 1;
 839
 840         $task->{mode} = $mode;
 841
 842         debugmsg ('info', "backup mode: $mode", $logfd);
 843
 844         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
 845             if $opts->{bwlimit};
 846
 847         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 848
 849         if ($mode eq 'stop') {
 850
 851             $plugin->prepare ($task, $vmid, $mode);
 852
 853             $self->run_hook_script ('backup-start', $task, $logfd);
 854
 855             if ($running) {
 856                 debugmsg ('info', "stopping vm", $logfd);
 857                 $task->{vmstoptime} = time();
 858                 $self->run_hook_script ('pre-stop', $task, $logfd);
 859                 $plugin->stop_vm ($task, $vmid);
 860                 $cleanup->{restart} = 1;
 861             }
 862
 863
 864         } elsif ($mode eq 'suspend') {
 865
 866             $plugin->prepare ($task, $vmid, $mode);
 867
 868             $self->run_hook_script ('backup-start', $task, $logfd);
 869
 870             if ($vmtype eq 'lxc') {
 871                 # pre-suspend rsync
 872                 $plugin->copy_data_phase1($task, $vmid);
 873             }
 874
 875             debugmsg ('info', "suspending guest", $logfd);
 876             $task->{vmstoptime} = time ();
 877             $self->run_hook_script ('pre-stop', $task, $logfd);
 878             $plugin->suspend_vm ($task, $vmid);
 879             $cleanup->{resume} = 1;
 880
 881             if ($vmtype eq 'lxc') {
 882                 # post-suspend rsync
 883                 $plugin->copy_data_phase2($task, $vmid);
 884
 885                 debugmsg ('info', "resuming guest", $logfd);
 886                 $cleanup->{resume} = 0;
 887                 $self->run_hook_script('pre-restart', $task, $logfd);
 888                 $plugin->resume_vm($task, $vmid);
 889                 $self->run_hook_script('post-restart', $task, $logfd);
 890                 $log_vm_online_again->();
 891             }
 892
 893         } elsif ($mode eq 'snapshot') {
 894
 895             $self->run_hook_script ('backup-start', $task, $logfd);
 896
 897             my $snapshot_count = $task->{snapshot_count} || 0;
 898
 899             $self->run_hook_script ('pre-stop', $task, $logfd);
 900
 901             if ($snapshot_count > 1) {
 902                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 903                 $task->{vmstoptime} = time ();
 904                 $plugin->suspend_vm ($task, $vmid);
 905                 $cleanup->{resume} = 1;
 906             }
 907
 908             $plugin->snapshot ($task, $vmid);
 909
 910             $self->run_hook_script ('pre-restart', $task, $logfd);
 911
 912             if ($snapshot_count > 1) {
 913                 debugmsg ('info', "resume vm", $logfd);
 914                 $cleanup->{resume} = 0;
 915                 $plugin->resume_vm ($task, $vmid);
 916                 $log_vm_online_again->();
 917             }
 918
 919             $self->run_hook_script ('post-restart', $task, $logfd);
 920
 921         } else {
 922             die "internal error - unknown mode '$mode'\n";
 923         }
 924
 925         # assemble archive image
 926         $plugin->assemble ($task, $vmid);
 927
 928         # produce archive
 929
 930         if ($opts->{stdout}) {
 931             debugmsg ('info', "sending archive to stdout", $logfd);
 932             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 933             $self->run_hook_script ('backup-end', $task, $logfd);
 934             return;
 935         }
 936
 937         my $archive_txt = $self->{opts}->{pbs} ? 'Proxmox Backup Server' : 'vzdump';
 938         debugmsg('info', "creating $archive_txt archive '$task->{target}'", $logfd);
 939         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 940
 941         if ($self->{opts}->{pbs}) {
 942             # size is added to task struct in guest vzdump plugins
 943         } else {
 944             rename ($task->{tmptar}, $task->{target}) ||
 945                 die "unable to rename '$task->{tmptar}' to '$task->{target}'\n";
 946
 947             # determine size
 948             $task->{size} = (-s $task->{target}) || 0;
 949             my $cs = format_size ($task->{size});
 950             debugmsg ('info', "archive file size: $cs", $logfd);
 951         }
 952
 953         # purge older backup
 954         if ($opts->{remove}) {
 955             if ($maxfiles) {
 956
 957                 if ($self->{opts}->{pbs}) {
 958                     my $args = [$pbs_group_name, '--quiet', '1', '--keep-last', $maxfiles];
 959                     my $logfunc = sub { my $line = shift; debugmsg ('info', $line, $logfd); };
 960                     PVE::Storage::PBSPlugin::run_raw_client_cmd(
 961                         $opts->{scfg}, $opts->{storage}, 'prune', $args, logfunc => $logfunc);
 962                 } else {
 963                     my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{target});
 964                     $bklist = [ sort { $b->{ctime} <=> $a->{ctime} } @$bklist ];
 965
 966                     while (scalar (@$bklist) >= $maxfiles) {
 967                         my $d = pop @$bklist;
 968                         my $archive_path = $d->{path};
 969                         debugmsg ('info', "delete old backup '$archive_path'", $logfd);
 970                         PVE::Storage::archive_remove($archive_path);
 971                     }
 972                 }
 973             } elsif (defined($prune_options)) {
 974                 my $logfunc = sub { debugmsg($_[0], $_[1], $logfd) };
 975                 PVE::Storage::prune_backups($cfg, $opts->{storage}, $prune_options, $vmid, $vmtype, 0, $logfunc);
 976             }
 977         }
 978
 979         $self->run_hook_script ('backup-end', $task, $logfd);
 980     };
 981     my $err = $@;
 982
 983     if ($plugin) {
 984         # clean-up
 985
 986         if ($cleanup->{unlock}) {
 987             eval { $plugin->unlock_vm ($vmid); };
 988             warn $@ if $@;
 989         }
 990
 991         if ($cleanup->{prepared}) {
 992             # only call cleanup when necessary (when prepare was executed)
 993             eval { $plugin->cleanup ($task, $vmid) };
 994             warn $@ if $@;
 995         }
 996
 997         eval { $plugin->set_logfd (undef); };
 998         warn $@ if $@;
 999
1000         if ($cleanup->{resume} || $cleanup->{restart}) {
1001             eval {
1002                 $self->run_hook_script ('pre-restart', $task, $logfd);
1003                 if ($cleanup->{resume}) {
1004                     debugmsg ('info', "resume vm", $logfd);
1005                     $plugin->resume_vm ($task, $vmid);
1006                 } else {
1007                     my $running = $plugin->vm_status($vmid);
1008                     if (!$running) {
1009                         debugmsg ('info', "restarting vm", $logfd);
1010                         $plugin->start_vm ($task, $vmid);
1011                     }
1012                 }
1013                 $self->run_hook_script ('post-restart', $task, $logfd);
1014             };
1015             my $err = $@;
1016             if ($err) {
1017                 warn $err;
1018             } else {
1019                 $log_vm_online_again->();
1020             }
1021         }
1022     }
1023
1024     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
1025     warn $@ if $@;
1026
1027     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
1028     warn $@ if $@;
1029
1030     my $delay = $task->{backuptime} = time () - $vmstarttime;
1031
1032     if ($err) {
1033         $task->{state} = 'err';
1034         $task->{msg} = $err;
1035         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1036         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
1037
1038         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1039
1040     } else {
1041         $task->{state} = 'ok';
1042         my $tstr = format_time ($delay);
1043         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1044         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1045     }
1046
1047     close ($logfd) if $logfd;
1048
1049     if ($task->{tmplog}) {
1050         if ($self->{opts}->{pbs}) {
1051             if ($task->{state} eq 'ok') {
1052                 my $param = [$pbs_snapshot_name, $task->{tmplog}];
1053                 PVE::Storage::PBSPlugin::run_raw_client_cmd(
1054                     $opts->{scfg}, $opts->{storage}, 'upload-log', $param, errmsg => "upload log failed");
1055             }
1056         } elsif ($task->{logfile}) {
1057             system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1058         }
1059     }
1060
1061     eval { $self->run_hook_script ('log-end', $task); };
1062
1063     die $err if $err && $err =~ m/^interrupted by signal$/;
1064 }
1065
1066 sub exec_backup {
1067     my ($self, $rpcenv, $authuser) = @_;
1068
1069     my $opts = $self->{opts};
1070
1071     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1072     debugmsg ('info', "skip external VMs: " . join(', ', @{$self->{skiplist}}))
1073         if scalar(@{$self->{skiplist}});
1074
1075     my $tasklist = [];
1076     my $vzdump_plugins =  {};
1077     foreach my $plugin (@{$self->{plugins}}) {
1078         my $type = $plugin->type();
1079         next if exists $vzdump_plugins->{$type};
1080         $vzdump_plugins->{$type} = $plugin;
1081     }
1082
1083     my $vmlist = PVE::Cluster::get_vmlist();
1084     foreach my $vmid (@{$opts->{vmids}}) {
1085         my $guest_type = $vmlist->{ids}->{$vmid}->{type};
1086         my $plugin = $vzdump_plugins->{$guest_type};
1087         next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all});
1088         push @$tasklist, {
1089             mode => $opts->{mode},
1090             plugin => $plugin,
1091             state => 'todo',
1092             vmid => $vmid,
1093         };
1094     }
1095
1096     # Use in-memory files for the outer hook logs to pass them to sendmail.
1097     my $job_start_log = '';
1098     my $job_end_log = '';
1099     open my $job_start_fd, '>', \$job_start_log;
1100     open my $job_end_fd, '>', \$job_end_log;
1101
1102     my $starttime = time();
1103     my $errcount = 0;
1104     eval {
1105
1106         $self->run_hook_script ('job-start', undef, $job_start_fd);
1107
1108         foreach my $task (@$tasklist) {
1109             $self->exec_backup_task ($task);
1110             $errcount += 1 if $task->{state} ne 'ok';
1111         }
1112
1113         $self->run_hook_script ('job-end', undef, $job_end_fd);
1114     };
1115     my $err = $@;
1116
1117     $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1118
1119     if ($err) {
1120         debugmsg ('err', "Backup job failed - $err", undef, 1);
1121     } else {
1122         if ($errcount) {
1123             debugmsg ('info', "Backup job finished with errors", undef, 1);
1124         } else {
1125             debugmsg ('info', "Backup job finished successfully", undef, 1);
1126         }
1127     }
1128
1129     close $job_start_fd;
1130     close $job_end_fd;
1131
1132     my $totaltime = time() - $starttime;
1133
1134     eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); };
1135     debugmsg ('err', $@) if $@;
1136
1137     die $err if $err;
1138
1139     die "job errors\n" if $errcount;
1140
1141     unlink $pidfile;
1142 }
1143
1144
1145 sub option_exists {
1146     my $key = shift;
1147     return defined($confdesc->{$key});
1148 }
1149
1150 sub verify_vzdump_parameters {
1151     my ($param, $check_missing) = @_;
1152
1153     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1154         if $param->{all} && $param->{vmid};
1155
1156     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1157         if $param->{exclude} && $param->{vmid};
1158
1159     raise_param_exc({ pool => "option conflicts with option 'vmid'"})
1160         if $param->{pool} && $param->{vmid};
1161
1162     raise_param_exc({ 'prune-backups' => "option conflicts with option 'maxfiles'"})
1163         if defined($param->{'prune-backups'}) && defined($param->{maxfiles});
1164
1165     $param->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $param->{'prune-backups'})
1166         if defined($param->{'prune-backups'});
1167
1168     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
1169
1170     warn "option 'size' is deprecated and will be removed in a future " .
1171          "release, please update your script/configuration!\n"
1172         if defined($param->{size});
1173
1174     return if !$check_missing;
1175
1176     raise_param_exc({ vmid => "property is missing"})
1177         if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
1178
1179 }
1180
1181 sub stop_running_backups {
1182     my($self) = @_;
1183
1184     my $upid = PVE::Tools::file_read_firstline($pidfile);
1185     return if !$upid;
1186
1187     my $task = PVE::Tools::upid_decode($upid);
1188
1189     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1190         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1191         kill(15, $task->{pid});
1192         # wait max 15 seconds to shut down (else, do nothing for now)
1193         my $i;
1194         for ($i = 15; $i > 0; $i--) {
1195             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1196             sleep (1);
1197         }
1198         die "stopping backup process $task->{pid} failed\n" if $i == 0;
1199     }
1200 }
1201
1202 sub get_included_guests {
1203     my ($job) = @_;
1204
1205     my $nodename = PVE::INotify::nodename();
1206     my $vmids = [];
1207     my $vmids_per_node = {};
1208
1209     my $vmlist = PVE::Cluster::get_vmlist();
1210
1211     if ($job->{pool}) {
1212         $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool});
1213     } elsif ($job->{vmid}) {
1214         $vmids = [ split_list($job->{vmid}) ];
1215     } elsif ($job->{all}) {
1216         # all or exclude
1217         my $exclude = check_vmids(split_list($job->{exclude}));
1218         my $excludehash = { map { $_ => 1 } @$exclude };
1219
1220         for my $id (keys %{$vmlist->{ids}}) {
1221             next if $excludehash->{$id};
1222             push @$vmids, $id;
1223         }
1224     } else {
1225         return $vmids_per_node;
1226     }
1227     $vmids = check_vmids(@$vmids);
1228
1229     for my $vmid (@$vmids) {
1230         my $node = $vmlist->{ids}->{$vmid}->{node};
1231         next if (defined $job->{node} && $job->{node} ne $node);
1232
1233         push @{$vmids_per_node->{$node}}, $vmid;
1234     }
1235
1236     return $vmids_per_node;
1237 }
1238
1239 1;