12 use POSIX
qw(strftime);
15 use PVE
::Cluster
qw(cfs_read_file);
16 use PVE
::DataCenterConfig
;
17 use PVE
::Exception
qw(raise_param_exc);
19 use PVE
::HA
::Env
::PVE2
;
20 use PVE
::JSONSchema
qw(get_standard_option);
21 use PVE
::RPCEnvironment
;
23 use PVE
::VZDump
::Common
;
24 use PVE
::VZDump
::Plugin
;
25 use PVE
::Tools
qw(extract_param split_list);
28 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
30 my $lockfile = '/var/run/vzdump.lock';
31 my $pidfile = '/var/run/vzdump.pid';
32 my $logdir = '/var/log/vzdump';
36 my $confdesc = PVE
::VZDump
::Common
::get_confdesc
();
38 # Load available plugins
39 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
40 foreach my $plug (@pve_vzdump_classes) {
41 my $filename = "/usr/share/perl5/$plug.pm";
42 $filename =~ s!::!/!g;
44 eval { require $filename; };
57 my ($mtype, $msg, $logfd, $syslog) = @_;
59 PVE
::VZDump
::Plugin
::debugmsg
(@_);
63 my ($logfd, $cmdstr, %param) = @_;
67 debugmsg
('info', $line, $logfd);
70 PVE
::Tools
::run_command
($cmdstr, %param, logfunc
=> $logfunc);
73 my $generate_notes = sub {
74 my ($notes_template, $task) = @_;
77 cluster
=> PVE
::Cluster
::get_clinfo
()->{cluster
}->{name
},
78 guestname
=> $task->{hostname
},
79 node
=> PVE
::INotify
::nodename
(),
80 vmid
=> $task->{vmid
},
85 return '\\' if $char eq '\\';
86 return "\n" if $char eq 'n';
87 die "unexpected escape character '$char'\n";
90 $notes_template =~ s/\\(.)/$unescape->($1)/eg;
92 my $vars = join('|', keys $info->%*);
93 $notes_template =~ s/\{\{($vars)\}\}/$info->{$1}/g;
95 die "unexpected variable name '$1'" if $notes_template =~ m/\{\{([^\s]+)\}\}/;
97 return $notes_template;
100 my $parse_prune_backups_maxfiles = sub {
101 my ($param, $kind) = @_;
103 my $maxfiles = delete $param->{maxfiles
};
104 my $prune_backups = $param->{'prune-backups'};
106 debugmsg
('warn', "both 'maxfiles' and 'prune-backups' defined as ${kind} - ignoring 'maxfiles'")
107 if defined($maxfiles) && defined($prune_backups);
109 if (defined($prune_backups)) {
110 return if ref($prune_backups) eq 'HASH'; # already parsed
111 $param->{'prune-backups'} = PVE
::JSONSchema
::parse_property_string
(
115 } elsif (defined($maxfiles)) {
117 $param->{'prune-backups'} = { 'keep-last' => $maxfiles };
119 $param->{'prune-backups'} = { 'keep-all' => 1 };
127 my $cfg = PVE
::Storage
::config
();
128 my $scfg = PVE
::Storage
::storage_config
($cfg, $storage);
129 my $type = $scfg->{type
};
131 die "can't use storage '$storage' for backups - wrong content type\n"
132 if (!$scfg->{content
}->{backup
});
138 $info->{'prune-backups'} = PVE
::JSONSchema
::parse_property_string
('prune-backups', $scfg->{'prune-backups'})
139 if defined($scfg->{'prune-backups'});
141 if ($type eq 'pbs') {
144 $info->{dumpdir
} = PVE
::Storage
::get_backup_dir
($cfg, $storage);
153 my $kb = $size / 1024;
156 return int ($kb) . "KB";
159 my $mb = $size / (1024*1024);
161 return int ($mb) . "MB";
165 return sprintf ("%.2fGB", $gb);
168 return sprintf ("%.2fTB", $tb);
174 my $hours = int ($seconds/3600);
175 $seconds = $seconds - $hours*3600;
176 my $min = int ($seconds/60);
177 $seconds = $seconds - $min*60;
179 return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
185 $str =~ s/^(.{990})/$1\n/mg; # reduce line length
203 foreach my $p (split (/:/, $ENV{PATH
})) {
210 die "unable to find command '$bin'\n";
217 for my $vmid (sort {$a <=> $b} @vmids) {
218 die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
219 $vmid = int ($vmid); # remove leading zeros
228 sub read_vzdump_defaults
{
230 my $fn = "/etc/vzdump.conf";
234 my $default = $confdesc->{$_}->{default};
235 defined($default) ?
($_ => $default) : ()
238 $parse_prune_backups_maxfiles->($defaults, "defaults in VZDump schema");
241 eval { $raw = PVE
::Tools
::file_get_contents
($fn); };
242 return $defaults if $@;
244 my $conf_schema = { type
=> 'object', properties
=> $confdesc, };
245 my $res = PVE
::JSONSchema
::parse_config
($conf_schema, $fn, $raw);
246 if (my $excludes = $res->{'exclude-path'}) {
247 $res->{'exclude-path'} = PVE
::Tools
::split_args
($excludes);
249 if (defined($res->{mailto
})) {
250 my @mailto = split_list
($res->{mailto
});
251 $res->{mailto
} = [ @mailto ];
253 $parse_prune_backups_maxfiles->($res, "options in '$fn'");
255 foreach my $key (keys %$defaults) {
256 $res->{$key} = $defaults->{$key} if !defined($res->{$key});
259 if (defined($res->{storage
}) && defined($res->{dumpdir
})) {
260 debugmsg
('warn', "both 'storage' and 'dumpdir' defined in '$fn' - ignoring 'dumpdir'");
261 delete $res->{dumpdir
};
267 use constant MAX_MAIL_SIZE
=> 1024*1024;
269 my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
271 my $opts = $self->{opts
};
273 my $mailto = $opts->{mailto
};
275 return if !($mailto && scalar(@$mailto));
277 my $cmdline = $self->{cmdline
};
280 foreach my $task (@$tasklist) {
281 $ecount++ if $task->{state} ne 'ok';
282 chomp $task->{msg
} if $task->{msg
};
283 $task->{backuptime
} = 0 if !$task->{backuptime
};
284 $task->{size
} = 0 if !$task->{size
};
285 $task->{target
} = 'unknown' if !$task->{target
};
286 $task->{hostname
} = "VM $task->{vmid}" if !$task->{hostname
};
288 if ($task->{state} eq 'todo') {
289 $task->{msg
} = 'aborted';
293 my $notify = $opts->{mailnotification
} || 'always';
294 return if (!$ecount && !$err && ($notify eq 'failure'));
296 my $stat = ($ecount || $err) ?
'backup failed' : 'backup successful';
299 $stat .= ": multiple problems";
306 my $hostname = `hostname -f` || PVE
::INotify
::nodename
();
310 my $text = $err ?
"$err\n\n" : '';
313 "%-10s %-${namelength}s %-6s %10s %10s %s\n",
314 qw(VMID NAME STATUS TIME SIZE FILENAME)
316 foreach my $task (@$tasklist) {
317 my $name = substr($task->{hostname
}, 0, $namelength);
318 my $successful = $task->{state} eq 'ok';
319 my $size = $successful ? format_size
($task->{size
}) : 0;
320 my $filename = $successful ?
$task->{target
} : '-';
321 my $size_fmt = $successful ?
"%10s": "%8.2fMB";
323 "%-10s %-${namelength}s %-6s %10s $size_fmt %s\n",
327 format_time
($task->{backuptime
}),
334 $text_log_part .= "\nDetailed backup logs:\n\n";
335 $text_log_part .= "$cmdline\n\n";
337 $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
338 foreach my $task (@$tasklist) {
339 my $vmid = $task->{vmid
};
340 my $log = $task->{tmplog
};
342 $text_log_part .= "$vmid: no log available\n\n";
345 if (open (my $TMP, '<', "$log")) {
346 while (my $line = <$TMP>) {
347 next if $line =~ /^status: \d+/; # not useful in mails
348 $text_log_part .= encode8bit
("$vmid: $line");
352 $text_log_part .= "$vmid: Could not open log file\n\n";
354 $text_log_part .= "\n";
356 $text_log_part .= $detail_post if defined($detail_post);
359 my $html = "<html><body>\n";
360 $html .= "<p>" . (escape_html
($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
361 $html .= "<table border=1 cellpadding=3>\n";
362 $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
365 foreach my $task (@$tasklist) {
366 my $vmid = $task->{vmid
};
367 my $name = $task->{hostname
};
369 if ($task->{state} eq 'ok') {
370 $ssize += $task->{size
};
373 "<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
376 format_time
($task->{backuptime
}),
377 format_size
($task->{size
}),
378 escape_html
($task->{target
}),
382 "<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
385 format_time
($task->{backuptime
}),
386 escape_html
($task->{msg
}),
391 $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
392 format_time
($totaltime), format_size
($ssize));
394 $html .= "\n</table><br><br>\n";
396 $html_log_part .= "Detailed backup logs:<br /><br />\n";
397 $html_log_part .= "<pre>\n";
398 $html_log_part .= escape_html
($cmdline) . "\n\n";
400 $html_log_part .= escape_html
($detail_pre) . "\n" if defined($detail_pre);
401 foreach my $task (@$tasklist) {
402 my $vmid = $task->{vmid
};
403 my $log = $task->{tmplog
};
405 $html_log_part .= "$vmid: no log available\n\n";
408 if (open (my $TMP, '<', "$log")) {
409 while (my $line = <$TMP>) {
410 next if $line =~ /^status: \d+/; # not useful in mails
411 if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
412 $html_log_part .= encode8bit
("$vmid: <font color=red>".
413 escape_html
($line) . "</font>");
415 $html_log_part .= encode8bit
("$vmid: " . escape_html
($line));
420 $html_log_part .= "$vmid: Could not open log file\n\n";
422 $html_log_part .= "\n";
424 $html_log_part .= escape_html
($detail_post) if defined($detail_post);
425 $html_log_part .= "</pre>";
426 my $html_end = "\n</body></html>\n";
429 if (length($text) + length($text_log_part) +
430 length($html) + length($html_log_part) +
431 length($html_end) < MAX_MAIL_SIZE
)
433 $html .= $html_log_part;
435 $text .= $text_log_part;
437 my $msg = "Log output was too long to be sent by mail. ".
438 "See Task History for details!\n";
440 $html .= "<p>$msg</p>";
444 my $subject = "vzdump backup status ($hostname) : $stat";
446 my $dcconf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
447 my $mailfrom = $dcconf->{email_from
} || "root";
449 PVE
::Tools
::sendmail
($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
453 my ($class, $cmdline, $opts, $skiplist) = @_;
459 check_bin
('sendmail');
463 check_bin
('umount');
464 check_bin
('cstream');
465 check_bin
('ionice');
467 if ($opts->{mode
} && $opts->{mode
} eq 'snapshot') {
468 check_bin
('lvcreate');
470 check_bin
('lvremove');
473 my $defaults = read_vzdump_defaults
();
475 foreach my $k (keys %$defaults) {
476 next if $k eq 'exclude-path' || $k eq 'prune-backups'; # dealt with separately
477 if ($k eq 'dumpdir' || $k eq 'storage') {
478 $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir
}) &&
479 !defined ($opts->{storage
});
481 $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
485 $opts->{dumpdir
} =~ s
|/+$|| if ($opts->{dumpdir
});
486 $opts->{tmpdir
} =~ s
|/+$|| if ($opts->{tmpdir
});
488 $skiplist = [] if !$skiplist;
492 skiplist
=> $skiplist,
495 my $findexcl = $self->{findexcl
} = [];
496 if ($defaults->{'exclude-path'}) {
497 push @$findexcl, @{$defaults->{'exclude-path'}};
500 if ($opts->{'exclude-path'}) {
501 push @$findexcl, @{$opts->{'exclude-path'}};
504 if ($opts->{stdexcludes
}) {
512 foreach my $p (@plugins) {
513 my $pd = $p->new($self);
515 push @{$self->{plugins
}}, $pd;
518 if (defined($opts->{storage
}) && $opts->{stdout
}) {
519 die "cannot use options 'storage' and 'stdout' at the same time\n";
520 } elsif (defined($opts->{storage
}) && defined($opts->{dumpdir
})) {
521 die "cannot use options 'storage' and 'dumpdir' at the same time\n";
524 if (!$opts->{dumpdir
} && !$opts->{storage
}) {
525 $opts->{storage
} = 'local';
529 my $add_error = sub {
531 $errors .= "\n" if $errors;
537 $self->{job_init_log
} = '';
538 open my $job_init_fd, '>', \
$self->{job_init_log
};
539 $self->run_hook_script('job-init', undef, $job_init_fd);
542 PVE
::Cluster
::cfs_update
(); # Pick up possible changes made by the hook script.
544 $add_error->($@) if $@;
546 if ($opts->{storage
}) {
547 my $storage_cfg = PVE
::Storage
::config
();
548 eval { PVE
::Storage
::activate_storage
($storage_cfg, $opts->{storage
}) };
549 $add_error->("could not activate storage '$opts->{storage}': $@") if $@;
551 my $info = eval { storage_info
($opts->{storage
}) };
553 $add_error->("could not get storage information for '$opts->{storage}': $err");
555 $opts->{dumpdir
} = $info->{dumpdir
};
556 $opts->{scfg
} = $info->{scfg
};
557 $opts->{pbs
} = $info->{pbs
};
558 $opts->{'prune-backups'} //= $info->{'prune-backups'};
560 } elsif ($opts->{dumpdir
}) {
561 $add_error->("dumpdir '$opts->{dumpdir}' does not exist")
562 if ! -d
$opts->{dumpdir
};
564 die "internal error";
567 $opts->{'prune-backups'} //= $defaults->{'prune-backups'};
569 # avoid triggering any remove code path if keep-all is set
570 $opts->{remove
} = 0 if $opts->{'prune-backups'}->{'keep-all'};
572 if ($opts->{tmpdir
} && ! -d
$opts->{tmpdir
}) {
573 $add_error->("tmpdir '$opts->{tmpdir}' does not exist");
577 eval { $self->sendmail([], 0, $errors); };
578 debugmsg
('err', $@) if $@;
588 # Note: df 'available' can be negative, and percentage set to '-'
590 my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
596 if (my ($fsid, $fstype, undef, $mp) = $line =~
597 m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
606 eval { PVE
::Tools
::run_command
($cmd, errfunc
=> sub {}, outfunc
=> $parser); };
613 my ($self, $upid) = @_;
617 my $maxwait = $self->{opts
}->{lockwait
} || $self->{lockwait
};
619 die "missing UPID" if !$upid; # should not happen
622 if (!open ($SERVER_FLCK, '>>', "$lockfile")) {
623 debugmsg
('err', "can't open lock on file '$lockfile' - $!", undef, 1);
624 die "can't open lock on file '$lockfile' - $!";
627 if (!flock ($SERVER_FLCK, LOCK_EX
|LOCK_NB
)) {
629 debugmsg
('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
630 die "can't acquire lock '$lockfile' (wait = 0)";
633 debugmsg
('info', "trying to get global lock - waiting...", undef, 1);
635 alarm ($maxwait * 60);
637 local $SIG{ALRM
} = sub { alarm (0); die "got timeout\n"; };
639 if (!flock ($SERVER_FLCK, LOCK_EX
)) {
641 close ($SERVER_FLCK);
652 debugmsg
('err', "can't acquire lock '$lockfile' - $err", undef, 1);
653 die "can't acquire lock '$lockfile' - $err";
656 debugmsg
('info', "got global lock", undef, 1);
659 PVE
::Tools
::file_set_contents
($pidfile, $upid);
664 sub run_hook_script
{
665 my ($self, $phase, $task, $logfd) = @_;
667 my $opts = $self->{opts
};
669 my $script = $opts->{script
};
672 die "Error: The hook script '$script' does not exist.\n" if ! -f
$script;
673 die "Error: The hook script '$script' is not executable.\n" if ! -x
$script;
675 my $cmd = [$script, $phase];
678 push @$cmd, $task->{mode
};
679 push @$cmd, $task->{vmid
};
683 # set immutable opts directly (so they are available in all phases)
684 $ENV{STOREID
} = $opts->{storage
} if $opts->{storage
};
685 $ENV{DUMPDIR
} = $opts->{dumpdir
} if $opts->{dumpdir
};
687 foreach my $ek (qw(vmtype hostname target logfile)) {
688 $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
691 run_command
($logfd, $cmd);
694 sub compressor_info
{
696 my $opt_compress = $opts->{compress
};
698 if (!$opt_compress || $opt_compress eq '0') {
700 } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
701 return ('lzop', 'lzo');
702 } elsif ($opt_compress eq 'gzip') {
703 if ($opts->{pigz
} > 0) {
704 my $pigz_threads = $opts->{pigz
};
705 if ($pigz_threads == 1) {
706 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
707 $pigz_threads = int(($cpuinfo->{cpus
} + 1)/2);
709 return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
711 return ('gzip --rsyncable', 'gz');
713 } elsif ($opt_compress eq 'zstd') {
714 my $zstd_threads = $opts->{zstd
} // 1;
715 if ($zstd_threads == 0) {
716 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
717 $zstd_threads = int(($cpuinfo->{cpus
} + 1)/2);
719 return ("zstd --rsyncable --threads=${zstd_threads}", 'zst');
721 die "internal error - unknown compression option '$opt_compress'";
725 sub get_backup_file_list
{
726 my ($dir, $bkname) = @_;
729 foreach my $fn (<$dir/${bkname
}-*>) {
730 my $archive_info = eval { PVE
::Storage
::archive_info
($fn) } // {};
731 if ($archive_info->{is_std_name
}) {
732 my $path = "$dir/$archive_info->{filename}";
735 'ctime' => $archive_info->{ctime
},
737 $backup->{mark
} = "protected"
738 if -e PVE
::Storage
::protection_file_path
($path);
739 push @{$bklist}, $backup;
746 sub exec_backup_task
{
747 my ($self, $task) = @_;
749 my $opts = $self->{opts
};
751 my $cfg = PVE
::Storage
::config
();
752 my $vmid = $task->{vmid
};
753 my $plugin = $task->{plugin
};
755 $task->{backup_time
} = time();
758 my $pbs_snapshot_name;
760 my $vmstarttime = time ();
766 my $log_vm_online_again = sub {
767 return if !defined($task->{vmstoptime
});
768 $task->{vmconttime
} //= time();
769 my $delay = $task->{vmconttime
} - $task->{vmstoptime
};
770 $delay = '<1' if $delay < 1;
771 debugmsg
('info', "guest is online again after $delay seconds", $logfd);
775 die "unable to find VM '$vmid'\n" if !$plugin;
777 my $vmtype = $plugin->type();
779 if ($self->{opts
}->{pbs
}) {
780 if ($vmtype eq 'lxc') {
781 $pbs_group_name = "ct/$vmid";
782 } elsif ($vmtype eq 'qemu') {
783 $pbs_group_name = "vm/$vmid";
785 die "pbs backup not implemented for plugin type '$vmtype'\n";
787 my $btime = strftime
("%FT%TZ", gmtime($task->{backup_time
}));
788 $pbs_snapshot_name = "$pbs_group_name/$btime";
791 # for now we deny backups of a running ha managed service in *stop* mode
792 # as it interferes with the HA stack (started services should not stop).
793 if ($opts->{mode
} eq 'stop' &&
794 PVE
::HA
::Config
::vm_is_ha_managed
($vmid, 'started'))
796 die "Cannot execute a backup with stop mode on a HA managed and".
797 " enabled Service. Use snapshot mode or disable the Service.\n";
800 my $tmplog = "$logdir/$vmtype-$vmid.log";
802 my $bkname = "vzdump-$vmtype-$vmid";
803 my $basename = $bkname . strftime
("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time
}));
805 my $prune_options = $opts->{'prune-backups'};
807 my $backup_limit = 0;
808 if (!$prune_options->{'keep-all'}) {
809 foreach my $keep (values %{$prune_options}) {
810 $backup_limit += $keep;
814 if (($backup_limit && !$opts->{remove
}) || $opts->{protected
}) {
817 if (my $storeid = $opts->{storage
}) {
819 !$_->{subtype
} || $_->{subtype
} eq $vmtype
820 } PVE
::Storage
::volume_list
($cfg, $storeid, $vmid, 'backup')->@*;
822 $count = grep { !$_->{protected
} } @backups;
823 $protected_count = scalar(@backups) - $count;
825 $count = grep { !$_->{mark
} || $_->{mark
} ne "protected" } get_backup_file_list
($opts->{dumpdir
}, $bkname)->@*;
828 if ($opts->{protected
}) {
829 my $max_protected = PVE
::Storage
::get_max_protected_backups
(
833 if ($max_protected > -1 && $protected_count >= $max_protected) {
834 die "The number of protected backups per guest is limited to $max_protected ".
835 "on storage '$opts->{storage}'\n";
837 } elsif ($count >= $backup_limit) {
838 die "There is a max backup limit of $backup_limit enforced by the target storage ".
839 "or the vzdump parameters. Either increase the limit or delete old backups.\n";
843 if (!$self->{opts
}->{pbs
}) {
844 $task->{logfile
} = "$opts->{dumpdir}/$basename.log";
847 my $ext = $vmtype eq 'qemu' ?
'.vma' : '.tar';
848 my ($comp, $comp_ext) = compressor_info
($opts);
849 if ($comp && $comp_ext) {
850 $ext .= ".${comp_ext}";
853 if ($self->{opts
}->{pbs
}) {
854 die "unable to pipe backup to stdout\n" if $opts->{stdout
};
855 $task->{target
} = $pbs_snapshot_name;
857 if ($opts->{stdout
}) {
858 $task->{target
} = '-';
860 $task->{target
} = $task->{tmptar
} = "$opts->{dumpdir}/$basename$ext";
861 $task->{tmptar
} =~ s/\.[^\.]+$/\.dat/;
862 unlink $task->{tmptar
};
866 $task->{vmtype
} = $vmtype;
869 if ($opts->{tmpdir
}) {
870 $task->{tmpdir
} = "$opts->{tmpdir}/vzdumptmp${pid}_$vmid/";
871 } elsif ($self->{opts
}->{pbs
}) {
872 $task->{tmpdir
} = "/var/tmp/vzdumptmp${pid}_$vmid";
874 # dumpdir is posix? then use it as temporary dir
875 my $info = get_mount_info
($opts->{dumpdir
});
876 if ($vmtype eq 'qemu' ||
877 grep ($_ eq $info->{fstype
}, @posix_filesystems)) {
878 $task->{tmpdir
} = "$opts->{dumpdir}/$basename.tmp";
880 $task->{tmpdir
} = "/var/tmp/vzdumptmp${pid}_$vmid";
881 debugmsg
('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
882 "using $task->{tmpdir} for temporary files", $logfd);
886 rmtree
$task->{tmpdir
};
887 mkdir $task->{tmpdir
};
888 -d
$task->{tmpdir
} ||
889 die "unable to create temporary directory '$task->{tmpdir}'";
891 $logfd = IO
::File-
>new (">$tmplog") ||
892 die "unable to create log file '$tmplog'";
894 $task->{dumpdir
} = $opts->{dumpdir
};
895 $task->{storeid
} = $opts->{storage
};
896 $task->{scfg
} = $opts->{scfg
};
897 $task->{tmplog
} = $tmplog;
899 unlink $task->{logfile
} if defined($task->{logfile
});
901 debugmsg
('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
902 debugmsg
('info', "Backup started at " . strftime
("%F %H:%M:%S", localtime()));
904 $plugin->set_logfd ($logfd);
906 # test is VM is running
907 my ($running, $status_text) = $plugin->vm_status ($vmid);
909 debugmsg
('info', "status = ${status_text}", $logfd);
911 # lock VM (prevent config changes)
912 $plugin->lock_vm ($vmid);
914 $cleanup->{unlock
} = 1;
918 my $mode = $running ?
$task->{mode
} : 'stop';
920 if ($mode eq 'snapshot') {
921 my %saved_task = %$task;
922 eval { $plugin->prepare ($task, $vmid, $mode); };
924 die $err if $err !~ m/^mode failure/;
925 debugmsg
('info', $err, $logfd);
926 debugmsg
('info', "trying 'suspend' mode instead", $logfd);
927 $mode = 'suspend'; # so prepare is called again below
928 %$task = %saved_task;
932 $cleanup->{prepared
} = 1;
934 $task->{mode
} = $mode;
936 debugmsg
('info', "backup mode: $mode", $logfd);
937 debugmsg
('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd) if $opts->{bwlimit
};
938 debugmsg
('info', "ionice priority: $opts->{ionice}", $logfd);
940 if ($mode eq 'stop') {
941 $plugin->prepare ($task, $vmid, $mode);
943 $self->run_hook_script ('backup-start', $task, $logfd);
946 debugmsg
('info', "stopping virtual guest", $logfd);
947 $task->{vmstoptime
} = time();
948 $self->run_hook_script ('pre-stop', $task, $logfd);
949 $plugin->stop_vm ($task, $vmid);
950 $cleanup->{restart
} = 1;
954 } elsif ($mode eq 'suspend') {
955 $plugin->prepare ($task, $vmid, $mode);
957 $self->run_hook_script ('backup-start', $task, $logfd);
959 if ($vmtype eq 'lxc') {
961 $plugin->copy_data_phase1($task, $vmid);
964 debugmsg
('info', "suspending guest", $logfd);
965 $task->{vmstoptime
} = time ();
966 $self->run_hook_script ('pre-stop', $task, $logfd);
967 $plugin->suspend_vm ($task, $vmid);
968 $cleanup->{resume
} = 1;
970 if ($vmtype eq 'lxc') {
972 $plugin->copy_data_phase2($task, $vmid);
974 debugmsg
('info', "resuming guest", $logfd);
975 $cleanup->{resume
} = 0;
976 $self->run_hook_script('pre-restart', $task, $logfd);
977 $plugin->resume_vm($task, $vmid);
978 $self->run_hook_script('post-restart', $task, $logfd);
979 $log_vm_online_again->();
982 } elsif ($mode eq 'snapshot') {
983 $self->run_hook_script ('backup-start', $task, $logfd);
985 my $snapshot_count = $task->{snapshot_count
} || 0;
987 $self->run_hook_script ('pre-stop', $task, $logfd);
989 if ($snapshot_count > 1) {
990 debugmsg
('info', "suspend vm to make snapshot", $logfd);
991 $task->{vmstoptime
} = time ();
992 $plugin->suspend_vm ($task, $vmid);
993 $cleanup->{resume
} = 1;
996 $plugin->snapshot ($task, $vmid);
998 $self->run_hook_script ('pre-restart', $task, $logfd);
1000 if ($snapshot_count > 1) {
1001 debugmsg
('info', "resume vm", $logfd);
1002 $cleanup->{resume
} = 0;
1003 $plugin->resume_vm ($task, $vmid);
1004 $log_vm_online_again->();
1007 $self->run_hook_script ('post-restart', $task, $logfd);
1010 die "internal error - unknown mode '$mode'\n";
1013 # assemble archive image
1014 $plugin->assemble ($task, $vmid);
1018 if ($opts->{stdout
}) {
1019 debugmsg
('info', "sending archive to stdout", $logfd);
1020 $plugin->archive($task, $vmid, $task->{tmptar
}, $comp);
1021 $self->run_hook_script ('backup-end', $task, $logfd);
1025 my $archive_txt = $self->{opts
}->{pbs
} ?
'Proxmox Backup Server' : 'vzdump';
1026 debugmsg
('info', "creating $archive_txt archive '$task->{target}'", $logfd);
1027 $plugin->archive($task, $vmid, $task->{tmptar
}, $comp);
1029 if ($self->{opts
}->{pbs
}) {
1030 # size is added to task struct in guest vzdump plugins
1032 rename ($task->{tmptar
}, $task->{target
}) ||
1033 die "unable to rename '$task->{tmptar}' to '$task->{target}'\n";
1036 $task->{size
} = (-s
$task->{target
}) || 0;
1037 my $cs = format_size
($task->{size
});
1038 debugmsg
('info', "archive file size: $cs", $logfd);
1041 # Mark as protected before pruning.
1042 if (my $storeid = $opts->{storage
}) {
1043 my $volname = $opts->{pbs
} ?
$task->{target
} : basename
($task->{target
});
1044 my $volid = "${storeid}:backup/${volname}";
1046 if ($opts->{'notes-template'} && $opts->{'notes-template'} ne '') {
1047 debugmsg
('info', "adding notes to backup", $logfd);
1048 my $notes = eval { $generate_notes->($opts->{'notes-template'}, $task); };
1050 debugmsg
('warn', "unable to add notes - $err", $logfd);
1052 eval { PVE
::Storage
::update_volume_attribute
($cfg, $volid, 'notes', $notes) };
1053 debugmsg
('warn', "unable to add notes - $@", $logfd) if $@;
1057 if ($opts->{protected
}) {
1058 debugmsg
('info', "marking backup as protected", $logfd);
1059 eval { PVE
::Storage
::update_volume_attribute
($cfg, $volid, 'protected', 1) };
1060 die "unable to set protected flag - $@\n" if $@;
1064 if ($opts->{remove
}) {
1065 my $keepstr = join(', ', map { "$_=$prune_options->{$_}" } sort keys %$prune_options);
1066 debugmsg
('info', "prune older backups with retention: $keepstr", $logfd);
1068 if (!defined($opts->{storage
})) {
1069 my $bklist = get_backup_file_list
($opts->{dumpdir
}, $bkname);
1071 PVE
::Storage
::prune_mark_backup_group
($bklist, $prune_options);
1073 foreach my $prune_entry (@{$bklist}) {
1074 next if $prune_entry->{mark
} ne 'remove';
1076 my $archive_path = $prune_entry->{path
};
1077 debugmsg
('info', "delete old backup '$archive_path'", $logfd);
1078 PVE
::Storage
::archive_remove
($archive_path);
1081 my $pruned_list = PVE
::Storage
::prune_backups
(
1088 sub { debugmsg
($_[0], $_[1], $logfd) },
1090 $pruned = scalar(grep { $_->{mark
} eq 'remove' } $pruned_list->@*);
1092 my $log_pruned_extra = $pruned > 0 ?
" not covered by keep-retention policy" : "";
1093 debugmsg
('info', "pruned $pruned backup(s)${log_pruned_extra}", $logfd);
1096 $self->run_hook_script ('backup-end', $task, $logfd);
1103 if ($cleanup->{unlock
}) {
1104 eval { $plugin->unlock_vm ($vmid); };
1108 if ($cleanup->{prepared
}) {
1109 # only call cleanup when necessary (when prepare was executed)
1110 eval { $plugin->cleanup ($task, $vmid) };
1114 eval { $plugin->set_logfd (undef); };
1117 if ($cleanup->{resume
} || $cleanup->{restart
}) {
1119 $self->run_hook_script ('pre-restart', $task, $logfd);
1120 if ($cleanup->{resume
}) {
1121 debugmsg
('info', "resume vm", $logfd);
1122 $plugin->resume_vm ($task, $vmid);
1124 my $running = $plugin->vm_status($vmid);
1126 debugmsg
('info', "restarting vm", $logfd);
1127 $plugin->start_vm ($task, $vmid);
1130 $self->run_hook_script ('post-restart', $task, $logfd);
1136 $log_vm_online_again->();
1141 eval { unlink $task->{tmptar
} if $task->{tmptar
} && -f
$task->{tmptar
}; };
1144 eval { rmtree
$task->{tmpdir
} if $task->{tmpdir
} && -d
$task->{tmpdir
}; };
1147 my $delay = $task->{backuptime
} = time () - $vmstarttime;
1150 $task->{state} = 'err';
1151 $task->{msg
} = $err;
1152 debugmsg
('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1153 debugmsg
('info', "Failed at " . strftime
("%F %H:%M:%S", localtime()));
1155 eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1158 $task->{state} = 'ok';
1159 my $tstr = format_time
($delay);
1160 debugmsg
('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1161 debugmsg
('info', "Backup finished at " . strftime
("%F %H:%M:%S", localtime()));
1164 close ($logfd) if $logfd;
1166 if ($task->{tmplog
}) {
1167 if ($self->{opts
}->{pbs
}) {
1168 if ($task->{state} eq 'ok') {
1170 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
(
1174 [ $pbs_snapshot_name, $task->{tmplog
} ],
1175 errmsg
=> "uploading backup task log failed",
1179 debugmsg
('warn', "$@") if $@; # $@ contains already error prefix
1181 } elsif ($task->{logfile
}) {
1182 system {'cp'} 'cp', $task->{tmplog
}, $task->{logfile
};
1186 eval { $self->run_hook_script ('log-end', $task); };
1188 die $err if $err && $err =~ m/^interrupted by signal$/;
1192 my ($self, $rpcenv, $authuser) = @_;
1194 my $opts = $self->{opts
};
1196 debugmsg
('info', "starting new backup job: $self->{cmdline}", undef, 1);
1198 if (scalar(@{$self->{skiplist
}})) {
1199 my $skip_string = join(', ', sort { $a <=> $b } @{$self->{skiplist
}});
1200 debugmsg
('info', "skip external VMs: $skip_string");
1204 my $vzdump_plugins = {};
1205 foreach my $plugin (@{$self->{plugins
}}) {
1206 my $type = $plugin->type();
1207 next if exists $vzdump_plugins->{$type};
1208 $vzdump_plugins->{$type} = $plugin;
1211 my $vmlist = PVE
::Cluster
::get_vmlist
();
1212 my $vmids = [ sort { $a <=> $b } @{$opts->{vmids
}} ];
1213 foreach my $vmid (@{$vmids}) {
1215 if (defined($vmlist->{ids
}->{$vmid})) {
1216 my $guest_type = $vmlist->{ids
}->{$vmid}->{type
};
1217 $plugin = $vzdump_plugins->{$guest_type};
1218 next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all
});
1221 mode
=> $opts->{mode
},
1228 # Use in-memory files for the outer hook logs to pass them to sendmail.
1229 my $job_start_log = '';
1230 my $job_end_log = '';
1231 open my $job_start_fd, '>', \
$job_start_log;
1232 open my $job_end_fd, '>', \
$job_end_log;
1234 my $starttime = time();
1238 $self->run_hook_script ('job-start', undef, $job_start_fd);
1240 foreach my $task (@$tasklist) {
1241 $self->exec_backup_task ($task);
1242 $errcount += 1 if $task->{state} ne 'ok';
1245 $self->run_hook_script ('job-end', undef, $job_end_fd);
1249 $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1252 debugmsg
('err', "Backup job failed - $err", undef, 1);
1255 debugmsg
('info', "Backup job finished with errors", undef, 1);
1257 debugmsg
('info', "Backup job finished successfully", undef, 1);
1261 close $job_start_fd;
1264 my $totaltime = time() - $starttime;
1271 $self->{job_init_log
} . $job_start_log,
1275 debugmsg
('err', $@) if $@;
1279 die "job errors\n" if $errcount;
1287 return defined($confdesc->{$key});
1290 # NOTE it might make sense to merge this and verify_vzdump_parameters(), but one
1291 # needs to adapt command_line() in guest-common's PVE/VZDump/Common.pm and detect
1292 # a second parsing attempt, because verify_vzdump_parameters() is called twice
1293 # during the update_job API call.
1294 sub parse_mailto_exclude_path
{
1297 # exclude-path list need to be 0 separated
1298 if (defined($param->{'exclude-path'})) {
1299 my @expaths = split(/\0/, $param->{'exclude-path'} || '');
1300 $param->{'exclude-path'} = [ @expaths ];
1303 if (defined($param->{mailto
})) {
1304 my @mailto = PVE
::Tools
::split_list
(extract_param
($param, 'mailto'));
1305 $param->{mailto
} = [ @mailto ];
1311 sub verify_vzdump_parameters
{
1312 my ($param, $check_missing) = @_;
1314 raise_param_exc
({ all
=> "option conflicts with option 'vmid'"})
1315 if $param->{all
} && $param->{vmid
};
1317 raise_param_exc
({ exclude
=> "option conflicts with option 'vmid'"})
1318 if $param->{exclude
} && $param->{vmid
};
1320 raise_param_exc
({ pool
=> "option conflicts with option 'vmid'"})
1321 if $param->{pool
} && $param->{vmid
};
1323 raise_param_exc
({ 'prune-backups' => "option conflicts with option 'maxfiles'"})
1324 if defined($param->{'prune-backups'}) && defined($param->{maxfiles
});
1326 $parse_prune_backups_maxfiles->($param, 'CLI parameters');
1328 raise_param_exc
({'notes-template' => "contains a line feed"})
1329 if $param->{'notes-template'} && $param->{'notes-template'} =~ m/\n/;
1331 $param->{all
} = 1 if (defined($param->{exclude
}) && !$param->{pool
});
1333 return if !$check_missing;
1335 raise_param_exc
({ vmid
=> "property is missing"})
1336 if !($param->{all
} || $param->{stop
} || $param->{pool
}) && !$param->{vmid
};
1340 sub stop_running_backups
{
1343 my $upid = PVE
::Tools
::file_read_firstline
($pidfile);
1346 my $task = PVE
::Tools
::upid_decode
($upid);
1348 if (PVE
::ProcFSTools
::check_process_running
($task->{pid
}, $task->{pstart
}) &&
1349 PVE
::ProcFSTools
::read_proc_starttime
($task->{pid
}) == $task->{pstart
}) {
1350 kill(15, $task->{pid
});
1351 # wait max 15 seconds to shut down (else, do nothing for now)
1353 for ($i = 15; $i > 0; $i--) {
1354 last if !PVE
::ProcFSTools
::check_process_running
(($task->{pid
}, $task->{pstart
}));
1357 die "stopping backup process $task->{pid} failed\n" if $i == 0;
1361 sub get_included_guests
{
1365 my $vmids_per_node = {};
1367 my $vmlist = PVE
::Cluster
::get_vmlist
();
1370 $vmids = PVE
::API2Tools
::get_resource_pool_guest_members
($job->{pool
});
1371 } elsif ($job->{vmid
}) {
1372 $vmids = [ split_list
($job->{vmid
}) ];
1373 } elsif ($job->{all
}) {
1375 my $exclude = check_vmids
(split_list
($job->{exclude
}));
1376 my $excludehash = { map { $_ => 1 } @$exclude };
1378 for my $id (keys %{$vmlist->{ids
}}) {
1379 next if $excludehash->{$id};
1383 return $vmids_per_node;
1385 $vmids = check_vmids
(@$vmids);
1387 for my $vmid (@$vmids) {
1388 if (defined($vmlist->{ids
}->{$vmid})) {
1389 my $node = $vmlist->{ids
}->{$vmid}->{node
};
1390 next if (defined $job->{node
} && $job->{node
} ne $node);
1392 push @{$vmids_per_node->{$node}}, $vmid;
1394 push @{$vmids_per_node->{''}}, $vmid;
1398 return $vmids_per_node;