13 use POSIX
qw(strftime);
16 use PVE
::Cluster
qw(cfs_read_file);
17 use PVE
::DataCenterConfig
;
18 use PVE
::Exception
qw(raise_param_exc);
20 use PVE
::HA
::Env
::PVE2
;
21 use PVE
::JSONSchema
qw(get_standard_option);
23 use PVE
::RPCEnvironment
;
25 use PVE
::VZDump
::Common
;
26 use PVE
::VZDump
::Plugin
;
27 use PVE
::Tools
qw(extract_param split_list);
30 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
32 my $lockfile = '/var/run/vzdump.lock';
33 my $pidfile = '/var/run/vzdump.pid';
34 my $logdir = '/var/log/vzdump';
38 my $confdesc = PVE
::VZDump
::Common
::get_confdesc
();
40 my $confdesc_for_defaults = Clone
::clone
($confdesc);
41 delete $confdesc_for_defaults->{$_}->{requires
} for qw(notes-template protected);
43 # Load available plugins
44 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
45 foreach my $plug (@pve_vzdump_classes) {
46 my $filename = "/usr/share/perl5/$plug.pm";
47 $filename =~ s!::!/!g;
49 eval { require $filename; };
59 sub get_storage_param
{
62 return if $param->{dumpdir
};
63 return $param->{storage
} || 'local';
69 my ($mtype, $msg, $logfd, $syslog) = @_;
71 PVE
::VZDump
::Plugin
::debugmsg
(@_);
75 my ($logfd, $cmdstr, %param) = @_;
79 debugmsg
('info', $line, $logfd);
82 PVE
::Tools
::run_command
($cmdstr, %param, logfunc
=> $logfunc);
85 my $verify_notes_template = sub {
88 die "contains a line feed\n" if $template =~ /\n/;
91 while ($template =~ /\\(.)/g) {
93 push @problematic, "escape sequence '\\$char' at char " . (pos($template) - 2)
94 if $char !~ /^[n\\]$/;
97 while ($template =~ /\{\{([^\s{}]+)\}\}/g) {
99 push @problematic, "variable '$var' at char " . (pos($template) - length($var))
100 if $var !~ /^(cluster|guestname|node|vmid)$/;
103 die "found unknown: " . join(', ', @problematic) . "\n" if scalar(@problematic);
106 my $generate_notes = sub {
107 my ($notes_template, $task) = @_;
109 $verify_notes_template->($notes_template);
112 cluster
=> PVE
::Cluster
::get_clinfo
()->{cluster
}->{name
} // 'standalone node',
113 guestname
=> $task->{hostname
} // "VM $task->{vmid}", # is always set for CTs
114 node
=> PVE
::INotify
::nodename
(),
115 vmid
=> $task->{vmid
},
120 return '\\' if $char eq '\\';
121 return "\n" if $char eq 'n';
122 die "unexpected escape character '$char'\n";
125 $notes_template =~ s/\\(.)/$unescape->($1)/eg;
127 my $vars = join('|', keys $info->%*);
128 $notes_template =~ s/\{\{($vars)\}\}/$info->{$1}/g;
130 return $notes_template;
133 my sub parse_performance
{
136 if (defined(my $perf = $param->{performance
})) {
137 return if ref($perf) eq 'HASH'; # already parsed
138 $param->{performance
} = PVE
::JSONSchema
::parse_property_string
('backup-performance', $perf);
142 my $parse_prune_backups_maxfiles = sub {
143 my ($param, $kind) = @_;
145 my $maxfiles = delete $param->{maxfiles
};
146 my $prune_backups = $param->{'prune-backups'};
148 debugmsg
('warn', "both 'maxfiles' and 'prune-backups' defined as ${kind} - ignoring 'maxfiles'")
149 if defined($maxfiles) && defined($prune_backups);
151 if (defined($prune_backups)) {
152 return if ref($prune_backups) eq 'HASH'; # already parsed
153 $param->{'prune-backups'} = PVE
::JSONSchema
::parse_property_string
(
157 } elsif (defined($maxfiles)) {
159 $param->{'prune-backups'} = { 'keep-last' => $maxfiles };
161 $param->{'prune-backups'} = { 'keep-all' => 1 };
169 my $cfg = PVE
::Storage
::config
();
170 my $scfg = PVE
::Storage
::storage_config
($cfg, $storage);
171 my $type = $scfg->{type
};
173 die "can't use storage '$storage' for backups - wrong content type\n"
174 if (!$scfg->{content
}->{backup
});
180 $info->{'prune-backups'} = PVE
::JSONSchema
::parse_property_string
('prune-backups', $scfg->{'prune-backups'})
181 if defined($scfg->{'prune-backups'});
183 if ($type eq 'pbs') {
186 $info->{dumpdir
} = PVE
::Storage
::get_backup_dir
($cfg, $storage);
195 my $kb = $size / 1024;
198 return int ($kb) . "KB";
201 my $mb = $size / (1024*1024);
203 return int ($mb) . "MB";
207 return sprintf ("%.2fGB", $gb);
210 return sprintf ("%.2fTB", $tb);
216 my $hours = int ($seconds/3600);
217 $seconds = $seconds - $hours*3600;
218 my $min = int ($seconds/60);
219 $seconds = $seconds - $min*60;
221 return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
227 $str =~ s/^(.{990})/$1\n/mg; # reduce line length
245 foreach my $p (split (/:/, $ENV{PATH
})) {
252 die "unable to find command '$bin'\n";
259 for my $vmid (sort {$a <=> $b} @vmids) {
260 die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
261 $vmid = int ($vmid); # remove leading zeros
270 sub read_vzdump_defaults
{
272 my $fn = "/etc/vzdump.conf";
276 my $default = $confdesc->{$_}->{default};
277 defined($default) ?
($_ => $default) : ()
278 } keys %$confdesc_for_defaults
280 $parse_prune_backups_maxfiles->($defaults, "defaults in VZDump schema");
281 parse_performance
($defaults);
284 eval { $raw = PVE
::Tools
::file_get_contents
($fn); };
285 return $defaults if $@;
287 my $conf_schema = { type
=> 'object', properties
=> $confdesc_for_defaults };
288 my $res = PVE
::JSONSchema
::parse_config
($conf_schema, $fn, $raw);
289 if (my $excludes = $res->{'exclude-path'}) {
290 if (ref($excludes) eq 'ARRAY') {
292 for my $path ($excludes->@*) {
293 # We still use `split_args` here to be compatible with old configs where one line
294 # still has multiple space separated entries.
295 push $list->@*, PVE
::Tools
::split_args
($path)->@*;
297 $res->{'exclude-path'} = $list;
299 $res->{'exclude-path'} = PVE
::Tools
::split_args
($excludes);
302 if (defined($res->{mailto
})) {
303 my @mailto = split_list
($res->{mailto
});
304 $res->{mailto
} = [ @mailto ];
306 $parse_prune_backups_maxfiles->($res, "options in '$fn'");
307 parse_performance
($res);
309 foreach my $key (keys %$defaults) {
310 $res->{$key} = $defaults->{$key} if !defined($res->{$key});
313 if (defined($res->{storage
}) && defined($res->{dumpdir
})) {
314 debugmsg
('warn', "both 'storage' and 'dumpdir' defined in '$fn' - ignoring 'dumpdir'");
315 delete $res->{dumpdir
};
321 sub read_backup_task_logs
{
322 my ($task_list) = @_;
326 for my $task (@$task_list) {
327 my $vmid = $task->{vmid
};
328 my $log_file = $task->{tmplog
};
329 if (!$task->{tmplog
}) {
330 $task_logs .= "$vmid: no log available\n\n";
333 if (open (my $TMP, '<', "$log_file")) {
334 while (my $line = <$TMP>) {
335 next if $line =~ /^status: \d+/; # not useful in mails
336 $task_logs .= encode8bit
("$vmid: $line");
340 $task_logs .= "$vmid: Could not open log file\n\n";
348 sub build_guest_table
{
349 my ($task_list) = @_;
369 renderer
=> "duration"
374 renderer
=> "human-bytes"
385 for my $task (@$task_list) {
386 my $successful = $task->{state} eq 'ok';
387 my $size = $successful ?
$task->{size
} : 0;
388 my $filename = $successful ?
$task->{target
} : undef;
389 push @{$table->{data
}}, {
390 "vmid" => int($task->{vmid
}),
391 "name" => $task->{hostname
},
392 "status" => $task->{state},
393 "time" => int($task->{backuptime
}),
394 "size" => int($size),
395 "filename" => $filename,
402 sub sanitize_task_list
{
403 my ($task_list) = @_;
404 for my $task (@$task_list) {
405 chomp $task->{msg
} if $task->{msg
};
406 $task->{backuptime
} = 0 if !$task->{backuptime
};
407 $task->{size
} = 0 if !$task->{size
};
408 $task->{target
} = 'unknown' if !$task->{target
};
409 $task->{hostname
} = "VM $task->{vmid}" if !$task->{hostname
};
411 if ($task->{state} eq 'todo') {
412 $task->{msg
} = 'aborted';
417 sub count_failed_tasks
{
421 for my $task (@$tasklist) {
422 $error_count++ if $task->{state} ne 'ok';
429 my $hostname = `hostname -f` || PVE
::INotify
::nodename
();
434 my $subject_template = "vzdump backup status ({{hostname}}): {{status-text}}";
436 my $body_template = <<EOT;
438 {{heading-1 "Details"}}
439 {{table guest-table}}
441 Total running time: {{duration total-time}}
444 {{verbatim-monospaced logs}}
447 use constant MAX_LOG_SIZE
=> 1024*1024;
449 sub send_notification
{
450 my ($self, $tasklist, $total_time, $err, $detail_pre, $detail_post) = @_;
452 my $opts = $self->{opts
};
453 my $mailto = $opts->{mailto
};
454 my $cmdline = $self->{cmdline
};
455 my $target = $opts->{"notification-target"};
456 # Fall back to 'mailnotification' if 'notification-policy' is not set.
457 # If both are set, 'notification-policy' takes precedence
458 my $policy = $opts->{"notification-policy"} // $opts->{mailnotification
} // 'always';
460 return if ($policy eq 'never');
462 sanitize_task_list
($tasklist);
463 my $error_count = count_failed_tasks
($tasklist);
465 my $failed = ($error_count || $err);
467 return if (!$failed && ($policy eq 'failure'));
469 my $status_text = $failed ?
'backup failed' : 'backup successful';
473 $status_text .= ": multiple problems";
475 $status_text .= ": $err";
480 my $text_log_part = "$cmdline\n\n";
481 $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
482 $text_log_part .= read_backup_task_logs
($tasklist);
483 $text_log_part .= $detail_post if defined($detail_post);
485 if (length($text_log_part) > MAX_LOG_SIZE
)
487 # Let's limit the maximum length of included logs
488 $text_log_part = "Log output was too long to be sent. ".
489 "See Task History for details!\n";
492 my $notification_props = {
493 "hostname" => get_hostname
(),
494 "error-message" => $err,
495 "guest-table" => build_guest_table
($tasklist),
496 "logs" => $text_log_part,
497 "status-text" => $status_text,
498 "total-time" => $total_time,
501 my $notification_config = PVE
::Notify
::read_config
();
503 if ($mailto && scalar(@$mailto)) {
504 # <, >, @ are not allowed in endpoint names, but that is only
505 # verified once the config is serialized. That means that
506 # we can rely on that fact that no other endpoint with this name exists.
507 my $endpoint_name = "<mail-to-" . join(",", @$mailto) . ">";
508 $notification_config->add_sendmail_endpoint(
513 "vzdump backup tool");
515 my $endpoints = [$endpoint_name];
517 # Create an anonymous group containing the sendmail endpoint and the
518 # $target endpoint, if specified
520 push @$endpoints, $target;
523 $target = "<group-$endpoint_name>";
524 $notification_config->add_group(
530 return if (!$target);
532 my $severity = $failed ?
"error" : "info";
545 my ($class, $cmdline, $opts, $skiplist) = @_;
551 check_bin
('sendmail');
555 check_bin
('umount');
556 check_bin
('cstream');
557 check_bin
('ionice');
559 if ($opts->{mode
} && $opts->{mode
} eq 'snapshot') {
560 check_bin
('lvcreate');
562 check_bin
('lvremove');
565 my $defaults = read_vzdump_defaults
();
567 foreach my $k (keys %$defaults) {
568 next if $k eq 'exclude-path' || $k eq 'prune-backups'; # dealt with separately
569 if ($k eq 'dumpdir' || $k eq 'storage') {
570 $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir
}) &&
571 !defined ($opts->{storage
});
573 $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
577 $opts->{dumpdir
} =~ s
|/+$|| if ($opts->{dumpdir
});
578 $opts->{tmpdir
} =~ s
|/+$|| if ($opts->{tmpdir
});
580 $skiplist = [] if !$skiplist;
584 skiplist
=> $skiplist,
587 my $findexcl = $self->{findexcl
} = [];
588 if ($defaults->{'exclude-path'}) {
589 push @$findexcl, @{$defaults->{'exclude-path'}};
592 if ($opts->{'exclude-path'}) {
593 push @$findexcl, @{$opts->{'exclude-path'}};
596 if ($opts->{stdexcludes
}) {
604 foreach my $p (@plugins) {
605 my $pd = $p->new($self);
607 push @{$self->{plugins
}}, $pd;
610 if (defined($opts->{storage
}) && $opts->{stdout
}) {
611 die "cannot use options 'storage' and 'stdout' at the same time\n";
612 } elsif (defined($opts->{storage
}) && defined($opts->{dumpdir
})) {
613 die "cannot use options 'storage' and 'dumpdir' at the same time\n";
616 if (my $storage = get_storage_param
($opts)) {
617 $opts->{storage
} = $storage;
620 # Enforced by the API too, but these options might come in via defaults. Drop them if necessary.
621 if (!$opts->{storage
}) {
622 delete $opts->{$_} for qw(notes-template protected);
626 my $add_error = sub {
628 $errors .= "\n" if $errors;
634 $self->{job_init_log
} = '';
635 open my $job_init_fd, '>', \
$self->{job_init_log
};
636 $self->run_hook_script('job-init', undef, $job_init_fd);
639 PVE
::Cluster
::cfs_update
(); # Pick up possible changes made by the hook script.
641 $add_error->($@) if $@;
643 if ($opts->{storage
}) {
644 my $storage_cfg = PVE
::Storage
::config
();
645 eval { PVE
::Storage
::activate_storage
($storage_cfg, $opts->{storage
}) };
646 $add_error->("could not activate storage '$opts->{storage}': $@") if $@;
648 my $info = eval { storage_info
($opts->{storage
}) };
650 $add_error->("could not get storage information for '$opts->{storage}': $err");
652 $opts->{dumpdir
} = $info->{dumpdir
};
653 $opts->{scfg
} = $info->{scfg
};
654 $opts->{pbs
} = $info->{pbs
};
655 $opts->{'prune-backups'} //= $info->{'prune-backups'};
657 } elsif ($opts->{dumpdir
}) {
658 $add_error->("dumpdir '$opts->{dumpdir}' does not exist")
659 if ! -d
$opts->{dumpdir
};
661 die "internal error";
664 $opts->{'prune-backups'} //= $defaults->{'prune-backups'};
666 # avoid triggering any remove code path if keep-all is set
667 $opts->{remove
} = 0 if $opts->{'prune-backups'}->{'keep-all'};
669 if ($opts->{tmpdir
} && ! -d
$opts->{tmpdir
}) {
670 $add_error->("tmpdir '$opts->{tmpdir}' does not exist");
674 eval { $self->send_notification([], 0, $errors); };
675 debugmsg
('err', $@) if $@;
685 # Note: df 'available' can be negative, and percentage set to '-'
687 my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
693 if (my ($fsid, $fstype, undef, $mp) = $line =~
694 m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
703 eval { PVE
::Tools
::run_command
($cmd, errfunc
=> sub {}, outfunc
=> $parser); };
710 my ($self, $upid) = @_;
714 my $maxwait = $self->{opts
}->{lockwait
} || $self->{lockwait
};
716 die "missing UPID" if !$upid; # should not happen
719 if (!open ($SERVER_FLCK, '>>', "$lockfile")) {
720 debugmsg
('err', "can't open lock on file '$lockfile' - $!", undef, 1);
721 die "can't open lock on file '$lockfile' - $!";
724 if (!flock ($SERVER_FLCK, LOCK_EX
|LOCK_NB
)) {
726 debugmsg
('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
727 die "can't acquire lock '$lockfile' (wait = 0)";
730 debugmsg
('info', "trying to get global lock - waiting...", undef, 1);
732 alarm ($maxwait * 60);
734 local $SIG{ALRM
} = sub { alarm (0); die "got timeout\n"; };
736 if (!flock ($SERVER_FLCK, LOCK_EX
)) {
738 close ($SERVER_FLCK);
749 debugmsg
('err', "can't acquire lock '$lockfile' - $err", undef, 1);
750 die "can't acquire lock '$lockfile' - $err";
753 debugmsg
('info', "got global lock", undef, 1);
756 PVE
::Tools
::file_set_contents
($pidfile, $upid);
761 sub run_hook_script
{
762 my ($self, $phase, $task, $logfd) = @_;
764 my $opts = $self->{opts
};
766 my $script = $opts->{script
};
769 die "Error: The hook script '$script' does not exist.\n" if ! -f
$script;
770 die "Error: The hook script '$script' is not executable.\n" if ! -x
$script;
772 my $cmd = [$script, $phase];
775 push @$cmd, $task->{mode
};
776 push @$cmd, $task->{vmid
};
780 # set immutable opts directly (so they are available in all phases)
781 $ENV{STOREID
} = $opts->{storage
} if $opts->{storage
};
782 $ENV{DUMPDIR
} = $opts->{dumpdir
} if $opts->{dumpdir
};
784 foreach my $ek (qw(vmtype hostname target logfile)) {
785 $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
788 run_command
($logfd, $cmd);
791 sub compressor_info
{
793 my $opt_compress = $opts->{compress
};
795 if (!$opt_compress || $opt_compress eq '0') {
797 } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
798 return ('lzop', 'lzo');
799 } elsif ($opt_compress eq 'gzip') {
800 if ($opts->{pigz
} > 0) {
801 my $pigz_threads = $opts->{pigz
};
802 if ($pigz_threads == 1) {
803 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
804 $pigz_threads = int(($cpuinfo->{cpus
} + 1)/2);
806 return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
808 return ('gzip --rsyncable', 'gz');
810 } elsif ($opt_compress eq 'zstd') {
811 my $zstd_threads = $opts->{zstd
} // 1;
812 if ($zstd_threads == 0) {
813 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
814 $zstd_threads = int(($cpuinfo->{cpus
} + 1)/2);
816 return ("zstd --threads=${zstd_threads}", 'zst');
818 die "internal error - unknown compression option '$opt_compress'";
822 sub get_backup_file_list
{
823 my ($dir, $bkname) = @_;
826 foreach my $fn (<$dir/${bkname
}-*>) {
827 my $archive_info = eval { PVE
::Storage
::archive_info
($fn) } // {};
828 if ($archive_info->{is_std_name
}) {
829 my $path = "$dir/$archive_info->{filename}";
832 'ctime' => $archive_info->{ctime
},
834 $backup->{mark
} = "protected"
835 if -e PVE
::Storage
::protection_file_path
($path);
836 push @{$bklist}, $backup;
843 sub exec_backup_task
{
844 my ($self, $task) = @_;
846 my $opts = $self->{opts
};
848 my $cfg = PVE
::Storage
::config
();
849 my $vmid = $task->{vmid
};
850 my $plugin = $task->{plugin
};
852 $task->{backup_time
} = time();
855 my $pbs_snapshot_name;
857 my $vmstarttime = time ();
863 my $log_vm_online_again = sub {
864 return if !defined($task->{vmstoptime
});
865 $task->{vmconttime
} //= time();
866 my $delay = $task->{vmconttime
} - $task->{vmstoptime
};
867 $delay = '<1' if $delay < 1;
868 debugmsg
('info', "guest is online again after $delay seconds", $logfd);
872 die "unable to find VM '$vmid'\n" if !$plugin;
874 my $vmtype = $plugin->type();
876 if ($self->{opts
}->{pbs
}) {
877 if ($vmtype eq 'lxc') {
878 $pbs_group_name = "ct/$vmid";
879 } elsif ($vmtype eq 'qemu') {
880 $pbs_group_name = "vm/$vmid";
882 die "pbs backup not implemented for plugin type '$vmtype'\n";
884 my $btime = strftime
("%FT%TZ", gmtime($task->{backup_time
}));
885 $pbs_snapshot_name = "$pbs_group_name/$btime";
888 # for now we deny backups of a running ha managed service in *stop* mode
889 # as it interferes with the HA stack (started services should not stop).
890 if ($opts->{mode
} eq 'stop' &&
891 PVE
::HA
::Config
::vm_is_ha_managed
($vmid, 'started'))
893 die "Cannot execute a backup with stop mode on a HA managed and".
894 " enabled Service. Use snapshot mode or disable the Service.\n";
897 my $tmplog = "$logdir/$vmtype-$vmid.log";
899 my $bkname = "vzdump-$vmtype-$vmid";
900 my $basename = $bkname . strftime
("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time
}));
902 my $prune_options = $opts->{'prune-backups'};
904 my $backup_limit = 0;
905 if (!$prune_options->{'keep-all'}) {
906 foreach my $keep (values %{$prune_options}) {
907 $backup_limit += $keep;
911 if (($backup_limit && !$opts->{remove
}) || $opts->{protected
}) {
914 if (my $storeid = $opts->{storage
}) {
916 !$_->{subtype
} || $_->{subtype
} eq $vmtype
917 } PVE
::Storage
::volume_list
($cfg, $storeid, $vmid, 'backup')->@*;
919 $count = grep { !$_->{protected
} } @backups;
920 $protected_count = scalar(@backups) - $count;
922 $count = grep { !$_->{mark
} || $_->{mark
} ne "protected" } get_backup_file_list
($opts->{dumpdir
}, $bkname)->@*;
925 if ($opts->{protected
}) {
926 my $max_protected = PVE
::Storage
::get_max_protected_backups
(
930 if ($max_protected > -1 && $protected_count >= $max_protected) {
931 die "The number of protected backups per guest is limited to $max_protected ".
932 "on storage '$opts->{storage}'\n";
934 } elsif ($count >= $backup_limit) {
935 die "There is a max backup limit of $backup_limit enforced by the target storage ".
936 "or the vzdump parameters. Either increase the limit or delete old backups.\n";
940 if (!$self->{opts
}->{pbs
}) {
941 $task->{logfile
} = "$opts->{dumpdir}/$basename.log";
944 my $ext = $vmtype eq 'qemu' ?
'.vma' : '.tar';
945 my ($comp, $comp_ext) = compressor_info
($opts);
946 if ($comp && $comp_ext) {
947 $ext .= ".${comp_ext}";
950 if ($self->{opts
}->{pbs
}) {
951 die "unable to pipe backup to stdout\n" if $opts->{stdout
};
952 $task->{target
} = $pbs_snapshot_name;
954 if ($opts->{stdout
}) {
955 $task->{target
} = '-';
957 $task->{target
} = $task->{tmptar
} = "$opts->{dumpdir}/$basename$ext";
958 $task->{tmptar
} =~ s/\.[^\.]+$/\.dat/;
959 unlink $task->{tmptar
};
963 $task->{vmtype
} = $vmtype;
966 if ($opts->{tmpdir
}) {
967 $task->{tmpdir
} = "$opts->{tmpdir}/vzdumptmp${pid}_$vmid/";
968 } elsif ($self->{opts
}->{pbs
}) {
969 $task->{tmpdir
} = "/var/tmp/vzdumptmp${pid}_$vmid";
971 # dumpdir is posix? then use it as temporary dir
972 my $info = get_mount_info
($opts->{dumpdir
});
973 if ($vmtype eq 'qemu' ||
974 grep ($_ eq $info->{fstype
}, @posix_filesystems)) {
975 $task->{tmpdir
} = "$opts->{dumpdir}/$basename.tmp";
977 $task->{tmpdir
} = "/var/tmp/vzdumptmp${pid}_$vmid";
978 debugmsg
('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
979 "using $task->{tmpdir} for temporary files", $logfd);
983 rmtree
$task->{tmpdir
};
984 mkdir $task->{tmpdir
};
985 -d
$task->{tmpdir
} ||
986 die "unable to create temporary directory '$task->{tmpdir}'";
988 $logfd = IO
::File-
>new (">$tmplog") ||
989 die "unable to create log file '$tmplog'";
991 $task->{dumpdir
} = $opts->{dumpdir
};
992 $task->{storeid
} = $opts->{storage
};
993 $task->{scfg
} = $opts->{scfg
};
994 $task->{tmplog
} = $tmplog;
996 unlink $task->{logfile
} if defined($task->{logfile
});
998 debugmsg
('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
999 debugmsg
('info', "Backup started at " . strftime
("%F %H:%M:%S", localtime()));
1001 $plugin->set_logfd ($logfd);
1003 # test is VM is running
1004 my ($running, $status_text) = $plugin->vm_status ($vmid);
1006 debugmsg
('info', "status = ${status_text}", $logfd);
1008 # lock VM (prevent config changes)
1009 $plugin->lock_vm ($vmid);
1011 $cleanup->{unlock
} = 1;
1015 my $mode = $running ?
$task->{mode
} : 'stop';
1017 if ($mode eq 'snapshot') {
1018 my %saved_task = %$task;
1019 eval { $plugin->prepare ($task, $vmid, $mode); };
1021 die $err if $err !~ m/^mode failure/;
1022 debugmsg
('info', $err, $logfd);
1023 debugmsg
('info', "trying 'suspend' mode instead", $logfd);
1024 $mode = 'suspend'; # so prepare is called again below
1025 %$task = %saved_task;
1029 $cleanup->{prepared
} = 1;
1031 $task->{mode
} = $mode;
1033 debugmsg
('info', "backup mode: $mode", $logfd);
1034 debugmsg
('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd) if $opts->{bwlimit
};
1035 debugmsg
('info', "ionice priority: $opts->{ionice}", $logfd);
1037 if ($mode eq 'stop') {
1038 $plugin->prepare ($task, $vmid, $mode);
1040 $self->run_hook_script ('backup-start', $task, $logfd);
1043 debugmsg
('info', "stopping virtual guest", $logfd);
1044 $task->{vmstoptime
} = time();
1045 $self->run_hook_script ('pre-stop', $task, $logfd);
1046 $plugin->stop_vm ($task, $vmid);
1047 $cleanup->{restart
} = 1;
1051 } elsif ($mode eq 'suspend') {
1052 $plugin->prepare ($task, $vmid, $mode);
1054 $self->run_hook_script ('backup-start', $task, $logfd);
1056 if ($vmtype eq 'lxc') {
1058 $plugin->copy_data_phase1($task, $vmid);
1061 debugmsg
('info', "suspending guest", $logfd);
1062 $task->{vmstoptime
} = time ();
1063 $self->run_hook_script ('pre-stop', $task, $logfd);
1064 $plugin->suspend_vm ($task, $vmid);
1065 $cleanup->{resume
} = 1;
1067 if ($vmtype eq 'lxc') {
1068 # post-suspend rsync
1069 $plugin->copy_data_phase2($task, $vmid);
1071 debugmsg
('info', "resuming guest", $logfd);
1072 $cleanup->{resume
} = 0;
1073 $self->run_hook_script('pre-restart', $task, $logfd);
1074 $plugin->resume_vm($task, $vmid);
1075 $self->run_hook_script('post-restart', $task, $logfd);
1076 $log_vm_online_again->();
1079 } elsif ($mode eq 'snapshot') {
1080 $self->run_hook_script ('backup-start', $task, $logfd);
1082 my $snapshot_count = $task->{snapshot_count
} || 0;
1084 $self->run_hook_script ('pre-stop', $task, $logfd);
1086 if ($snapshot_count > 1) {
1087 debugmsg
('info', "suspend vm to make snapshot", $logfd);
1088 $task->{vmstoptime
} = time ();
1089 $plugin->suspend_vm ($task, $vmid);
1090 $cleanup->{resume
} = 1;
1093 $plugin->snapshot ($task, $vmid);
1095 $self->run_hook_script ('pre-restart', $task, $logfd);
1097 if ($snapshot_count > 1) {
1098 debugmsg
('info', "resume vm", $logfd);
1099 $cleanup->{resume
} = 0;
1100 $plugin->resume_vm ($task, $vmid);
1101 $log_vm_online_again->();
1104 $self->run_hook_script ('post-restart', $task, $logfd);
1107 die "internal error - unknown mode '$mode'\n";
1110 # assemble archive image
1111 $plugin->assemble ($task, $vmid);
1115 if ($opts->{stdout
}) {
1116 debugmsg
('info', "sending archive to stdout", $logfd);
1117 $plugin->archive($task, $vmid, $task->{tmptar
}, $comp);
1118 $self->run_hook_script ('backup-end', $task, $logfd);
1122 my $archive_txt = $self->{opts
}->{pbs
} ?
'Proxmox Backup Server' : 'vzdump';
1123 debugmsg
('info', "creating $archive_txt archive '$task->{target}'", $logfd);
1124 $plugin->archive($task, $vmid, $task->{tmptar
}, $comp);
1126 if ($self->{opts
}->{pbs
}) {
1127 # size is added to task struct in guest vzdump plugins
1129 rename ($task->{tmptar
}, $task->{target
}) ||
1130 die "unable to rename '$task->{tmptar}' to '$task->{target}'\n";
1133 $task->{size
} = (-s
$task->{target
}) || 0;
1134 my $cs = format_size
($task->{size
});
1135 debugmsg
('info', "archive file size: $cs", $logfd);
1138 # Mark as protected before pruning.
1139 if (my $storeid = $opts->{storage
}) {
1140 my $volname = $opts->{pbs
} ?
$task->{target
} : basename
($task->{target
});
1141 my $volid = "${storeid}:backup/${volname}";
1143 if ($opts->{'notes-template'} && $opts->{'notes-template'} ne '') {
1144 debugmsg
('info', "adding notes to backup", $logfd);
1145 my $notes = eval { $generate_notes->($opts->{'notes-template'}, $task); };
1147 debugmsg
('warn', "unable to add notes - $err", $logfd);
1149 eval { PVE
::Storage
::update_volume_attribute
($cfg, $volid, 'notes', $notes) };
1150 debugmsg
('warn', "unable to add notes - $@", $logfd) if $@;
1154 if ($opts->{protected
}) {
1155 debugmsg
('info', "marking backup as protected", $logfd);
1156 eval { PVE
::Storage
::update_volume_attribute
($cfg, $volid, 'protected', 1) };
1157 die "unable to set protected flag - $@\n" if $@;
1161 if ($opts->{remove
}) {
1162 my $keepstr = join(', ', map { "$_=$prune_options->{$_}" } sort keys %$prune_options);
1163 debugmsg
('info', "prune older backups with retention: $keepstr", $logfd);
1165 if (!defined($opts->{storage
})) {
1166 my $bklist = get_backup_file_list
($opts->{dumpdir
}, $bkname);
1168 PVE
::Storage
::prune_mark_backup_group
($bklist, $prune_options);
1170 foreach my $prune_entry (@{$bklist}) {
1171 next if $prune_entry->{mark
} ne 'remove';
1173 my $archive_path = $prune_entry->{path
};
1174 debugmsg
('info', "delete old backup '$archive_path'", $logfd);
1175 PVE
::Storage
::archive_remove
($archive_path);
1178 my $pruned_list = PVE
::Storage
::prune_backups
(
1185 sub { debugmsg
($_[0], $_[1], $logfd) },
1187 $pruned = scalar(grep { $_->{mark
} eq 'remove' } $pruned_list->@*);
1189 my $log_pruned_extra = $pruned > 0 ?
" not covered by keep-retention policy" : "";
1190 debugmsg
('info', "pruned $pruned backup(s)${log_pruned_extra}", $logfd);
1193 $self->run_hook_script ('backup-end', $task, $logfd);
1200 if ($cleanup->{unlock
}) {
1201 eval { $plugin->unlock_vm ($vmid); };
1205 if ($cleanup->{prepared
}) {
1206 # only call cleanup when necessary (when prepare was executed)
1207 eval { $plugin->cleanup ($task, $vmid) };
1211 eval { $plugin->set_logfd (undef); };
1214 if ($cleanup->{resume
} || $cleanup->{restart
}) {
1216 $self->run_hook_script ('pre-restart', $task, $logfd);
1217 if ($cleanup->{resume
}) {
1218 debugmsg
('info', "resume vm", $logfd);
1219 $plugin->resume_vm ($task, $vmid);
1221 my $running = $plugin->vm_status($vmid);
1223 debugmsg
('info', "restarting vm", $logfd);
1224 $plugin->start_vm ($task, $vmid);
1227 $self->run_hook_script ('post-restart', $task, $logfd);
1233 $log_vm_online_again->();
1238 eval { unlink $task->{tmptar
} if $task->{tmptar
} && -f
$task->{tmptar
}; };
1241 eval { rmtree
$task->{tmpdir
} if $task->{tmpdir
} && -d
$task->{tmpdir
}; };
1244 my $delay = $task->{backuptime
} = time () - $vmstarttime;
1247 $task->{state} = 'err';
1248 $task->{msg
} = $err;
1249 debugmsg
('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1250 debugmsg
('info', "Failed at " . strftime
("%F %H:%M:%S", localtime()));
1252 eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1255 $task->{state} = 'ok';
1256 my $tstr = format_time
($delay);
1257 debugmsg
('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1258 debugmsg
('info', "Backup finished at " . strftime
("%F %H:%M:%S", localtime()));
1261 close ($logfd) if $logfd;
1263 if ($task->{tmplog
}) {
1264 if ($self->{opts
}->{pbs
}) {
1265 if ($task->{state} eq 'ok') {
1267 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
(
1271 [ $pbs_snapshot_name, $task->{tmplog
} ],
1272 errmsg
=> "uploading backup task log failed",
1276 debugmsg
('warn', "$@") if $@; # $@ contains already error prefix
1278 } elsif ($task->{logfile
}) {
1279 system {'cp'} 'cp', $task->{tmplog
}, $task->{logfile
};
1283 eval { $self->run_hook_script ('log-end', $task); };
1285 die $err if $err && $err =~ m/^interrupted by signal$/;
1289 my ($self, $rpcenv, $authuser) = @_;
1291 my $opts = $self->{opts
};
1293 debugmsg
('info', "starting new backup job: $self->{cmdline}", undef, 1);
1295 if (scalar(@{$self->{skiplist
}})) {
1296 my $skip_string = join(', ', sort { $a <=> $b } @{$self->{skiplist
}});
1297 debugmsg
('info', "skip external VMs: $skip_string");
1301 my $vzdump_plugins = {};
1302 foreach my $plugin (@{$self->{plugins
}}) {
1303 my $type = $plugin->type();
1304 next if exists $vzdump_plugins->{$type};
1305 $vzdump_plugins->{$type} = $plugin;
1308 my $vmlist = PVE
::Cluster
::get_vmlist
();
1309 my $vmids = [ sort { $a <=> $b } @{$opts->{vmids
}} ];
1310 foreach my $vmid (@{$vmids}) {
1312 if (defined($vmlist->{ids
}->{$vmid})) {
1313 my $guest_type = $vmlist->{ids
}->{$vmid}->{type
};
1314 $plugin = $vzdump_plugins->{$guest_type};
1315 next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all
});
1318 mode
=> $opts->{mode
},
1325 # Use in-memory files for the outer hook logs to pass them to sendmail.
1326 my $job_start_log = '';
1327 my $job_end_log = '';
1328 open my $job_start_fd, '>', \
$job_start_log;
1329 open my $job_end_fd, '>', \
$job_end_log;
1331 my $starttime = time();
1335 $self->run_hook_script ('job-start', undef, $job_start_fd);
1337 foreach my $task (@$tasklist) {
1338 $self->exec_backup_task ($task);
1339 $errcount += 1 if $task->{state} ne 'ok';
1342 $self->run_hook_script ('job-end', undef, $job_end_fd);
1347 eval { $self->run_hook_script ('job-abort', undef, $job_end_fd); };
1349 debugmsg
('err', "Backup job failed - $err", undef, 1);
1352 debugmsg
('info', "Backup job finished with errors", undef, 1);
1354 debugmsg
('info', "Backup job finished successfully", undef, 1);
1358 close $job_start_fd;
1361 my $totaltime = time() - $starttime;
1364 # otherwise $self->send_notification() will interpret it as multiple problems
1365 my $chomped_err = $err;
1366 chomp($chomped_err) if $chomped_err;
1368 $self->send_notification(
1372 $self->{job_init_log
} . $job_start_log,
1376 debugmsg
('err', $@) if $@;
1380 die "job errors\n" if $errcount;
1388 return defined($confdesc->{$key});
1391 # NOTE it might make sense to merge this and verify_vzdump_parameters(), but one
1392 # needs to adapt command_line() in guest-common's PVE/VZDump/Common.pm and detect
1393 # a second parsing attempt, because verify_vzdump_parameters() is called twice
1394 # during the update_job API call.
1395 sub parse_mailto_exclude_path
{
1398 # exclude-path list need to be 0 separated or be an array
1399 if (defined($param->{'exclude-path'})) {
1401 if (ref($param->{'exclude-path'}) eq 'ARRAY') {
1402 $expaths = $param->{'exclude-path'};
1404 $expaths = [split(/\0/, $param->{'exclude-path'} || '')];
1406 $param->{'exclude-path'} = $expaths;
1409 if (defined($param->{mailto
})) {
1410 my @mailto = PVE
::Tools
::split_list
(extract_param
($param, 'mailto'));
1411 $param->{mailto
} = [ @mailto ];
1417 sub verify_vzdump_parameters
{
1418 my ($param, $check_missing) = @_;
1420 raise_param_exc
({ all
=> "option conflicts with option 'vmid'"})
1421 if $param->{all
} && $param->{vmid
};
1423 raise_param_exc
({ exclude
=> "option conflicts with option 'vmid'"})
1424 if $param->{exclude
} && $param->{vmid
};
1426 raise_param_exc
({ pool
=> "option conflicts with option 'vmid'"})
1427 if $param->{pool
} && $param->{vmid
};
1429 raise_param_exc
({ 'prune-backups' => "option conflicts with option 'maxfiles'"})
1430 if defined($param->{'prune-backups'}) && defined($param->{maxfiles
});
1432 $parse_prune_backups_maxfiles->($param, 'CLI parameters');
1433 parse_performance
($param);
1435 if (my $template = $param->{'notes-template'}) {
1436 eval { $verify_notes_template->($template); };
1437 raise_param_exc
({'notes-template' => $@}) if $@;
1440 $param->{all
} = 1 if (defined($param->{exclude
}) && !$param->{pool
});
1442 return if !$check_missing;
1444 raise_param_exc
({ vmid
=> "property is missing"})
1445 if !($param->{all
} || $param->{stop
} || $param->{pool
}) && !$param->{vmid
};
1449 sub stop_running_backups
{
1452 my $upid = PVE
::Tools
::file_read_firstline
($pidfile);
1455 my $task = PVE
::Tools
::upid_decode
($upid);
1457 if (PVE
::ProcFSTools
::check_process_running
($task->{pid
}, $task->{pstart
}) &&
1458 PVE
::ProcFSTools
::read_proc_starttime
($task->{pid
}) == $task->{pstart
}) {
1459 kill(15, $task->{pid
});
1460 # wait max 15 seconds to shut down (else, do nothing for now)
1462 for ($i = 15; $i > 0; $i--) {
1463 last if !PVE
::ProcFSTools
::check_process_running
(($task->{pid
}, $task->{pstart
}));
1466 die "stopping backup process $task->{pid} failed\n" if $i == 0;
1470 sub get_included_guests
{
1474 my $vmids_per_node = {};
1476 my $vmlist = PVE
::Cluster
::get_vmlist
();
1479 $vmids = PVE
::API2Tools
::get_resource_pool_guest_members
($job->{pool
});
1480 } elsif ($job->{vmid
}) {
1481 $vmids = [ split_list
($job->{vmid
}) ];
1482 } elsif ($job->{all
}) {
1484 my $exclude = check_vmids
(split_list
($job->{exclude
}));
1485 my $excludehash = { map { $_ => 1 } @$exclude };
1487 for my $id (keys %{$vmlist->{ids
}}) {
1488 next if $excludehash->{$id};
1492 return $vmids_per_node;
1494 $vmids = check_vmids
(@$vmids);
1496 for my $vmid (@$vmids) {
1497 if (defined($vmlist->{ids
}->{$vmid})) {
1498 my $node = $vmlist->{ids
}->{$vmid}->{node
};
1499 next if (defined $job->{node
} && $job->{node
} ne $node);
1501 push @{$vmids_per_node->{$node}}, $vmid;
1503 push @{$vmids_per_node->{''}}, $vmid;
1507 return $vmids_per_node;