use strict;
use warnings;
+use Clone;
use Fcntl ':flock';
+use File::Basename;
use File::Path;
use IO::File;
use IO::Select;
use PVE::HA::Config;
use PVE::HA::Env::PVE2;
use PVE::JSONSchema qw(get_standard_option);
+use PVE::Notify;
use PVE::RPCEnvironment;
use PVE::Storage;
use PVE::VZDump::Common;
my $confdesc = PVE::VZDump::Common::get_confdesc();
+my $confdesc_for_defaults = Clone::clone($confdesc);
+delete $confdesc_for_defaults->{$_}->{requires} for qw(notes-template protected);
+
# Load available plugins
my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
foreach my $plug (@pve_vzdump_classes) {
}
}
+sub get_storage_param {
+ my ($param) = @_;
+
+ return if $param->{dumpdir};
+ return $param->{storage} || 'local';
+}
+
# helper functions
sub debugmsg {
PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
}
+my $verify_notes_template = sub {
+ my ($template) = @_;
+
+ die "contains a line feed\n" if $template =~ /\n/;
+
+ my @problematic = ();
+ while ($template =~ /\\(.)/g) {
+ my $char = $1;
+ push @problematic, "escape sequence '\\$char' at char " . (pos($template) - 2)
+ if $char !~ /^[n\\]$/;
+ }
+
+ while ($template =~ /\{\{([^\s{}]+)\}\}/g) {
+ my $var = $1;
+ push @problematic, "variable '$var' at char " . (pos($template) - length($var))
+ if $var !~ /^(cluster|guestname|node|vmid)$/;
+ }
+
+ die "found unknown: " . join(', ', @problematic) . "\n" if scalar(@problematic);
+};
+
+my $generate_notes = sub {
+ my ($notes_template, $task) = @_;
+
+ $verify_notes_template->($notes_template);
+
+ my $info = {
+ cluster => PVE::Cluster::get_clinfo()->{cluster}->{name} // 'standalone node',
+ guestname => $task->{hostname} // "VM $task->{vmid}", # is always set for CTs
+ node => PVE::INotify::nodename(),
+ vmid => $task->{vmid},
+ };
+
+ my $unescape = sub {
+ my ($char) = @_;
+ return '\\' if $char eq '\\';
+ return "\n" if $char eq 'n';
+ die "unexpected escape character '$char'\n";
+ };
+
+ $notes_template =~ s/\\(.)/$unescape->($1)/eg;
+
+ my $vars = join('|', keys $info->%*);
+ $notes_template =~ s/\{\{($vars)\}\}/$info->{$1}/g;
+
+ return $notes_template;
+};
+
+sub parse_fleecing {
+ my ($param) = @_;
+
+ if (defined(my $fleecing = $param->{fleecing})) {
+ return $fleecing if ref($fleecing) eq 'HASH'; # already parsed
+ $param->{fleecing} = PVE::JSONSchema::parse_property_string('backup-fleecing', $fleecing);
+ }
+
+ return $param->{fleecing};
+}
+
+my sub parse_performance {
+ my ($param) = @_;
+
+ if (defined(my $perf = $param->{performance})) {
+ return $perf if ref($perf) eq 'HASH'; # already parsed
+ $param->{performance} = PVE::JSONSchema::parse_property_string('backup-performance', $perf);
+ }
+
+ return $param->{performance};
+}
+
+my sub merge_performance {
+ my ($prefer, $fallback) = @_;
+
+ my $res = {};
+ for my $opt (keys PVE::JSONSchema::get_format('backup-performance')->%*) {
+ $res->{$opt} = $prefer->{$opt} // $fallback->{$opt}
+ if defined($prefer->{$opt}) || defined($fallback->{$opt});
+ }
+ return $res;
+}
+
+my $parse_prune_backups_maxfiles = sub {
+ my ($param, $kind) = @_;
+
+ my $maxfiles = delete $param->{maxfiles};
+ my $prune_backups = $param->{'prune-backups'};
+
+ debugmsg('warn', "both 'maxfiles' and 'prune-backups' defined as ${kind} - ignoring 'maxfiles'")
+ if defined($maxfiles) && defined($prune_backups);
+
+ if (defined($prune_backups)) {
+ return $prune_backups if ref($prune_backups) eq 'HASH'; # already parsed
+ $param->{'prune-backups'} = PVE::JSONSchema::parse_property_string(
+ 'prune-backups',
+ $prune_backups
+ );
+ } elsif (defined($maxfiles)) {
+ if ($maxfiles) {
+ $param->{'prune-backups'} = { 'keep-last' => $maxfiles };
+ } else {
+ $param->{'prune-backups'} = { 'keep-all' => 1 };
+ }
+ }
+
+ return $param->{'prune-backups'};
+};
+
sub storage_info {
my $storage = shift;
my $scfg = PVE::Storage::storage_config($cfg, $storage);
my $type = $scfg->{type};
- die "can't use storage type '$type' for backup\n"
- if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'
- || $type eq 'cifs' || $type eq 'cephfs' || $type eq 'pbs'));
die "can't use storage '$storage' for backups - wrong content type\n"
if (!$scfg->{content}->{backup});
- PVE::Storage::activate_storage($cfg, $storage);
-
my $info = {
scfg => $scfg,
- maxfiles => $scfg->{maxfiles},
};
+ $info->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'})
+ if defined($scfg->{'prune-backups'});
+
if ($type eq 'pbs') {
$info->{pbs} = 1;
} else {
map {
my $default = $confdesc->{$_}->{default};
defined($default) ? ($_ => $default) : ()
- } keys %$confdesc
+ } keys %$confdesc_for_defaults
+ };
+ my $performance_fmt = PVE::JSONSchema::get_format('backup-performance');
+ $defaults->{performance} = {
+ map {
+ my $default = $performance_fmt->{$_}->{default};
+ defined($default) ? ($_ => $default) : ()
+ } keys $performance_fmt->%*
+ };
+ my $fleecing_fmt = PVE::JSONSchema::get_format('backup-fleecing');
+ $defaults->{fleecing} = {
+ map {
+ my $default = $fleecing_fmt->{$_}->{default};
+ defined($default) ? ($_ => $default) : ()
+ } keys $fleecing_fmt->%*
};
+ $parse_prune_backups_maxfiles->($defaults, "defaults in VZDump schema");
my $raw;
eval { $raw = PVE::Tools::file_get_contents($fn); };
return $defaults if $@;
- my $conf_schema = { type => 'object', properties => $confdesc, };
+ my $conf_schema = { type => 'object', properties => $confdesc_for_defaults };
my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
if (my $excludes = $res->{'exclude-path'}) {
- $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
+ if (ref($excludes) eq 'ARRAY') {
+ my $list = [];
+ for my $path ($excludes->@*) {
+ # We still use `split_args` here to be compatible with old configs where one line
+ # still has multiple space separated entries.
+ push $list->@*, PVE::Tools::split_args($path)->@*;
+ }
+ $res->{'exclude-path'} = $list;
+ } else {
+ $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
+ }
}
if (defined($res->{mailto})) {
my @mailto = split_list($res->{mailto});
$res->{mailto} = [ @mailto ];
}
+ $parse_prune_backups_maxfiles->($res, "options in '$fn'");
+ parse_fleecing($res);
+ parse_performance($res);
+
+ for my $key (keys $defaults->%*) {
+ if (!defined($res->{$key})) {
+ $res->{$key} = $defaults->{$key};
+ } elsif ($key eq 'performance') {
+ $res->{$key} = merge_performance($res->{$key}, $defaults->{$key});
+ }
+ }
- foreach my $key (keys %$defaults) {
- $res->{$key} = $defaults->{$key} if !defined($res->{$key});
+ if (defined($res->{storage}) && defined($res->{dumpdir})) {
+ debugmsg('warn', "both 'storage' and 'dumpdir' defined in '$fn' - ignoring 'dumpdir'");
+ delete $res->{dumpdir};
}
return $res;
}
-use constant MAX_MAIL_SIZE => 1024*1024;
-sub sendmail {
- my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
+my sub read_backup_task_logs {
+ my ($task_list) = @_;
- my $opts = $self->{opts};
+ my $task_logs = "";
- my $mailto = $opts->{mailto};
+ for my $task (@$task_list) {
+ my $vmid = $task->{vmid};
+ my $log_file = $task->{tmplog};
+ if (!$task->{tmplog}) {
+ $task_logs .= "$vmid: no log available\n\n";
+ next;
+ }
+ if (open (my $TMP, '<', "$log_file")) {
+ while (my $line = <$TMP>) {
+ next if $line =~ /^status: \d+/; # not useful in mails
+ $task_logs .= encode8bit ("$vmid: $line");
+ }
+ close ($TMP);
+ } else {
+ $task_logs .= "$vmid: Could not open log file\n\n";
+ }
+ $task_logs .= "\n";
+ }
- return if !($mailto && scalar(@$mailto));
+ return $task_logs;
+}
- my $cmdline = $self->{cmdline};
+my sub build_guest_table {
+ my ($task_list) = @_;
+
+ my $table = {
+ schema => {
+ columns => [
+ {
+ label => "VMID",
+ id => "vmid"
+ },
+ {
+ label => "Name",
+ id => "name"
+ },
+ {
+ label => "Status",
+ id => "status"
+ },
+ {
+ label => "Time",
+ id => "time",
+ renderer => "duration"
+ },
+ {
+ label => "Size",
+ id => "size",
+ renderer => "human-bytes"
+ },
+ {
+ label => "Filename",
+ id => "filename"
+ },
+ ]
+ },
+ data => []
+ };
+
+ for my $task (@$task_list) {
+ my $successful = $task->{state} eq 'ok';
+ my $size = $successful ? $task->{size} : 0;
+ my $filename = $successful ? $task->{target} : undef;
+ push @{$table->{data}}, {
+ "vmid" => int($task->{vmid}),
+ "name" => $task->{hostname},
+ "status" => $task->{state},
+ "time" => int($task->{backuptime}),
+ "size" => int($size),
+ "filename" => $filename,
+ };
+ }
- my $ecount = 0;
- foreach my $task (@$tasklist) {
- $ecount++ if $task->{state} ne 'ok';
+ return $table;
+}
+
+my sub sanitize_task_list {
+ my ($task_list) = @_;
+ for my $task (@$task_list) {
chomp $task->{msg} if $task->{msg};
$task->{backuptime} = 0 if !$task->{backuptime};
$task->{size} = 0 if !$task->{size};
$task->{msg} = 'aborted';
}
}
+}
- my $notify = $opts->{mailnotification} || 'always';
- return if (!$ecount && !$err && ($notify eq 'failure'));
+my sub aggregate_task_statistics {
+ my ($tasklist) = @_;
- my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
- if ($err) {
- if ($err =~ /\n/) {
- $stat .= ": multiple problems";
- } else {
- $stat .= ": $err";
- $err = undef;
- }
+ my $error_count = 0;
+ my $total_size = 0;
+ for my $task (@$tasklist) {
+ $error_count++ if $task->{state} ne 'ok';
+ $total_size += $task->{size} if $task->{state} eq 'ok';
}
+ return ($error_count, $total_size);
+}
+
+my sub get_hostname {
my $hostname = `hostname -f` || PVE::INotify::nodename();
chomp $hostname;
+ return $hostname;
+}
- # text part
- my $text = $err ? "$err\n\n" : '';
- $text .= sprintf ("%-10s %-6s %10s %10s %s\n", qw(VMID STATUS TIME SIZE FILENAME));
- foreach my $task (@$tasklist) {
- my $vmid = $task->{vmid};
- if ($task->{state} eq 'ok') {
+my $subject_template = "vzdump backup status ({{hostname}}): {{status-text}}";
- $text .= sprintf ("%-10s %-6s %10s %10s %s\n", $vmid,
- $task->{state},
- format_time($task->{backuptime}),
- format_size ($task->{size}),
- $task->{target});
- } else {
- $text .= sprintf ("%-10s %-6s %10s %8.2fMB %s\n", $vmid,
- $task->{state},
- format_time($task->{backuptime}),
- 0, '-');
- }
- }
-
- my $text_log_part;
- $text_log_part .= "\nDetailed backup logs:\n\n";
- $text_log_part .= "$cmdline\n\n";
+my $body_template = <<EOT;
+{{error-message}}
+{{heading-1 "Details"}}
+{{table guest-table}}
+{{#verbatim}}
+Total running time: {{duration total-time}}
+Total size: {{human-bytes total-size}}
+{{/verbatim}}
+{{heading-1 "Logs"}}
+{{verbatim-monospaced logs}}
+EOT
- $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
- foreach my $task (@$tasklist) {
- my $vmid = $task->{vmid};
- my $log = $task->{tmplog};
- if (!$log) {
- $text_log_part .= "$vmid: no log available\n\n";
- next;
- }
- if (open (TMP, "$log")) {
- while (my $line = <TMP>) {
- next if $line =~ /^status: \d+/; # not useful in mails
- $text_log_part .= encode8bit ("$vmid: $line");
- }
- } else {
- $text_log_part .= "$vmid: Could not open log file\n\n";
- }
- close (TMP);
- $text_log_part .= "\n";
- }
- $text_log_part .= $detail_post if defined($detail_post);
+use constant MAX_LOG_SIZE => 1024*1024;
- # html part
- my $html = "<html><body>\n";
- $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
- $html .= "<table border=1 cellpadding=3>\n";
- $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
+sub send_notification {
+ my ($self, $tasklist, $total_time, $err, $detail_pre, $detail_post) = @_;
- my $ssize = 0;
+ my $opts = $self->{opts};
+ my $mailto = $opts->{mailto};
+ my $cmdline = $self->{cmdline};
+ my $policy = $opts->{mailnotification} // 'always';
+ my $mode = $opts->{"notification-mode"} // 'auto';
- foreach my $task (@$tasklist) {
- my $vmid = $task->{vmid};
- my $name = $task->{hostname};
+ sanitize_task_list($tasklist);
+ my ($error_count, $total_size) = aggregate_task_statistics($tasklist);
- if ($task->{state} eq 'ok') {
+ my $failed = ($error_count || $err);
- $ssize += $task->{size};
+ my $status_text = $failed ? 'backup failed' : 'backup successful';
- $html .= sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
- $vmid, $name,
- format_time($task->{backuptime}),
- format_size ($task->{size}),
- escape_html ($task->{target}));
+ if ($err) {
+ if ($err =~ /\n/) {
+ $status_text .= ": multiple problems";
} else {
- $html .= sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
- $vmid, $name, format_time($task->{backuptime}),
- escape_html ($task->{msg}));
+ $status_text .= ": $err";
+ $err = undef;
}
}
- $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
- format_time ($totaltime), format_size ($ssize));
-
- $html .= "\n</table><br><br>\n";
- my $html_log_part;
- $html_log_part .= "Detailed backup logs:<br /><br />\n";
- $html_log_part .= "<pre>\n";
- $html_log_part .= escape_html($cmdline) . "\n\n";
-
- $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre);
- foreach my $task (@$tasklist) {
- my $vmid = $task->{vmid};
- my $log = $task->{tmplog};
- if (!$log) {
- $html_log_part .= "$vmid: no log available\n\n";
- next;
- }
- if (open (TMP, "$log")) {
- while (my $line = <TMP>) {
- next if $line =~ /^status: \d+/; # not useful in mails
- if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
- $html_log_part .= encode8bit ("$vmid: <font color=red>".
- escape_html ($line) . "</font>");
- } else {
- $html_log_part .= encode8bit ("$vmid: " . escape_html ($line));
- }
- }
- } else {
- $html_log_part .= "$vmid: Could not open log file\n\n";
- }
- close (TMP);
- $html_log_part .= "\n";
- }
- $html_log_part .= escape_html($detail_post) if defined($detail_post);
- $html_log_part .= "</pre>";
- my $html_end .= "\n</body></html>\n";
- # end html part
+ my $text_log_part = "$cmdline\n\n";
+ $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
+ $text_log_part .= read_backup_task_logs($tasklist);
+ $text_log_part .= $detail_post if defined($detail_post);
- if (length($text) + length($text_log_part) +
- length($html) + length($html_log_part) < MAX_MAIL_SIZE)
+ if (length($text_log_part) > MAX_LOG_SIZE)
{
- $html .= $html_log_part;
- $text .= $text_log_part;
- } else {
- my $msg = "Log output was too long to be sent by mail. ".
+ # Let's limit the maximum length of included logs
+ $text_log_part = "Log output was too long to be sent. ".
"See Task History for details!\n";
- $text .= $msg;
- $html .= "<p>$msg</p>";
- $html .= $html_end;
- }
+ };
- my $subject = "vzdump backup status ($hostname) : $stat";
+ my $hostname = get_hostname();
- my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
- my $mailfrom = $dcconf->{email_from} || "root";
+ my $notification_props = {
+ "hostname" => $hostname,
+ "error-message" => $err,
+ "guest-table" => build_guest_table($tasklist),
+ "logs" => $text_log_part,
+ "status-text" => $status_text,
+ "total-time" => $total_time,
+ "total-size" => $total_size,
+ };
+
+ my $fields = {
+ # TODO: There is no straight-forward way yet to get the
+ # backup job id here... (I think pvescheduler would need
+ # to pass that to the vzdump call?)
+ type => "vzdump",
+ hostname => $hostname,
+ };
- PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
+ my $severity = $failed ? "error" : "info";
+ my $email_configured = $mailto && scalar(@$mailto);
+
+ if (($mode eq 'auto' && $email_configured) || $mode eq 'legacy-sendmail') {
+ if ($email_configured && ($policy eq "always" || ($policy eq "failure" && $failed))) {
+ # Start out with an empty config. Might still contain
+ # built-ins, so we need to disable/remove them.
+ my $notification_config = Proxmox::RS::Notify->parse_config('', '');
+
+ # Remove built-in matchers, since we only want to send an
+ # email to the specified recipients and nobody else.
+ for my $matcher (@{$notification_config->get_matchers()}) {
+ $notification_config->delete_matcher($matcher->{name});
+ }
+
+ # <, >, @ are not allowed in endpoint names, but that is only
+ # verified once the config is serialized. That means that
+ # we can rely on that fact that no other endpoint with this name exists.
+ my $endpoint_name = "<" . join(",", @$mailto) . ">";
+ $notification_config->add_sendmail_endpoint(
+ $endpoint_name,
+ $mailto,
+ undef,
+ undef,
+ "vzdump backup tool"
+ );
+
+ my $endpoints = [$endpoint_name];
+
+ # Add a matcher that matches all notifications, set our
+ # newly created target as a target.
+ $notification_config->add_matcher(
+ "<matcher-$endpoint_name>",
+ $endpoints,
+ );
+
+ PVE::Notify::notify(
+ $severity,
+ $subject_template,
+ $body_template,
+ $notification_props,
+ $fields,
+ $notification_config
+ );
+ }
+ } else {
+ # We use the 'new' system, or we are set to 'auto' and
+ # no email addresses were configured.
+ PVE::Notify::notify(
+ $severity,
+ $subject_template,
+ $body_template,
+ $notification_props,
+ $fields,
+ );
+ }
};
sub new {
my $defaults = read_vzdump_defaults();
- $opts->{remove} = 1 if !defined($opts->{remove});
-
foreach my $k (keys %$defaults) {
- next if $k eq 'exclude-path' || $k eq 'maxfiles'; # dealt with separately
+ next if $k eq 'exclude-path' || $k eq 'prune-backups'; # dealt with separately
if ($k eq 'dumpdir' || $k eq 'storage') {
$opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
!defined ($opts->{storage});
- } else {
- $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
+ } elsif (!defined($opts->{$k})) {
+ $opts->{$k} = $defaults->{$k};
+ } elsif ($k eq 'performance') {
+ $opts->{$k} = merge_performance($opts->{$k}, $defaults->{$k});
}
}
$opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
$skiplist = [] if !$skiplist;
- my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist };
+ my $self = bless {
+ cmdline => $cmdline,
+ opts => $opts,
+ skiplist => $skiplist,
+ }, $class;
my $findexcl = $self->{findexcl} = [];
if ($defaults->{'exclude-path'}) {
}
if ($opts->{stdexcludes}) {
- push @$findexcl, '/tmp/?*',
- '/var/tmp/?*',
- '/var/run/?*.pid';
+ push @$findexcl,
+ '/tmp/?*',
+ '/var/tmp/?*',
+ '/var/run/?*.pid',
+ ;
}
foreach my $p (@plugins) {
-
- my $pd = $p->new ($self);
+ my $pd = $p->new($self);
push @{$self->{plugins}}, $pd;
}
die "cannot use options 'storage' and 'dumpdir' at the same time\n";
}
- if (!$opts->{dumpdir} && !$opts->{storage}) {
- $opts->{storage} = 'local';
+ if (my $storage = get_storage_param($opts)) {
+ $opts->{storage} = $storage;
+ }
+
+ # Enforced by the API too, but these options might come in via defaults. Drop them if necessary.
+ if (!$opts->{storage}) {
+ delete $opts->{$_} for qw(notes-template protected);
}
my $errors = '';
+ my $add_error = sub {
+ my ($error) = @_;
+ $errors .= "\n" if $errors;
+ chomp($error);
+ $errors .= $error;
+ };
+
+ eval {
+ $self->{job_init_log} = '';
+ open my $job_init_fd, '>', \$self->{job_init_log};
+ $self->run_hook_script('job-init', undef, $job_init_fd);
+ close $job_init_fd;
+
+ PVE::Cluster::cfs_update(); # Pick up possible changes made by the hook script.
+ };
+ $add_error->($@) if $@;
if ($opts->{storage}) {
+ my $storage_cfg = PVE::Storage::config();
+ eval { PVE::Storage::activate_storage($storage_cfg, $opts->{storage}) };
+ $add_error->("could not activate storage '$opts->{storage}': $@") if $@;
+
my $info = eval { storage_info ($opts->{storage}) };
- $errors .= "could not get storage information for '$opts->{storage}': $@"
- if ($@);
- $opts->{dumpdir} = $info->{dumpdir};
- $opts->{scfg} = $info->{scfg};
- $opts->{pbs} = $info->{pbs};
- $opts->{maxfiles} //= $info->{maxfiles};
+ if (my $err = $@) {
+ $add_error->("could not get storage information for '$opts->{storage}': $err");
+ } else {
+ $opts->{dumpdir} = $info->{dumpdir};
+ $opts->{scfg} = $info->{scfg};
+ $opts->{pbs} = $info->{pbs};
+ $opts->{'prune-backups'} //= $info->{'prune-backups'};
+ }
} elsif ($opts->{dumpdir}) {
- $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
+ $add_error->("dumpdir '$opts->{dumpdir}' does not exist")
if ! -d $opts->{dumpdir};
} else {
die "internal error";
}
- $opts->{maxfiles} //= $defaults->{maxfiles};
+ $opts->{'prune-backups'} //= $defaults->{'prune-backups'};
+
+ # avoid triggering any remove code path if keep-all is set
+ $opts->{remove} = 0 if $opts->{'prune-backups'}->{'keep-all'};
if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
- $errors .= "\n" if $errors;
- $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
+ $add_error->("tmpdir '$opts->{tmpdir}' does not exist");
}
if ($errors) {
- eval { $self->sendmail([], 0, $errors); };
+ eval { $self->send_notification([], 0, $errors); };
debugmsg ('err', $@) if $@;
die "$errors\n";
}
my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
- die "missimg UPID" if !$upid; # should not happen
+ die "missing UPID" if !$upid; # should not happen
- if (!open (SERVER_FLCK, ">>$lockfile")) {
+ my $SERVER_FLCK;
+ if (!open ($SERVER_FLCK, '>>', "$lockfile")) {
debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
die "can't open lock on file '$lockfile' - $!";
}
- if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
-
+ if (!flock ($SERVER_FLCK, LOCK_EX|LOCK_NB)) {
if (!$maxwait) {
debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
die "can't acquire lock '$lockfile' (wait = 0)";
}
debugmsg('info', "trying to get global lock - waiting...", undef, 1);
-
eval {
alarm ($maxwait * 60);
local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
- if (!flock (SERVER_FLCK, LOCK_EX)) {
+ if (!flock ($SERVER_FLCK, LOCK_EX)) {
my $err = $!;
- close (SERVER_FLCK);
+ close ($SERVER_FLCK);
alarm (0);
die "$err\n";
}
}
PVE::Tools::file_set_contents($pidfile, $upid);
+
+ return $SERVER_FLCK;
}
sub run_hook_script {
my $script = $opts->{script};
return if !$script;
- if (!-x $script) {
- die "The hook script '$script' is not executable.\n";
- }
+ die "Error: The hook script '$script' does not exist.\n" if ! -f $script;
+ die "Error: The hook script '$script' is not executable.\n" if ! -x $script;
- my $cmd = "$script $phase";
+ my $cmd = [$script, $phase];
- $cmd .= " $task->{mode} $task->{vmid}" if ($task);
+ if ($task) {
+ push @$cmd, $task->{mode};
+ push @$cmd, $task->{vmid};
+ }
local %ENV;
# set immutable opts directly (so they are available in all phases)
foreach my $ek (qw(vmtype hostname target logfile)) {
$ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
}
- # FIXME: for backwards compatibility - drop with PVE 7.0
- $ENV{TARFILE} = $task->{target} if $task->{target};
run_command ($logfd, $cmd);
}
my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
$zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
}
- return ("zstd --rsyncable --threads=${zstd_threads}", 'zst');
+ return ("zstd --threads=${zstd_threads}", 'zst');
} else {
die "internal error - unknown compression option '$opt_compress'";
}
}
sub get_backup_file_list {
- my ($dir, $bkname, $exclude_fn) = @_;
+ my ($dir, $bkname) = @_;
my $bklist = [];
foreach my $fn (<$dir/${bkname}-*>) {
- next if $exclude_fn && $fn eq $exclude_fn;
-
my $archive_info = eval { PVE::Storage::archive_info($fn) } // {};
if ($archive_info->{is_std_name}) {
- my $filename = $archive_info->{filename};
+ my $path = "$dir/$archive_info->{filename}";
my $backup = {
- 'path' => "$dir/$filename",
+ 'path' => $path,
'ctime' => $archive_info->{ctime},
};
+ $backup->{mark} = "protected"
+ if -e PVE::Storage::protection_file_path($path);
push @{$bklist}, $backup;
}
}
my $opts = $self->{opts};
+ my $cfg = PVE::Storage::config();
my $vmid = $task->{vmid};
my $plugin = $task->{plugin};
- my $vmtype = $plugin->type();
$task->{backup_time} = time();
my $pbs_group_name;
my $pbs_snapshot_name;
- if ($self->{opts}->{pbs}) {
- if ($vmtype eq 'lxc') {
- $pbs_group_name = "ct/$vmid";
- } elsif ($vmtype eq 'qemu') {
- $pbs_group_name = "vm/$vmid";
- } else {
- die "pbs backup not implemented for plugin type '$vmtype'\n";
- }
- my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
- $pbs_snapshot_name = "$pbs_group_name/$btime";
- }
-
my $vmstarttime = time ();
my $logfd;
eval {
die "unable to find VM '$vmid'\n" if !$plugin;
+ my $vmtype = $plugin->type();
+
+ if ($self->{opts}->{pbs}) {
+ if ($vmtype eq 'lxc') {
+ $pbs_group_name = "ct/$vmid";
+ } elsif ($vmtype eq 'qemu') {
+ $pbs_group_name = "vm/$vmid";
+ } else {
+ die "pbs backup not implemented for plugin type '$vmtype'\n";
+ }
+ my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
+ $pbs_snapshot_name = "$pbs_group_name/$btime";
+ }
+
# for now we deny backups of a running ha managed service in *stop* mode
# as it interferes with the HA stack (started services should not stop).
if ($opts->{mode} eq 'stop' &&
my $bkname = "vzdump-$vmtype-$vmid";
my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
- my $maxfiles = $opts->{maxfiles};
+ my $prune_options = $opts->{'prune-backups'};
- if ($maxfiles && !$opts->{remove}) {
+ my $backup_limit = 0;
+ if (!$prune_options->{'keep-all'}) {
+ foreach my $keep (values %{$prune_options}) {
+ $backup_limit += $keep;
+ }
+ }
+
+ if (($backup_limit && !$opts->{remove}) || $opts->{protected}) {
my $count;
- if ($self->{opts}->{pbs}) {
- my $res = PVE::Storage::PBSPlugin::run_client_cmd($opts->{scfg}, $opts->{storage}, 'snapshots', $pbs_group_name);
- $count = scalar(@$res);
+ my $protected_count;
+ if (my $storeid = $opts->{storage}) {
+ my @backups = grep {
+ !$_->{subtype} || $_->{subtype} eq $vmtype
+ } PVE::Storage::volume_list($cfg, $storeid, $vmid, 'backup')->@*;
+
+ $count = grep { !$_->{protected} } @backups;
+ $protected_count = scalar(@backups) - $count;
} else {
- my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
- $count = scalar(@$bklist);
+ $count = grep { !$_->{mark} || $_->{mark} ne "protected" } get_backup_file_list($opts->{dumpdir}, $bkname)->@*;
+ }
+
+ if ($opts->{protected}) {
+ my $max_protected = PVE::Storage::get_max_protected_backups(
+ $opts->{scfg},
+ $opts->{storage},
+ );
+ if ($max_protected > -1 && $protected_count >= $max_protected) {
+ die "The number of protected backups per guest is limited to $max_protected ".
+ "on storage '$opts->{storage}'\n";
+ }
+ } elsif ($count >= $backup_limit) {
+ die "There is a max backup limit of $backup_limit enforced by the target storage ".
+ "or the vzdump parameters. Either increase the limit or delete old backups.\n";
}
- die "There is a max backup limit of ($maxfiles) enforced by the".
- " target storage or the vzdump parameters.".
- " Either increase the limit or delete old backup(s).\n"
- if $count >= $maxfiles;
}
if (!$self->{opts}->{pbs}) {
$task->{vmtype} = $vmtype;
+ my $pid = $$;
if ($opts->{tmpdir}) {
- $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp$$";
+ $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp${pid}_$vmid/";
} elsif ($self->{opts}->{pbs}) {
- $task->{tmpdir} = "/var/tmp/vzdumptmp$$"; #fixme
+ $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
} else {
# dumpdir is posix? then use it as temporary dir
my $info = get_mount_info($opts->{dumpdir});
grep ($_ eq $info->{fstype}, @posix_filesystems)) {
$task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
} else {
- $task->{tmpdir} = "/var/tmp/vzdumptmp$$";
+ $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
"using $task->{tmpdir} for temporary files", $logfd);
}
$task->{mode} = $mode;
debugmsg ('info', "backup mode: $mode", $logfd);
-
- debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
- if $opts->{bwlimit};
-
+ debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd) if $opts->{bwlimit};
debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
if ($mode eq 'stop') {
-
$plugin->prepare ($task, $vmid, $mode);
$self->run_hook_script ('backup-start', $task, $logfd);
if ($running) {
- debugmsg ('info', "stopping vm", $logfd);
+ debugmsg ('info', "stopping virtual guest", $logfd);
$task->{vmstoptime} = time();
$self->run_hook_script ('pre-stop', $task, $logfd);
$plugin->stop_vm ($task, $vmid);
} elsif ($mode eq 'suspend') {
-
$plugin->prepare ($task, $vmid, $mode);
$self->run_hook_script ('backup-start', $task, $logfd);
}
} elsif ($mode eq 'snapshot') {
-
$self->run_hook_script ('backup-start', $task, $logfd);
my $snapshot_count = $task->{snapshot_count} || 0;
debugmsg ('info', "archive file size: $cs", $logfd);
}
- # purge older backup
- if ($maxfiles && $opts->{remove}) {
+ # Mark as protected before pruning.
+ if (my $storeid = $opts->{storage}) {
+ my $volname = $opts->{pbs} ? $task->{target} : basename($task->{target});
+ my $volid = "${storeid}:backup/${volname}";
- if ($self->{opts}->{pbs}) {
- my $args = [$pbs_group_name, '--quiet', '1', '--keep-last', $maxfiles];
- my $logfunc = sub { my $line = shift; debugmsg ('info', $line, $logfd); };
- PVE::Storage::PBSPlugin::run_raw_client_cmd(
- $opts->{scfg}, $opts->{storage}, 'prune', $args, logfunc => $logfunc);
- } else {
- my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{target});
- $bklist = [ sort { $b->{ctime} <=> $a->{ctime} } @$bklist ];
+ if ($opts->{'notes-template'} && $opts->{'notes-template'} ne '') {
+ debugmsg('info', "adding notes to backup", $logfd);
+ my $notes = eval { $generate_notes->($opts->{'notes-template'}, $task); };
+ if (my $err = $@) {
+ debugmsg('warn', "unable to add notes - $err", $logfd);
+ } else {
+ eval { PVE::Storage::update_volume_attribute($cfg, $volid, 'notes', $notes) };
+ debugmsg('warn', "unable to add notes - $@", $logfd) if $@;
+ }
+ }
- while (scalar (@$bklist) >= $maxfiles) {
- my $d = pop @$bklist;
- my $archive_path = $d->{path};
+ if ($opts->{protected}) {
+ debugmsg('info', "marking backup as protected", $logfd);
+ eval { PVE::Storage::update_volume_attribute($cfg, $volid, 'protected', 1) };
+ die "unable to set protected flag - $@\n" if $@;
+ }
+ }
+
+ if ($opts->{remove}) {
+ my $keepstr = join(', ', map { "$_=$prune_options->{$_}" } sort keys %$prune_options);
+ debugmsg ('info', "prune older backups with retention: $keepstr", $logfd);
+ my $pruned = 0;
+ if (!defined($opts->{storage})) {
+ my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
+
+ PVE::Storage::prune_mark_backup_group($bklist, $prune_options);
+
+ foreach my $prune_entry (@{$bklist}) {
+ next if $prune_entry->{mark} ne 'remove';
+ $pruned++;
+ my $archive_path = $prune_entry->{path};
debugmsg ('info', "delete old backup '$archive_path'", $logfd);
PVE::Storage::archive_remove($archive_path);
}
+ } else {
+ my $pruned_list = PVE::Storage::prune_backups(
+ $cfg,
+ $opts->{storage},
+ $prune_options,
+ $vmid,
+ $vmtype,
+ 0,
+ sub { debugmsg($_[0], $_[1], $logfd) },
+ );
+ $pruned = scalar(grep { $_->{mark} eq 'remove' } $pruned_list->@*);
}
+ my $log_pruned_extra = $pruned > 0 ? " not covered by keep-retention policy" : "";
+ debugmsg ('info', "pruned $pruned backup(s)${log_pruned_extra}", $logfd);
}
$self->run_hook_script ('backup-end', $task, $logfd);
debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
+ debugmsg('warn', $@) if $@; # message already contains command with phase name
} else {
$task->{state} = 'ok';
if ($task->{tmplog}) {
if ($self->{opts}->{pbs}) {
if ($task->{state} eq 'ok') {
- my $param = [$pbs_snapshot_name, $task->{tmplog}];
- PVE::Storage::PBSPlugin::run_raw_client_cmd(
- $opts->{scfg}, $opts->{storage}, 'upload-log', $param, errmsg => "upload log failed");
+ eval {
+ PVE::Storage::PBSPlugin::run_raw_client_cmd(
+ $opts->{scfg},
+ $opts->{storage},
+ 'upload-log',
+ [ $pbs_snapshot_name, $task->{tmplog} ],
+ errmsg => "uploading backup task log failed",
+ outfunc => sub {},
+ );
+ };
+ debugmsg('warn', "$@") if $@; # $@ contains already error prefix
}
} elsif ($task->{logfile}) {
system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
}
eval { $self->run_hook_script ('log-end', $task); };
+ debugmsg('warn', $@) if $@; # message already contains command with phase name
die $err if $err && $err =~ m/^interrupted by signal$/;
}
my $opts = $self->{opts};
debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
- debugmsg ('info', "skip external VMs: " . join(', ', @{$self->{skiplist}}))
- if scalar(@{$self->{skiplist}});
+
+ if (scalar(@{$self->{skiplist}})) {
+ my $skip_string = join(', ', sort { $a <=> $b } @{$self->{skiplist}});
+ debugmsg ('info', "skip external VMs: $skip_string");
+ }
my $tasklist = [];
my $vzdump_plugins = {};
}
my $vmlist = PVE::Cluster::get_vmlist();
- foreach my $vmid (@{$opts->{vmids}}) {
- my $guest_type = $vmlist->{ids}->{$vmid}->{type};
- my $plugin = $vzdump_plugins->{$guest_type};
- next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all});
+ my $vmids = [ sort { $a <=> $b } @{$opts->{vmids}} ];
+ foreach my $vmid (@{$vmids}) {
+ my $plugin;
+ if (defined($vmlist->{ids}->{$vmid})) {
+ my $guest_type = $vmlist->{ids}->{$vmid}->{type};
+ $plugin = $vzdump_plugins->{$guest_type};
+ next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all});
+ }
push @$tasklist, {
mode => $opts->{mode},
plugin => $plugin,
};
my $err = $@;
- $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
-
if ($err) {
+ eval { $self->run_hook_script ('job-abort', undef, $job_end_fd); };
+ $err .= $@ if $@;
debugmsg ('err', "Backup job failed - $err", undef, 1);
} else {
if ($errcount) {
my $totaltime = time() - $starttime;
- eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); };
+ eval {
+ # otherwise $self->send_notification() will interpret it as multiple problems
+ my $chomped_err = $err;
+ chomp($chomped_err) if $chomped_err;
+
+ $self->send_notification(
+ $tasklist,
+ $totaltime,
+ $chomped_err,
+ $self->{job_init_log} . $job_start_log,
+ $job_end_log,
+ );
+ };
debugmsg ('err', $@) if $@;
die $err if $err;
return defined($confdesc->{$key});
}
+# NOTE it might make sense to merge this and verify_vzdump_parameters(), but one
+# needs to adapt command_line() in guest-common's PVE/VZDump/Common.pm and detect
+# a second parsing attempt, because verify_vzdump_parameters() is called twice
+# during the update_job API call.
+sub parse_mailto_exclude_path {
+ my ($param) = @_;
+
+ # exclude-path list need to be 0 separated or be an array
+ if (defined($param->{'exclude-path'})) {
+ my $expaths;
+ if (ref($param->{'exclude-path'}) eq 'ARRAY') {
+ $expaths = $param->{'exclude-path'};
+ } else {
+ $expaths = [split(/\0/, $param->{'exclude-path'} || '')];
+ }
+ $param->{'exclude-path'} = $expaths;
+ }
+
+ if (defined($param->{mailto})) {
+ my @mailto = PVE::Tools::split_list(extract_param($param, 'mailto'));
+ $param->{mailto} = [ @mailto ];
+ }
+
+ return;
+}
+
sub verify_vzdump_parameters {
my ($param, $check_missing) = @_;
raise_param_exc({ pool => "option conflicts with option 'vmid'"})
if $param->{pool} && $param->{vmid};
- $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
+ raise_param_exc({ 'prune-backups' => "option conflicts with option 'maxfiles'"})
+ if defined($param->{'prune-backups'}) && defined($param->{maxfiles});
- warn "option 'size' is deprecated and will be removed in a future " .
- "release, please update your script/configuration!\n"
- if defined($param->{size});
+ $parse_prune_backups_maxfiles->($param, 'CLI parameters');
+ parse_fleecing($param);
+ parse_performance($param);
+
+ if (my $template = $param->{'notes-template'}) {
+ eval { $verify_notes_template->($template); };
+ raise_param_exc({'notes-template' => $@}) if $@;
+ }
+
+ $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
return if !$check_missing;
sub get_included_guests {
my ($job) = @_;
- my $nodename = PVE::INotify::nodename();
my $vmids = [];
my $vmids_per_node = {};
$vmids = check_vmids(@$vmids);
for my $vmid (@$vmids) {
- my $node = $vmlist->{ids}->{$vmid}->{node};
- next if (defined $job->{node} && $job->{node} ne $node);
+ if (defined($vmlist->{ids}->{$vmid})) {
+ my $node = $vmlist->{ids}->{$vmid}->{node};
+ next if (defined $job->{node} && $job->{node} ne $node);
- push @{$vmids_per_node->{$node}}, $vmid;
+ push @{$vmids_per_node->{$node}}, $vmid;
+ } else {
+ push @{$vmids_per_node->{''}}, $vmid;
+ }
}
return $vmids_per_node;
projects / pve-manager.git / blobdiff