use warnings;
use Fcntl ':flock';
use PVE::Exception qw(raise_param_exc);
-use PVE::SafeSyslog;
use IO::File;
use IO::Select;
use IPC::Open3;
-use POSIX qw(strftime);
use File::Path;
use PVE::RPCEnvironment;
use PVE::Storage;
use PVE::Cluster qw(cfs_read_file);
-use Time::localtime;
+use POSIX qw(strftime);
use Time::Local;
use PVE::JSONSchema qw(get_standard_option);
use PVE::HA::Env::PVE2;
use PVE::HA::Config;
+use PVE::VZDump::Plugin;
my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
my $lockfile = '/var/run/vzdump.lock';
-
my $pidfile = '/var/run/vzdump.pid';
-
my $logdir = '/var/log/vzdump';
my @plugins = qw();
},
mailto => {
type => 'string', format => 'string-list',
- description => "",
+ description => "Comma-separated list of email addresses that should" .
+ " receive email notifications.",
optional => 1,
},
mailnotification => {
}),
stop => {
type => 'boolean',
- description => "Stop runnig backup jobs on this host.",
+ description => "Stop running backup jobs on this host.",
optional => 1,
default => 0,
},
size => {
type => 'integer',
- description => "LVM snapshot size in MB.",
+ description => "Unused, will be removed in a future release.",
optional => 1,
minimum => 500,
default => 1024,
optional => 1,
default => 1,
},
+ pool => {
+ type => 'string',
+ description => 'Backup all known guest systems included in the specified pool.',
+ optional => 1,
+ }
};
# Load available plugins
$plug->import ();
push @plugins, $plug;
} else {
- warn $@;
+ die $@;
}
}
}
# helper functions
-my $debugstattxt = {
- err => 'ERROR:',
- info => 'INFO:',
- warn => 'WARN:',
-};
-
sub debugmsg {
my ($mtype, $msg, $logfd, $syslog) = @_;
- chomp $msg;
-
- return if !$msg;
-
- my $pre = $debugstattxt->{$mtype} || $debugstattxt->{'err'};
-
- my $timestr = strftime ("%b %d %H:%M:%S", CORE::localtime);
-
- syslog ($mtype eq 'info' ? 'info' : 'err', "$pre $msg") if $syslog;
-
- foreach my $line (split (/\n/, $msg)) {
- print STDERR "$pre $line\n";
- print $logfd "$timestr $pre $line\n" if $logfd;
- }
+ PVE::VZDump::Plugin::debugmsg(@_);
}
sub run_command {
my $cfg = PVE::Storage::config();
my $scfg = PVE::Storage::storage_config($cfg, $storage);
my $type = $scfg->{type};
-
- die "can't use storage type '$type' for backup\n"
- if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'));
- die "can't use storage '$storage' for backups - wrong content type\n"
+
+ die "can't use storage type '$type' for backup\n"
+ if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'
+ || $type eq 'cifs' || $type eq 'cephfs'));
+ die "can't use storage '$storage' for backups - wrong content type\n"
if (!$scfg->{content}->{backup});
PVE::Storage::activate_storage($cfg, $storage);
} else {
my $gb = $mb / 1024;
return sprintf ("%.2fGB", $gb);
- }
+ }
}
sub format_time {
if (my $excludes = $res->{'exclude-path'}) {
$res->{'exclude-path'} = PVE::Tools::split_args($excludes);
}
+ if (defined($res->{mailto})) {
+ my @mailto = PVE::Tools::split_list($res->{mailto});
+ $res->{mailto} = [ @mailto ];
+ }
foreach my $key (keys %$defaults) {
$res->{$key} = $defaults->{$key} if !defined($res->{$key});
}
sub sendmail {
- my ($self, $tasklist, $totaltime, $err) = @_;
+ my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
my $opts = $self->{opts};
return if (!$ecount && !$err && ($notify eq 'failure'));
my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
- $stat .= ": $err" if $err;
+ if ($err) {
+ if ($err =~ /\n/) {
+ $stat .= ": multiple problems";
+ } else {
+ $stat .= ": $err";
+ $err = undef;
+ }
+ }
my $hostname = `hostname -f` || PVE::INotify::nodename();
chomp $hostname;
# text part
- my $text = sprintf ("%-10s %-6s %10s %10s %s\n", qw(VMID STATUS TIME SIZE FILENAME));
+ my $text = $err ? "$err\n\n" : '';
+ $text .= sprintf ("%-10s %-6s %10s %10s %s\n", qw(VMID STATUS TIME SIZE FILENAME));
foreach my $task (@$tasklist) {
my $vmid = $task->{vmid};
if ($task->{state} eq 'ok') {
}
}
- $text .= "Detailed backup logs:\n\n";
+ $text .= "\nDetailed backup logs:\n\n";
$text .= "$cmdline\n\n";
+ $text .= $detail_pre . "\n" if defined($detail_pre);
foreach my $task (@$tasklist) {
my $vmid = $task->{vmid};
my $log = $task->{tmplog};
close (TMP);
$text .= "\n";
}
+ $text .= $detail_post if defined($detail_post);
# html part
my $html = "<html><body>\n";
+ $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
$html .= "<table border=1 cellpadding=3>\n";
$html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
$html .= "<pre>\n";
$html .= escape_html($cmdline) . "\n\n";
+ $html .= escape_html($detail_pre) . "\n" if defined($detail_pre);
foreach my $task (@$tasklist) {
my $vmid = $task->{vmid};
my $log = $task->{tmplog};
close (TMP);
$html .= "\n";
}
+ $html .= escape_html($detail_post) if defined($detail_post);
$html .= "</pre></body></html>\n";
# end html part
}
if ($opts->{stdexcludes}) {
- push @$findexcl, '/var/log/?*',
- '/tmp/?*',
+ push @$findexcl, '/tmp/?*',
'/var/tmp/?*',
'/var/run/?*.pid';
}
$opts->{storage} = 'local';
}
+ my $errors = '';
+
if ($opts->{storage}) {
- my $info = storage_info ($opts->{storage});
+ my $info = eval { storage_info ($opts->{storage}) };
+ $errors .= "could not get storage information for '$opts->{storage}': $@"
+ if ($@);
$opts->{dumpdir} = $info->{dumpdir};
- $maxfiles = $info->{maxfiles} if !defined($maxfiles) && defined($info->{maxfiles});
+ $maxfiles //= $info->{maxfiles};
} elsif ($opts->{dumpdir}) {
- die "dumpdir '$opts->{dumpdir}' does not exist\n"
+ $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
if ! -d $opts->{dumpdir};
} else {
- die "internal error";
+ die "internal error";
}
if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
- die "tmpdir '$opts->{tmpdir}' does not exist\n";
+ $errors .= "\n" if $errors;
+ $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
+ }
+
+ if ($errors) {
+ eval { $self->sendmail([], 0, $errors); };
+ debugmsg ('err', $@) if $@;
+ die "$errors\n";
}
$opts->{maxfiles} = $maxfiles if defined($maxfiles);
}
-sub get_lvm_mapping {
-
- my $devmapper;
-
- my $cmd = ['lvs', '--units', 'm', '--separator', ':', '--noheadings',
- '-o', 'vg_name,lv_name,lv_size' ];
-
- my $parser = sub {
- my $line = shift;
- if ($line =~ m|^\s*(\S+):(\S+):(\d+(\.\d+))[Mm]$|) {
- my $vg = $1;
- my $lv = $2;
- $devmapper->{"/dev/$vg/$lv"} = [$vg, $lv];
- my $qlv = $lv;
- $qlv =~ s/-/--/g;
- my $qvg = $vg;
- $qvg =~ s/-/--/g;
- $devmapper->{"/dev/mapper/$qvg-$qlv"} = [$vg, $lv];
- }
- };
-
- eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
- warn $@ if $@;
-
- return $devmapper;
-}
-
sub get_mount_info {
my ($dir) = @_;
return $res;
}
-sub get_lvm_device {
- my ($dir, $mapping) = @_;
-
- my $info = get_mount_info($dir);
-
- return undef if !$info;
-
- my $dev = $info->{device};
-
- my ($vg, $lv);
-
- ($vg, $lv) = @{$mapping->{$dev}} if defined $mapping->{$dev};
-
- return wantarray ? ($dev, $info->{mountpoint}, $vg, $lv, $info->{fstype}) : $dev;
-}
-
sub getlock {
my ($self, $upid) = @_;
my $fh;
-
+
my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
-
+
die "missimg UPID" if !$upid; # should not happen
if (!open (SERVER_FLCK, ">>$lockfile")) {
if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
if (!$maxwait) {
- debugmsg ('err', "can't aquire lock '$lockfile' (wait = 0)", undef, 1);
- die "can't aquire lock '$lockfile' (wait = 0)";
+ debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
+ die "can't acquire lock '$lockfile' (wait = 0)";
}
debugmsg('info', "trying to get global lock - waiting...", undef, 1);
eval {
alarm ($maxwait * 60);
-
+
local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
if (!flock (SERVER_FLCK, LOCK_EX)) {
alarm (0);
};
alarm (0);
-
+
my $err = $@;
-
+
if ($err) {
- debugmsg ('err', "can't aquire lock '$lockfile' - $err", undef, 1);
- die "can't aquire lock '$lockfile' - $err";
+ debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
+ die "can't acquire lock '$lockfile' - $err";
}
debugmsg('info', "got global lock", undef, 1);
return ('lzop', 'lzo');
} elsif ($opt_compress eq 'gzip') {
if ($opts->{pigz} > 0) {
- # As default use int((#cores + 1)/2), we need #cores+1 for the case that #cores = 1
- my $cores = POSIX::sysconf(84);
- my $pigz_threads = ($opts->{pigz} > 1) ? $opts->{pigz} : int(($cores + 1)/2);
+ my $pigz_threads = $opts->{pigz};
+ if ($pigz_threads == 1) {
+ my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
+ $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
+ }
return ("pigz -p ${pigz_threads}", 'gz');
} else {
- return ('gzip', 'gz');
+ return ('gzip --rsyncable', 'gz');
}
} else {
die "internal error - unknown compression option '$opt_compress'";
next if $exclude_fn && $fn eq $exclude_fn;
if ($fn =~ m!/(${bkname}-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?)))$!) {
$fn = "$dir/$1"; # untaint
- my $t = timelocal ($7, $6, $5, $4, $3 - 1, $2 - 1900);
+ my $t = timelocal ($7, $6, $5, $4, $3 - 1, $2);
push @$bklist, [$fn, $t];
}
}
return $bklist;
}
-
+
sub exec_backup_task {
my ($self, $task) = @_;
-
+
my $opts = $self->{opts};
my $vmid = $task->{vmid};
my $plugin = $task->{plugin};
my $vmstarttime = time ();
-
+
my $logfd;
my $cleanup = {};
- my $vmstoptime = 0;
+ my $log_vm_online_again = sub {
+ return if !defined($task->{vmstoptime});
+ $task->{vmconttime} //= time();
+ my $delay = $task->{vmconttime} - $task->{vmstoptime};
+ debugmsg ('info', "guest is online again after $delay seconds", $logfd);
+ };
eval {
die "unable to find VM '$vmid'\n" if !$plugin;
# for now we deny backups of a running ha managed service in *stop* mode
- # as it interferes with the HA stack (enabled services should not stop).
+ # as it interferes with the HA stack (started services should not stop).
if ($opts->{mode} eq 'stop' &&
- PVE::HA::Config::vm_is_ha_managed($vmid, 'enabled'))
+ PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
{
die "Cannot execute a backup with stop mode on a HA managed and".
" enabled Service. Use snapshot mode or disable the Service.\n";
my $tmplog = "$logdir/$vmtype-$vmid.log";
- my $lt = localtime();
-
my $bkname = "vzdump-$vmtype-$vmid";
- my $basename = sprintf "${bkname}-%04d_%02d_%02d-%02d_%02d_%02d",
- $lt->year + 1900, $lt->mon + 1, $lt->mday,
- $lt->hour, $lt->min, $lt->sec;
+ my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime());
my $maxfiles = $opts->{maxfiles};
if ($maxfiles && !$opts->{remove}) {
my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
- die "only $maxfiles backup(s) allowed - please consider to remove old backup files.\n"
+ die "There is a max backup limit of ($maxfiles) enforced by the".
+ " target storage or the vzdump parameters.".
+ " Either increase the limit or delete old backup(s).\n"
if scalar(@$bklist) >= $maxfiles;
}
$task->{vmtype} = $vmtype;
if ($opts->{tmpdir}) {
- $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp$$";
+ $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp$$";
} else {
# dumpdir is posix? then use it as temporary dir
my $info = get_mount_info($opts->{dumpdir});
- if ($vmtype eq 'qemu' ||
+ if ($vmtype eq 'qemu' ||
grep ($_ eq $info->{fstype}, @posix_filesystems)) {
$task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
} else {
unlink $logfile;
- debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
+ debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
+ debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
$plugin->set_logfd ($logfd);
# prepare
- my $mode = $running ? $opts->{mode} : 'stop';
+ my $mode = $running ? $task->{mode} : 'stop';
if ($mode eq 'snapshot') {
my %saved_task = %$task;
debugmsg ('info', $err, $logfd);
debugmsg ('info', "trying 'suspend' mode instead", $logfd);
$mode = 'suspend'; # so prepare is called again below
- %$task = %saved_task;
+ %$task = %saved_task;
}
}
+ $cleanup->{prepared} = 1;
+
$task->{mode} = $mode;
debugmsg ('info', "backup mode: $mode", $logfd);
if ($running) {
debugmsg ('info', "stopping vm", $logfd);
- $vmstoptime = time ();
+ $task->{vmstoptime} = time();
$self->run_hook_script ('pre-stop', $task, $logfd);
$plugin->stop_vm ($task, $vmid);
$cleanup->{restart} = 1;
}
-
+
} elsif ($mode eq 'suspend') {
}
debugmsg ('info', "suspend vm", $logfd);
- $vmstoptime = time ();
+ $task->{vmstoptime} = time ();
$self->run_hook_script ('pre-stop', $task, $logfd);
$plugin->suspend_vm ($task, $vmid);
$cleanup->{resume} = 1;
$cleanup->{resume} = 0;
$self->run_hook_script('pre-restart', $task, $logfd);
$plugin->resume_vm($task, $vmid);
- my $delay = time () - $vmstoptime;
- debugmsg('info', "vm is online again after $delay seconds", $logfd);
+ $self->run_hook_script('post-restart', $task, $logfd);
+ $log_vm_online_again->();
}
-
+
} elsif ($mode eq 'snapshot') {
$self->run_hook_script ('backup-start', $task, $logfd);
if ($snapshot_count > 1) {
debugmsg ('info', "suspend vm to make snapshot", $logfd);
- $vmstoptime = time ();
+ $task->{vmstoptime} = time ();
$plugin->suspend_vm ($task, $vmid);
$cleanup->{resume} = 1;
}
debugmsg ('info', "resume vm", $logfd);
$cleanup->{resume} = 0;
$plugin->resume_vm ($task, $vmid);
- my $delay = time () - $vmstoptime;
- debugmsg ('info', "vm is online again after $delay seconds", $logfd);
+ $log_vm_online_again->();
}
+ $self->run_hook_script ('post-restart', $task, $logfd);
+
} else {
die "internal error - unknown mode '$mode'\n";
}
# assemble archive image
$plugin->assemble ($task, $vmid);
-
- # produce archive
+
+ # produce archive
if ($opts->{stdout}) {
debugmsg ('info', "sending archive to stdout", $logfd);
# determine size
$task->{size} = (-s $task->{tarfile}) || 0;
- my $cs = format_size ($task->{size});
+ my $cs = format_size ($task->{size});
debugmsg ('info', "archive file size: $cs", $logfd);
# purge older backup
warn $@ if $@;
}
- if (defined($task->{mode})) {
+ if ($cleanup->{prepared}) {
# only call cleanup when necessary (when prepare was executed)
eval { $plugin->cleanup ($task, $vmid) };
warn $@ if $@;
eval { $plugin->set_logfd (undef); };
warn $@ if $@;
- if ($cleanup->{resume} || $cleanup->{restart}) {
- eval {
+ if ($cleanup->{resume} || $cleanup->{restart}) {
+ eval {
$self->run_hook_script ('pre-restart', $task, $logfd);
if ($cleanup->{resume}) {
debugmsg ('info', "resume vm", $logfd);
debugmsg ('info', "restarting vm", $logfd);
$plugin->start_vm ($task, $vmid);
}
- }
+ }
+ $self->run_hook_script ('post-restart', $task, $logfd);
};
my $err = $@;
if ($err) {
warn $err;
} else {
- my $delay = time () - $vmstoptime;
- debugmsg ('info', "vm is online again after $delay seconds", $logfd);
+ $log_vm_online_again->();
}
}
}
$task->{state} = 'err';
$task->{msg} = $err;
debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
+ debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
$task->{state} = 'ok';
my $tstr = format_time ($delay);
debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
+ debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
}
close ($logfd) if $logfd;
-
+
if ($task->{tmplog} && $task->{logfile}) {
- system ("cp '$task->{tmplog}' '$task->{logfile}'");
+ system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
}
eval { $self->run_hook_script ('log-end', $task); };
debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
debugmsg ('info', "skip external VMs: " . join(', ', @{$self->{skiplist}}))
if scalar(@{$self->{skiplist}});
-
+
my $tasklist = [];
if ($opts->{all}) {
foreach my $vmid (sort @$vmlist) {
next if grep { $_ eq $vmid } @{$opts->{exclude}};
next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], 1);
- push @$tasklist, { vmid => $vmid, state => 'todo', plugin => $plugin };
+ push @$tasklist, { vmid => $vmid, state => 'todo', plugin => $plugin, mode => $opts->{mode} };
}
}
} else {
}
}
$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ]);
- push @$tasklist, { vmid => $vmid, state => 'todo', plugin => $plugin };
+ push @$tasklist, { vmid => $vmid, state => 'todo', plugin => $plugin, mode => $opts->{mode} };
}
}
+ # Use in-memory files for the outer hook logs to pass them to sendmail.
+ my $job_start_log = '';
+ my $job_end_log = '';
+ open my $job_start_fd, '>', \$job_start_log;
+ open my $job_end_fd, '>', \$job_end_log;
+
my $starttime = time();
my $errcount = 0;
eval {
- $self->run_hook_script ('job-start');
+ $self->run_hook_script ('job-start', undef, $job_start_fd);
foreach my $task (@$tasklist) {
$self->exec_backup_task ($task);
$errcount += 1 if $task->{state} ne 'ok';
}
- $self->run_hook_script ('job-end');
+ $self->run_hook_script ('job-end', undef, $job_end_fd);
};
my $err = $@;
- $self->run_hook_script ('job-abort') if $err;
+ $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
if ($err) {
debugmsg ('err', "Backup job failed - $err", undef, 1);
}
}
+ close $job_start_fd;
+ close $job_end_fd;
+
my $totaltime = time() - $starttime;
- eval { $self->sendmail ($tasklist, $totaltime); };
+ eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); };
debugmsg ('err', $@) if $@;
die $err if $err;
- die "job errors\n" if $errcount;
+ die "job errors\n" if $errcount;
unlink $pidfile;
}
raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
if $param->{exclude} && $param->{vmid};
- $param->{all} = 1 if defined($param->{exclude});
+ raise_param_exc({ pool => "option conflicts with option 'vmid'"})
+ if $param->{pool} && $param->{vmid};
+
+ $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
+
+ warn "option 'size' is deprecated and will be removed in a future " .
+ "release, please update your script/configuration!\n"
+ if defined($param->{size});
return if !$check_missing;
raise_param_exc({ vmid => "property is missing"})
- if !($param->{all} || $param->{stop}) && !$param->{vmid};
+ if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
}
my $task = PVE::Tools::upid_decode($upid);
- if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
+ if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
kill(15, $task->{pid});
# wait max 15 seconds to shut down (else, do nothing for now)
last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
sleep (1);
}
- die "stoping backup process $task->{pid} failed\n" if $i == 0;
+ die "stopping backup process $task->{pid} failed\n" if $i == 0;
}
}
projects / pve-manager.git / blobdiff