use strict;
use warnings;
+
use Fcntl ':flock';
-use PVE::Exception qw(raise_param_exc);
+use File::Path;
use IO::File;
use IO::Select;
use IPC::Open3;
-use File::Path;
-use PVE::RPCEnvironment;
-use PVE::Storage;
-use PVE::Cluster qw(cfs_read_file);
-use PVE::DataCenterConfig;
use POSIX qw(strftime);
use Time::Local;
-use PVE::JSONSchema qw(get_standard_option);
-use PVE::HA::Env::PVE2;
+
+use PVE::Cluster qw(cfs_read_file);
+use PVE::DataCenterConfig;
+use PVE::Exception qw(raise_param_exc);
use PVE::HA::Config;
-use PVE::VZDump::Plugin;
+use PVE::HA::Env::PVE2;
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::RPCEnvironment;
+use PVE::Storage;
use PVE::VZDump::Common;
+use PVE::VZDump::Plugin;
+use PVE::Tools qw(extract_param);
+use PVE::API2Tools;
my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
die "can't use storage type '$type' for backup\n"
if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'
- || $type eq 'cifs' || $type eq 'cephfs'));
+ || $type eq 'cifs' || $type eq 'cephfs' || $type eq 'pbs'));
die "can't use storage '$storage' for backups - wrong content type\n"
if (!$scfg->{content}->{backup});
PVE::Storage::activate_storage($cfg, $storage);
- return {
- dumpdir => PVE::Storage::get_backup_dir($cfg, $storage),
- maxfiles => $scfg->{maxfiles},
- };
+ if ($type eq 'pbs') {
+ return {
+ scfg => $scfg,
+ maxfiles => $scfg->{maxfiles},
+ pbs => 1,
+ };
+ } else {
+ return {
+ scfg => $scfg,
+ dumpdir => PVE::Storage::get_backup_dir($cfg, $storage),
+ maxfiles => $scfg->{maxfiles},
+ };
+ }
}
sub format_size {
}
my $mb = $size / (1024*1024);
-
if ($mb < 1024) {
return int ($mb) . "MB";
- } else {
- my $gb = $mb / 1024;
+ }
+ my $gb = $mb / 1024;
+ if ($gb < 1024) {
return sprintf ("%.2fGB", $gb);
}
+ my $tb = $gb / 1024;
+ return sprintf ("%.2fTB", $tb);
}
sub format_time {
push @{$self->{plugins}}, $pd;
}
+ if (defined($opts->{storage}) && $opts->{stdout}) {
+ die "unable to use option 'storage' with option 'stdout'\n";
+ }
+
if (!$opts->{dumpdir} && !$opts->{storage}) {
$opts->{storage} = 'local';
}
$errors .= "could not get storage information for '$opts->{storage}': $@"
if ($@);
$opts->{dumpdir} = $info->{dumpdir};
+ $opts->{scfg} = $info->{scfg};
+ $opts->{pbs} = $info->{pbs};
$maxfiles //= $info->{maxfiles};
} elsif ($opts->{dumpdir}) {
$errors .= "dumpdir '$opts->{dumpdir}' does not exist"
my $opts = $self->{opts};
my $script = $opts->{script};
-
return if !$script;
+ if (!-x $script) {
+ die "The hook script '$script' is not executable.\n";
+ }
+
my $cmd = "$script $phase";
$cmd .= " $task->{mode} $task->{vmid}" if ($task);
local %ENV;
-
# set immutable opts directly (so they are available in all phases)
$ENV{STOREID} = $opts->{storage} if $opts->{storage};
$ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
} else {
return ('gzip --rsyncable', 'gz');
}
+ } elsif ($opt_compress eq 'zstd') {
+ my $zstd_threads = $opts->{zstd} // 1;
+ if ($zstd_threads == 0) {
+ my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
+ $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
+ }
+ return ("zstd --rsyncable --threads=${zstd_threads}", 'zst');
} else {
die "internal error - unknown compression option '$opt_compress'";
}
my $bklist = [];
foreach my $fn (<$dir/${bkname}-*>) {
next if $exclude_fn && $fn eq $exclude_fn;
- if ($fn =~ m!/(${bkname}-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?)))$!) {
+ if ($fn =~ m!/(${bkname}-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?)))$!) {
$fn = "$dir/$1"; # untaint
my $t = timelocal ($7, $6, $5, $4, $3 - 1, $2);
push @$bklist, [$fn, $t];
my $vmid = $task->{vmid};
my $plugin = $task->{plugin};
+ my $vmtype = $plugin->type();
+
+ $task->{backup_time} = time();
+
+ my $pbs_group_name;
+ my $pbs_snapshot_name;
+
+ if ($self->{opts}->{pbs}) {
+ if ($vmtype eq 'lxc') {
+ $pbs_group_name = "ct/$vmid";
+ } elsif ($vmtype eq 'qemu') {
+ $pbs_group_name = "vm/$vmid";
+ } else {
+ die "pbs backup not implemented for plugin type '$vmtype'\n";
+ }
+ my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
+ $pbs_snapshot_name = "$pbs_group_name/$btime";
+ }
my $vmstarttime = time ();
" enabled Service. Use snapshot mode or disable the Service.\n";
}
- my $vmtype = $plugin->type();
-
my $tmplog = "$logdir/$vmtype-$vmid.log";
my $bkname = "vzdump-$vmtype-$vmid";
- my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime());
+ my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
my $maxfiles = $opts->{maxfiles};
if ($maxfiles && !$opts->{remove}) {
- my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
+ my $count;
+ if ($self->{opts}->{pbs}) {
+ my $res = PVE::Storage::PBSPlugin::run_client_cmd($opts->{scfg}, $opts->{storage}, 'snapshots', $pbs_group_name);
+ $count = scalar(@$res);
+ } else {
+ my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
+ $count = scalar(@$bklist);
+ }
die "There is a max backup limit of ($maxfiles) enforced by the".
- " target storage or the vzdump parameters.".
- " Either increase the limit or delete old backup(s).\n"
- if scalar(@$bklist) >= $maxfiles;
+ " target storage or the vzdump parameters.".
+ " Either increase the limit or delete old backup(s).\n"
+ if $count >= $maxfiles;
}
- my $logfile = $task->{logfile} = "$opts->{dumpdir}/$basename.log";
+ if (!$self->{opts}->{pbs}) {
+ $task->{logfile} = "$opts->{dumpdir}/$basename.log";
+ }
my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
my ($comp, $comp_ext) = compressor_info($opts);
$ext .= ".${comp_ext}";
}
- if ($opts->{stdout}) {
- $task->{tarfile} = '-';
+ if ($self->{opts}->{pbs}) {
+ die "unable to pipe backup to stdout\n" if $opts->{stdout};
} else {
- my $tarfile = $task->{tarfile} = "$opts->{dumpdir}/$basename$ext";
- $task->{tmptar} = $task->{tarfile};
- $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
- unlink $task->{tmptar};
+ if ($opts->{stdout}) {
+ $task->{tarfile} = '-';
+ } else {
+ my $tarfile = $task->{tarfile} = "$opts->{dumpdir}/$basename$ext";
+ $task->{tmptar} = $task->{tarfile};
+ $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
+ unlink $task->{tmptar};
+ }
}
$task->{vmtype} = $vmtype;
- if ($opts->{tmpdir}) {
+ if ($self->{opts}->{pbs}) {
+ $task->{tmpdir} = "/var/tmp/vzdumptmp$$"; #fixme
+ } elsif ($opts->{tmpdir}) {
$task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp$$";
} else {
# dumpdir is posix? then use it as temporary dir
$task->{dumpdir} = $opts->{dumpdir};
$task->{storeid} = $opts->{storage};
+ $task->{scfg} = $opts->{scfg};
$task->{tmplog} = $tmplog;
- unlink $logfile;
+ unlink $task->{logfile} if defined($task->{logfile});
debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
return;
}
- debugmsg ('info', "creating archive '$task->{tarfile}'", $logfd);
+ # fixme: ??
+ if ($self->{opts}->{pbs}) {
+ debugmsg ('info', "creating pbs archive on storage '$opts->{storage}'", $logfd);
+ } else {
+ debugmsg ('info', "creating archive '$task->{tarfile}'", $logfd);
+ }
$plugin->archive($task, $vmid, $task->{tmptar}, $comp);
- rename ($task->{tmptar}, $task->{tarfile}) ||
- die "unable to rename '$task->{tmptar}' to '$task->{tarfile}'\n";
+ if ($self->{opts}->{pbs}) {
+ # fixme: log size ?
+ debugmsg ('info', "pbs upload finished", $logfd);
+ } else {
+ rename ($task->{tmptar}, $task->{tarfile}) ||
+ die "unable to rename '$task->{tmptar}' to '$task->{tarfile}'\n";
- # determine size
- $task->{size} = (-s $task->{tarfile}) || 0;
- my $cs = format_size ($task->{size});
- debugmsg ('info', "archive file size: $cs", $logfd);
+ # determine size
+ $task->{size} = (-s $task->{tarfile}) || 0;
+ my $cs = format_size ($task->{size});
+ debugmsg ('info', "archive file size: $cs", $logfd);
+ }
# purge older backup
-
if ($maxfiles && $opts->{remove}) {
- my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{tarfile});
- $bklist = [ sort { $b->[1] <=> $a->[1] } @$bklist ];
-
- while (scalar (@$bklist) >= $maxfiles) {
- my $d = pop @$bklist;
- debugmsg ('info', "delete old backup '$d->[0]'", $logfd);
- unlink $d->[0];
- my $logfn = $d->[0];
- $logfn =~ s/\.(tgz|((tar|vma)(\.(gz|lzo))?))$/\.log/;
- unlink $logfn;
+
+ if ($self->{opts}->{pbs}) {
+ my $args = [$pbs_group_name, '--quiet', '1', '--keep-last', $maxfiles];
+ my $logfunc = sub { my $line = shift; debugmsg ('info', $line, $logfd); };
+ PVE::Storage::PBSPlugin::run_raw_client_cmd(
+ $opts->{scfg}, $opts->{storage}, 'prune', $args, logfunc => $logfunc);
+ } else {
+ my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{tarfile});
+ $bklist = [ sort { $b->[1] <=> $a->[1] } @$bklist ];
+
+ while (scalar (@$bklist) >= $maxfiles) {
+ my $d = pop @$bklist;
+ debugmsg ('info', "delete old backup '$d->[0]'", $logfd);
+ unlink $d->[0];
+ my $logfn = $d->[0];
+ $logfn =~ s/\.(tgz|((tar|vma)(\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?))$/\.log/;
+ unlink $logfn;
+ }
}
}
close ($logfd) if $logfd;
- if ($task->{tmplog} && $task->{logfile}) {
- system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
+ if ($task->{tmplog}) {
+ if ($self->{opts}->{pbs}) {
+ if ($task->{state} eq 'ok') {
+ my $param = [$pbs_snapshot_name, $task->{tmplog}];
+ PVE::Storage::PBSPlugin::run_raw_client_cmd(
+ $opts->{scfg}, $opts->{storage}, 'upload-log', $param, errmsg => "upload log failed");
+ }
+ } elsif ($task->{logfile}) {
+ system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
+ }
}
eval { $self->run_hook_script ('log-end', $task); };
}
}
+sub get_included_guests {
+ my ($job) = @_;
+
+ my $nodename = PVE::INotify::nodename();
+ my $vmids = [];
+
+ # convert string lists to arrays
+ if ($job->{pool}) {
+ $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool});
+ } else {
+ $vmids = [ PVE::Tools::split_list(extract_param($job, 'vmid')) ];
+ }
+
+ my $skiplist = [];
+ if (!$job->{all}) {
+ if (!$job->{node} || $job->{node} eq $nodename) {
+ my $vmlist = PVE::Cluster::get_vmlist();
+ my $localvmids = [];
+ foreach my $vmid (@{$vmids}) {
+ my $d = $vmlist->{ids}->{$vmid};
+ if ($d && ($d->{node} ne $nodename)) {
+ push @{$skiplist}, $vmid;
+ } else {
+ push @{$localvmids}, $vmid;
+ }
+ }
+ $vmids = $localvmids;
+ }
+
+ $job->{vmids} = PVE::VZDump::check_vmids(@{$vmids})
+ }
+
+ return ($vmids, $skiplist);
+}
+
1;
projects / pve-manager.git / blobdiff