X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=PVE%2FStorage%2FPlugin.pm;h=4a10a1fef93ceec3e9409a157bd2a9351cf95d53;hb=9177cc2eda87c9a8f85a5ba73fa5f8e45cdd44de;hp=8c0dae1e09222426e12239e26a6551222465536f;hpb=e6f4eed43581de9b9706cc2263c9631ea2abfc1a;p=pve-storage.git diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm index 8c0dae1..4a10a1f 100644 --- a/PVE/Storage/Plugin.pm +++ b/PVE/Storage/Plugin.pm @@ -7,16 +7,20 @@ use Fcntl ':mode'; use File::chdir; use File::Path; use File::Basename; -use Time::Local qw(timelocal); +use File::stat qw(); use PVE::Tools qw(run_command); -use PVE::JSONSchema qw(get_standard_option); +use PVE::JSONSchema qw(get_standard_option register_standard_option); use PVE::Cluster qw(cfs_register_file); use JSON; use base qw(PVE::SectionConfig); +use constant COMPRESSOR_RE => 'gz|lzo|zst'; + +use constant NOTES_EXT => ".notes"; + our @COMMON_TAR_FLAGS = qw( --one-file-system -p --sparse --numeric-owner --acls @@ -33,7 +37,9 @@ our @SHARED_STORAGE = ( 'iscsidirect', 'glusterfs', 'zfs', - 'drbd'); + 'drbd', + 'pbs', +); our $MAX_VOLUMES_PER_GUEST = 1024; @@ -41,6 +47,72 @@ cfs_register_file ('storage.cfg', sub { __PACKAGE__->parse_config(@_); }, sub { __PACKAGE__->write_config(@_); }); +my %prune_option = ( + optional => 1, + type => 'integer', minimum => '0', + format_description => 'N', +); + +our $prune_backups_format = { + 'keep-all' => { + type => 'boolean', + description => 'Keep all backups. Conflicts with the other options when true.', + optional => 1, + }, + 'keep-last' => { + %prune_option, + description => 'Keep the last backups.', + }, + 'keep-hourly' => { + %prune_option, + description => 'Keep backups for the last different hours. If there is more' . + 'than one backup for a single hour, only the latest one is kept.' + }, + 'keep-daily' => { + %prune_option, + description => 'Keep backups for the last different days. If there is more' . + 'than one backup for a single day, only the latest one is kept.' + }, + 'keep-weekly' => { + %prune_option, + description => 'Keep backups for the last different weeks. If there is more' . + 'than one backup for a single week, only the latest one is kept.' + }, + 'keep-monthly' => { + %prune_option, + description => 'Keep backups for the last different months. If there is more' . + 'than one backup for a single month, only the latest one is kept.' + }, + 'keep-yearly' => { + %prune_option, + description => 'Keep backups for the last different years. If there is more' . + 'than one backup for a single year, only the latest one is kept.' + }, +}; +PVE::JSONSchema::register_format('prune-backups', $prune_backups_format, \&validate_prune_backups); +sub validate_prune_backups { + my ($prune_backups) = @_; + + my $keep_all = delete $prune_backups->{'keep-all'}; + + if (!scalar(grep {$_ > 0} values %{$prune_backups})) { + $prune_backups = { 'keep-all' => 1 }; + } elsif ($keep_all) { + die "keep-all cannot be set together with other options.\n"; + } + + return $prune_backups; +} +register_standard_option('prune-backups', { + description => "The retention options with shorter intervals are processed first " . + "with --keep-last being the very first one. Each option covers a " . + "specific period of time. We say that backups within this period " . + "are covered by this option. The next option does not take care " . + "of already covered backups and only considers older backups.", + optional => 1, + type => 'string', + format => 'prune-backups', +}); my $defaultData = { propertyList => { @@ -66,6 +138,7 @@ my $defaultData = { minimum => 0, optional => 1, }, + 'prune-backups' => get_standard_option('prune-backups'), shared => { description => "Mark storage as shared.", type => 'boolean', @@ -366,6 +439,7 @@ sub on_add_hook { my ($class, $storeid, $scfg, %param) = @_; # do nothing by default + return undef; } # called during storage configuration update (before the updated storage config got written) @@ -375,6 +449,7 @@ sub on_update_hook { my ($class, $storeid, $scfg, %param) = @_; # do nothing by default + return undef; } # called during deletion of storage (before the new storage config got written) @@ -386,6 +461,7 @@ sub on_delete_hook { my ($class, $storeid, $scfg) = @_; # do nothing by default + return undef; } sub cluster_lock_storage { @@ -434,7 +510,7 @@ sub parse_volname { return ('vztmpl', $1); } elsif ($volname =~ m!^rootdir/(\d+)$!) { return ('rootdir', $1, $1); - } elsif ($volname =~ m!^backup/([^/]+(\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo)))$!) { + } elsif ($volname =~ m!^backup/([^/]+(?:\.(?:tgz|(?:(?:tar|vma)(?:\.(?:${\COMPRESSOR_RE}))?))))$!) { my $fn = $1; if ($fn =~ m/^vzdump-(openvz|lxc|qemu)-(\d+)-.+/) { return ('backup', $fn, $2); @@ -456,6 +532,10 @@ my $vtype_subdirs = { snippets => 'snippets', }; +sub get_vtype_subdirs { + return $vtype_subdirs; +} + sub get_subdir { my ($class, $scfg, $vtype) = @_; @@ -628,7 +708,7 @@ sub clone_image { local $CWD = $imagedir; my $cmd = ['/usr/bin/qemu-img', 'create', '-b', "../$basevmid/$basename", - '-f', 'qcow2', $path]; + '-F', $format, '-f', 'qcow2', $path]; run_command($cmd); }; @@ -718,12 +798,16 @@ sub free_image { sub file_size_info { my ($filename, $timeout) = @_; - my @fs = stat($filename); - my $mode = $fs[2]; - my $ctime = $fs[10]; + my $st = File::stat::stat($filename); + + if (!defined($st)) { + my $extramsg = -l $filename ? ' - dangling symlink?' : ''; + warn "failed to stat '$filename'$extramsg\n"; + return undef; + } - if (S_ISDIR($mode)) { - return wantarray ? (0, 'subvol', 0, undef, $ctime) : 1; + if (S_ISDIR($st->mode)) { + return wantarray ? (0, 'subvol', 0, undef, $st->ctime) : 1; } my $json = ''; @@ -741,7 +825,19 @@ sub file_size_info { my ($size, $format, $used, $parent) = $info->@{qw(virtual-size format actual-size backing-filename)}; - return wantarray ? ($size, $format, $used, $parent, $ctime) : $size; + return wantarray ? ($size, $format, $used, $parent, $st->ctime) : $size; +} + +sub get_volume_notes { + my ($class, $scfg, $storeid, $volname, $timeout) = @_; + + die "volume notes are not supported for $class"; +} + +sub update_volume_notes { + my ($class, $scfg, $storeid, $volname, $notes, $timeout) = @_; + + die "volume notes are not supported for $class"; } sub volume_size_info { @@ -821,6 +917,10 @@ sub volume_snapshot_delete { return undef; } +sub volume_snapshot_needs_fsfreeze { + + return 0; +} sub storage_can_replicate { my ($class, $scfg, $storeid, $format) = @_; @@ -917,23 +1017,9 @@ my $get_subdir_files = sub { my $res = []; foreach my $fn (<$path/*>) { + my $st = File::stat::stat($fn); - my ($dev, - $ino, - $mode, - $nlink, - $uid, - $gid, - $rdev, - $size, - $atime, - $mtime, - $ctime, - $blksize, - $blocks - ) = stat($fn); - - next if S_ISDIR($mode); + next if (!$st || S_ISDIR($st->mode)); my $info; @@ -948,21 +1034,29 @@ my $get_subdir_files = sub { $info = { volid => "$sid:vztmpl/$1", format => "t$2" }; } elsif ($tt eq 'backup') { - next if defined($vmid) && $fn !~ m/\S+-$vmid-\S+/; - next if $fn !~ m!/([^/]+\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo))$!; - + next if $fn !~ m!/([^/]+\.(tgz|(?:(?:tar|vma)(?:\.(${\COMPRESSOR_RE}))?)))$!; + my $original = $fn; my $format = $2; - $info = { volid => "$sid:backup/$1", format => $format }; + $fn = $1; - if ($fn =~ m!^vzdump\-(?:lxc|qemu)\-(?:[1-9][0-9]{2,8})\-(\d{4})_(\d{2})_(\d{2})\-(\d{2})_(\d{2})_(\d{2})\.${format}$!) { - my $epoch = timelocal($6, $5, $4, $3, $2-1, $1 - 1900); - $info->{ctime} = $epoch; - } + # only match for VMID now, to avoid false positives (VMID in parent directory name) + next if defined($vmid) && $fn !~ m/\S+-$vmid-\S+/; + + $info = { volid => "$sid:backup/$fn", format => $format }; + + my $archive_info = eval { PVE::Storage::archive_info($fn) } // {}; + + $info->{ctime} = $archive_info->{ctime} if defined($archive_info->{ctime}); if (defined($vmid) || $fn =~ m!\-([1-9][0-9]{2,8})\-[^/]+\.${format}$!) { $info->{vmid} = $vmid // $1; } + my $notes_fn = $original.NOTES_EXT; + if (-f $notes_fn) { + my $notes = PVE::Tools::file_read_firstline($notes_fn); + $info->{notes} = $notes if defined($notes); + } } elsif ($tt eq 'snippets') { @@ -972,8 +1066,8 @@ my $get_subdir_files = sub { }; } - $info->{size} = $size; - $info->{ctime} //= $ctime; + $info->{size} = $st->size; + $info->{ctime} //= $st->ctime; push @$res, $info; } @@ -1128,6 +1222,75 @@ sub check_connection { return 1; } +sub prune_backups { + my ($class, $scfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc) = @_; + + $logfunc //= sub { print "$_[1]\n" }; + + my $backups = $class->list_volumes($storeid, $scfg, $vmid, ['backup']); + + my $backup_groups = {}; + my $prune_list = []; + + foreach my $backup (@{$backups}) { + my $volid = $backup->{volid}; + my $archive_info = eval { PVE::Storage::archive_info($volid) } // {}; + my $backup_type = $archive_info->{type} // 'unknown'; + my $backup_vmid = $archive_info->{vmid} // $backup->{vmid}; + + next if defined($type) && $type ne $backup_type; + + my $prune_entry = { + ctime => $backup->{ctime}, + type => $backup_type, + volid => $volid, + }; + + $prune_entry->{vmid} = $backup_vmid if defined($backup_vmid); + + if ($archive_info->{is_std_name}) { + die "internal error - got no VMID\n" if !defined($backup_vmid); + die "internal error - got wrong VMID '$backup_vmid' != '$vmid'\n" + if defined($vmid) && $backup_vmid ne $vmid; + + $prune_entry->{ctime} = $archive_info->{ctime}; + my $group = "$backup_type/$backup_vmid"; + push @{$backup_groups->{$group}}, $prune_entry; + } else { + # ignore backups that don't use the standard naming scheme + $prune_entry->{mark} = 'protected'; + } + + push @{$prune_list}, $prune_entry; + } + + foreach my $backup_group (values %{$backup_groups}) { + PVE::Storage::prune_mark_backup_group($backup_group, $keep); + } + + my $failed; + if (!$dryrun) { + foreach my $prune_entry (@{$prune_list}) { + next if $prune_entry->{mark} ne 'remove'; + + my $volid = $prune_entry->{volid}; + $logfunc->('info', "removing backup '$volid'"); + eval { + my (undef, $volname) = parse_volume_id($volid); + my $archive_path = $class->filesystem_path($scfg, $volname); + PVE::Storage::archive_remove($archive_path); + }; + if (my $err = $@) { + $logfunc->('err', "error when removing backup '$volid' - $err\n"); + $failed = 1; + } + } + } + die "error pruning backups - check log\n" if $failed; + + return $prune_list; +} + # Import/Export interface: # Any path based storage is assumed to support 'raw' and 'tar' streams, so # the default implementations will return this if $scfg->{path} is set, @@ -1227,7 +1390,7 @@ sub volume_export_formats { # Import data from a stream, creating a new or replacing or adding to an existing volume. sub volume_import { - my ($class, $scfg, $storeid, $fh, $volname, $format, $base_snapshot, $with_snapshots) = @_; + my ($class, $scfg, $storeid, $fh, $volname, $format, $base_snapshot, $with_snapshots, $allow_rename) = @_; die "volume import format '$format' not available for $class\n" if $format !~ /^(raw|tar|qcow2|vmdk)\+size$/; @@ -1249,16 +1412,20 @@ sub volume_import { # Check for an existing file first since interrupting alloc_image doesn't # free it. my $file = $class->path($scfg, $volname, $storeid); - die "file '$file' already exists\n" if -e $file; + if (-e $file) { + die "file '$file' already exists\n" if !$allow_rename; + warn "file '$file' already exists - importing with a different name\n"; + $name = undef; + } my ($size) = read_common_header($fh); $size = int($size/1024); eval { my $allocname = $class->alloc_image($storeid, $scfg, $vmid, $file_format, $name, $size); - if ($allocname ne $volname) { - my $oldname = $volname; - $volname = $allocname; # Let the cleanup code know what to free + my $oldname = $volname; + $volname = $allocname; + if (defined($name) && $allocname ne $oldname) { die "internal error: unexpected allocated name: '$allocname' != '$oldname'\n"; } my $file = $class->path($scfg, $volname, $storeid) @@ -1278,6 +1445,8 @@ sub volume_import { warn $@ if $@; die $err; } + + return "$storeid:$volname"; } sub volume_import_formats {