X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=PVE%2FStorage%2FPlugin.pm;h=d7136a1b23b77c3a0d409aeb3433e4789b77f114;hb=22265bd9900e0af1d39880465d46dac494c10637;hp=dba6eb9adf7520f633c1e67b3292bc41fbb37b06;hpb=92ae59df9eb967761aa9205a89612f16e26b3d73;p=pve-storage.git diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm index dba6eb9..d7136a1 100644 --- a/PVE/Storage/Plugin.pm +++ b/PVE/Storage/Plugin.pm @@ -8,16 +8,19 @@ use File::chdir; use File::Path; use File::Basename; use File::stat qw(); -use Time::Local qw(timelocal); use PVE::Tools qw(run_command); -use PVE::JSONSchema qw(get_standard_option); +use PVE::JSONSchema qw(get_standard_option register_standard_option); use PVE::Cluster qw(cfs_register_file); use JSON; use base qw(PVE::SectionConfig); +use constant COMPRESSOR_RE => 'gz|lzo|zst'; + +use constant NOTES_EXT => ".notes"; + our @COMMON_TAR_FLAGS = qw( --one-file-system -p --sparse --numeric-owner --acls @@ -34,7 +37,9 @@ our @SHARED_STORAGE = ( 'iscsidirect', 'glusterfs', 'zfs', - 'drbd'); + 'drbd', + 'pbs', +); our $MAX_VOLUMES_PER_GUEST = 1024; @@ -42,6 +47,72 @@ cfs_register_file ('storage.cfg', sub { __PACKAGE__->parse_config(@_); }, sub { __PACKAGE__->write_config(@_); }); +my %prune_option = ( + optional => 1, + type => 'integer', minimum => '0', + format_description => 'N', +); + +our $prune_backups_format = { + 'keep-all' => { + type => 'boolean', + description => 'Keep all backups. Conflicts with the other options when true.', + optional => 1, + }, + 'keep-last' => { + %prune_option, + description => 'Keep the last backups.', + }, + 'keep-hourly' => { + %prune_option, + description => 'Keep backups for the last different hours. If there is more' . + 'than one backup for a single hour, only the latest one is kept.' + }, + 'keep-daily' => { + %prune_option, + description => 'Keep backups for the last different days. If there is more' . + 'than one backup for a single day, only the latest one is kept.' + }, + 'keep-weekly' => { + %prune_option, + description => 'Keep backups for the last different weeks. If there is more' . + 'than one backup for a single week, only the latest one is kept.' + }, + 'keep-monthly' => { + %prune_option, + description => 'Keep backups for the last different months. If there is more' . + 'than one backup for a single month, only the latest one is kept.' + }, + 'keep-yearly' => { + %prune_option, + description => 'Keep backups for the last different years. If there is more' . + 'than one backup for a single year, only the latest one is kept.' + }, +}; +PVE::JSONSchema::register_format('prune-backups', $prune_backups_format, \&validate_prune_backups); +sub validate_prune_backups { + my ($prune_backups) = @_; + + my $keep_all = delete $prune_backups->{'keep-all'}; + + if (!scalar(grep {$_ > 0} values %{$prune_backups})) { + $prune_backups = { 'keep-all' => 1 }; + } elsif ($keep_all) { + die "keep-all cannot be set together with other options.\n"; + } + + return $prune_backups; +} +register_standard_option('prune-backups', { + description => "The retention options with shorter intervals are processed first " . + "with --keep-last being the very first one. Each option covers a " . + "specific period of time. We say that backups within this period " . + "are covered by this option. The next option does not take care " . + "of already covered backups and only considers older backups.", + optional => 1, + type => 'string', + format => 'prune-backups', +}); my $defaultData = { propertyList => { @@ -67,6 +138,7 @@ my $defaultData = { minimum => 0, optional => 1, }, + 'prune-backups' => get_standard_option('prune-backups'), shared => { description => "Mark storage as shared.", type => 'boolean', @@ -367,6 +439,7 @@ sub on_add_hook { my ($class, $storeid, $scfg, %param) = @_; # do nothing by default + return undef; } # called during storage configuration update (before the updated storage config got written) @@ -376,6 +449,7 @@ sub on_update_hook { my ($class, $storeid, $scfg, %param) = @_; # do nothing by default + return undef; } # called during deletion of storage (before the new storage config got written) @@ -387,6 +461,7 @@ sub on_delete_hook { my ($class, $storeid, $scfg) = @_; # do nothing by default + return undef; } sub cluster_lock_storage { @@ -435,7 +510,7 @@ sub parse_volname { return ('vztmpl', $1); } elsif ($volname =~ m!^rootdir/(\d+)$!) { return ('rootdir', $1, $1); - } elsif ($volname =~ m!^backup/([^/]+(\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo)))$!) { + } elsif ($volname =~ m!^backup/([^/]+(?:\.(?:tgz|(?:(?:tar|vma)(?:\.(?:${\COMPRESSOR_RE}))?))))$!) { my $fn = $1; if ($fn =~ m/^vzdump-(openvz|lxc|qemu)-(\d+)-.+/) { return ('backup', $fn, $2); @@ -457,6 +532,10 @@ my $vtype_subdirs = { snippets => 'snippets', }; +sub get_vtype_subdirs { + return $vtype_subdirs; +} + sub get_subdir { my ($class, $scfg, $vtype) = @_; @@ -721,6 +800,12 @@ sub file_size_info { my $st = File::stat::stat($filename); + if (!defined($st)) { + my $extramsg = -l $filename ? ' - dangling symlink?' : ''; + warn "failed to stat '$filename'$extramsg\n"; + return undef; + } + if (S_ISDIR($st->mode)) { return wantarray ? (0, 'subvol', 0, undef, $st->ctime) : 1; } @@ -743,6 +828,18 @@ sub file_size_info { return wantarray ? ($size, $format, $used, $parent, $st->ctime) : $size; } +sub get_volume_notes { + my ($class, $scfg, $storeid, $volname, $timeout) = @_; + + die "volume notes are not supported for $class"; +} + +sub update_volume_notes { + my ($class, $scfg, $storeid, $volname, $notes, $timeout) = @_; + + die "volume notes are not supported for $class"; +} + sub volume_size_info { my ($class, $scfg, $storeid, $volname, $timeout) = @_; my $path = $class->filesystem_path($scfg, $volname); @@ -820,6 +917,10 @@ sub volume_snapshot_delete { return undef; } +sub volume_snapshot_needs_fsfreeze { + + return 0; +} sub storage_can_replicate { my ($class, $scfg, $storeid, $format) = @_; @@ -916,10 +1017,9 @@ my $get_subdir_files = sub { my $res = []; foreach my $fn (<$path/*>) { - my $st = File::stat::stat($fn); - next if S_ISDIR($st->mode); + next if (!$st || S_ISDIR($st->mode)); my $info; @@ -934,21 +1034,29 @@ my $get_subdir_files = sub { $info = { volid => "$sid:vztmpl/$1", format => "t$2" }; } elsif ($tt eq 'backup') { - next if defined($vmid) && $fn !~ m/\S+-$vmid-\S+/; - next if $fn !~ m!/([^/]+\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo))$!; - + next if $fn !~ m!/([^/]+\.(tgz|(?:(?:tar|vma)(?:\.(${\COMPRESSOR_RE}))?)))$!; + my $original = $fn; my $format = $2; - $info = { volid => "$sid:backup/$1", format => $format }; + $fn = $1; - if ($fn =~ m!^vzdump\-(?:lxc|qemu)\-(?:[1-9][0-9]{2,8})\-(\d{4})_(\d{2})_(\d{2})\-(\d{2})_(\d{2})_(\d{2})\.${format}$!) { - my $epoch = timelocal($6, $5, $4, $3, $2-1, $1 - 1900); - $info->{ctime} = $epoch; - } + # only match for VMID now, to avoid false positives (VMID in parent directory name) + next if defined($vmid) && $fn !~ m/\S+-$vmid-\S+/; + + $info = { volid => "$sid:backup/$fn", format => $format }; + + my $archive_info = eval { PVE::Storage::archive_info($fn) } // {}; + + $info->{ctime} = $archive_info->{ctime} if defined($archive_info->{ctime}); if (defined($vmid) || $fn =~ m!\-([1-9][0-9]{2,8})\-[^/]+\.${format}$!) { $info->{vmid} = $vmid // $1; } + my $notes_fn = $original.NOTES_EXT; + if (-f $notes_fn) { + my $notes = PVE::Tools::file_read_firstline($notes_fn); + $info->{notes} = $notes if defined($notes); + } } elsif ($tt eq 'snippets') { @@ -1114,6 +1222,75 @@ sub check_connection { return 1; } +sub prune_backups { + my ($class, $scfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc) = @_; + + $logfunc //= sub { print "$_[1]\n" }; + + my $backups = $class->list_volumes($storeid, $scfg, $vmid, ['backup']); + + my $backup_groups = {}; + my $prune_list = []; + + foreach my $backup (@{$backups}) { + my $volid = $backup->{volid}; + my $archive_info = eval { PVE::Storage::archive_info($volid) } // {}; + my $backup_type = $archive_info->{type} // 'unknown'; + my $backup_vmid = $archive_info->{vmid} // $backup->{vmid}; + + next if defined($type) && $type ne $backup_type; + + my $prune_entry = { + ctime => $backup->{ctime}, + type => $backup_type, + volid => $volid, + }; + + $prune_entry->{vmid} = $backup_vmid if defined($backup_vmid); + + if ($archive_info->{is_std_name}) { + die "internal error - got no VMID\n" if !defined($backup_vmid); + die "internal error - got wrong VMID '$backup_vmid' != '$vmid'\n" + if defined($vmid) && $backup_vmid ne $vmid; + + $prune_entry->{ctime} = $archive_info->{ctime}; + my $group = "$backup_type/$backup_vmid"; + push @{$backup_groups->{$group}}, $prune_entry; + } else { + # ignore backups that don't use the standard naming scheme + $prune_entry->{mark} = 'protected'; + } + + push @{$prune_list}, $prune_entry; + } + + foreach my $backup_group (values %{$backup_groups}) { + PVE::Storage::prune_mark_backup_group($backup_group, $keep); + } + + my $failed; + if (!$dryrun) { + foreach my $prune_entry (@{$prune_list}) { + next if $prune_entry->{mark} ne 'remove'; + + my $volid = $prune_entry->{volid}; + $logfunc->('info', "removing backup '$volid'"); + eval { + my (undef, $volname) = parse_volume_id($volid); + my $archive_path = $class->filesystem_path($scfg, $volname); + PVE::Storage::archive_remove($archive_path); + }; + if (my $err = $@) { + $logfunc->('err', "error when removing backup '$volid' - $err\n"); + $failed = 1; + } + } + } + die "error pruning backups - check log\n" if $failed; + + return $prune_list; +} + # Import/Export interface: # Any path based storage is assumed to support 'raw' and 'tar' streams, so # the default implementations will return this if $scfg->{path} is set,