X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=PVE%2FStorage%2FPlugin.pm;h=4a10a1fef93ceec3e9409a157bd2a9351cf95d53;hb=9177cc2eda87c9a8f85a5ba73fa5f8e45cdd44de;hp=7f04e85ae8cd00748f3286a6fdaae64992ff22d8;hpb=e9667b3418dff2269e47d2247025df3df0b4d42a;p=pve-storage.git diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm index 7f04e85..4a10a1f 100644 --- a/PVE/Storage/Plugin.pm +++ b/PVE/Storage/Plugin.pm @@ -10,7 +10,7 @@ use File::Basename; use File::stat qw(); use PVE::Tools qw(run_command); -use PVE::JSONSchema qw(get_standard_option); +use PVE::JSONSchema qw(get_standard_option register_standard_option); use PVE::Cluster qw(cfs_register_file); use JSON; @@ -19,6 +19,8 @@ use base qw(PVE::SectionConfig); use constant COMPRESSOR_RE => 'gz|lzo|zst'; +use constant NOTES_EXT => ".notes"; + our @COMMON_TAR_FLAGS = qw( --one-file-system -p --sparse --numeric-owner --acls @@ -35,7 +37,9 @@ our @SHARED_STORAGE = ( 'iscsidirect', 'glusterfs', 'zfs', - 'drbd'); + 'drbd', + 'pbs', +); our $MAX_VOLUMES_PER_GUEST = 1024; @@ -43,6 +47,72 @@ cfs_register_file ('storage.cfg', sub { __PACKAGE__->parse_config(@_); }, sub { __PACKAGE__->write_config(@_); }); +my %prune_option = ( + optional => 1, + type => 'integer', minimum => '0', + format_description => 'N', +); + +our $prune_backups_format = { + 'keep-all' => { + type => 'boolean', + description => 'Keep all backups. Conflicts with the other options when true.', + optional => 1, + }, + 'keep-last' => { + %prune_option, + description => 'Keep the last backups.', + }, + 'keep-hourly' => { + %prune_option, + description => 'Keep backups for the last different hours. If there is more' . + 'than one backup for a single hour, only the latest one is kept.' + }, + 'keep-daily' => { + %prune_option, + description => 'Keep backups for the last different days. If there is more' . + 'than one backup for a single day, only the latest one is kept.' + }, + 'keep-weekly' => { + %prune_option, + description => 'Keep backups for the last different weeks. If there is more' . + 'than one backup for a single week, only the latest one is kept.' + }, + 'keep-monthly' => { + %prune_option, + description => 'Keep backups for the last different months. If there is more' . + 'than one backup for a single month, only the latest one is kept.' + }, + 'keep-yearly' => { + %prune_option, + description => 'Keep backups for the last different years. If there is more' . + 'than one backup for a single year, only the latest one is kept.' + }, +}; +PVE::JSONSchema::register_format('prune-backups', $prune_backups_format, \&validate_prune_backups); +sub validate_prune_backups { + my ($prune_backups) = @_; + + my $keep_all = delete $prune_backups->{'keep-all'}; + + if (!scalar(grep {$_ > 0} values %{$prune_backups})) { + $prune_backups = { 'keep-all' => 1 }; + } elsif ($keep_all) { + die "keep-all cannot be set together with other options.\n"; + } + + return $prune_backups; +} +register_standard_option('prune-backups', { + description => "The retention options with shorter intervals are processed first " . + "with --keep-last being the very first one. Each option covers a " . + "specific period of time. We say that backups within this period " . + "are covered by this option. The next option does not take care " . + "of already covered backups and only considers older backups.", + optional => 1, + type => 'string', + format => 'prune-backups', +}); my $defaultData = { propertyList => { @@ -68,6 +138,7 @@ my $defaultData = { minimum => 0, optional => 1, }, + 'prune-backups' => get_standard_option('prune-backups'), shared => { description => "Mark storage as shared.", type => 'boolean', @@ -368,6 +439,7 @@ sub on_add_hook { my ($class, $storeid, $scfg, %param) = @_; # do nothing by default + return undef; } # called during storage configuration update (before the updated storage config got written) @@ -377,6 +449,7 @@ sub on_update_hook { my ($class, $storeid, $scfg, %param) = @_; # do nothing by default + return undef; } # called during deletion of storage (before the new storage config got written) @@ -388,6 +461,7 @@ sub on_delete_hook { my ($class, $storeid, $scfg) = @_; # do nothing by default + return undef; } sub cluster_lock_storage { @@ -634,7 +708,7 @@ sub clone_image { local $CWD = $imagedir; my $cmd = ['/usr/bin/qemu-img', 'create', '-b', "../$basevmid/$basename", - '-f', 'qcow2', $path]; + '-F', $format, '-f', 'qcow2', $path]; run_command($cmd); }; @@ -726,6 +800,12 @@ sub file_size_info { my $st = File::stat::stat($filename); + if (!defined($st)) { + my $extramsg = -l $filename ? ' - dangling symlink?' : ''; + warn "failed to stat '$filename'$extramsg\n"; + return undef; + } + if (S_ISDIR($st->mode)) { return wantarray ? (0, 'subvol', 0, undef, $st->ctime) : 1; } @@ -748,6 +828,18 @@ sub file_size_info { return wantarray ? ($size, $format, $used, $parent, $st->ctime) : $size; } +sub get_volume_notes { + my ($class, $scfg, $storeid, $volname, $timeout) = @_; + + die "volume notes are not supported for $class"; +} + +sub update_volume_notes { + my ($class, $scfg, $storeid, $volname, $notes, $timeout) = @_; + + die "volume notes are not supported for $class"; +} + sub volume_size_info { my ($class, $scfg, $storeid, $volname, $timeout) = @_; my $path = $class->filesystem_path($scfg, $volname); @@ -825,6 +917,10 @@ sub volume_snapshot_delete { return undef; } +sub volume_snapshot_needs_fsfreeze { + + return 0; +} sub storage_can_replicate { my ($class, $scfg, $storeid, $format) = @_; @@ -921,7 +1017,6 @@ my $get_subdir_files = sub { my $res = []; foreach my $fn (<$path/*>) { - my $st = File::stat::stat($fn); next if (!$st || S_ISDIR($st->mode)); @@ -939,10 +1034,14 @@ my $get_subdir_files = sub { $info = { volid => "$sid:vztmpl/$1", format => "t$2" }; } elsif ($tt eq 'backup') { - next if defined($vmid) && $fn !~ m/\S+-$vmid-\S+/; next if $fn !~ m!/([^/]+\.(tgz|(?:(?:tar|vma)(?:\.(${\COMPRESSOR_RE}))?)))$!; + my $original = $fn; my $format = $2; $fn = $1; + + # only match for VMID now, to avoid false positives (VMID in parent directory name) + next if defined($vmid) && $fn !~ m/\S+-$vmid-\S+/; + $info = { volid => "$sid:backup/$fn", format => $format }; my $archive_info = eval { PVE::Storage::archive_info($fn) } // {}; @@ -953,6 +1052,12 @@ my $get_subdir_files = sub { $info->{vmid} = $vmid // $1; } + my $notes_fn = $original.NOTES_EXT; + if (-f $notes_fn) { + my $notes = PVE::Tools::file_read_firstline($notes_fn); + $info->{notes} = $notes if defined($notes); + } + } elsif ($tt eq 'snippets') { $info = { @@ -1117,6 +1222,75 @@ sub check_connection { return 1; } +sub prune_backups { + my ($class, $scfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc) = @_; + + $logfunc //= sub { print "$_[1]\n" }; + + my $backups = $class->list_volumes($storeid, $scfg, $vmid, ['backup']); + + my $backup_groups = {}; + my $prune_list = []; + + foreach my $backup (@{$backups}) { + my $volid = $backup->{volid}; + my $archive_info = eval { PVE::Storage::archive_info($volid) } // {}; + my $backup_type = $archive_info->{type} // 'unknown'; + my $backup_vmid = $archive_info->{vmid} // $backup->{vmid}; + + next if defined($type) && $type ne $backup_type; + + my $prune_entry = { + ctime => $backup->{ctime}, + type => $backup_type, + volid => $volid, + }; + + $prune_entry->{vmid} = $backup_vmid if defined($backup_vmid); + + if ($archive_info->{is_std_name}) { + die "internal error - got no VMID\n" if !defined($backup_vmid); + die "internal error - got wrong VMID '$backup_vmid' != '$vmid'\n" + if defined($vmid) && $backup_vmid ne $vmid; + + $prune_entry->{ctime} = $archive_info->{ctime}; + my $group = "$backup_type/$backup_vmid"; + push @{$backup_groups->{$group}}, $prune_entry; + } else { + # ignore backups that don't use the standard naming scheme + $prune_entry->{mark} = 'protected'; + } + + push @{$prune_list}, $prune_entry; + } + + foreach my $backup_group (values %{$backup_groups}) { + PVE::Storage::prune_mark_backup_group($backup_group, $keep); + } + + my $failed; + if (!$dryrun) { + foreach my $prune_entry (@{$prune_list}) { + next if $prune_entry->{mark} ne 'remove'; + + my $volid = $prune_entry->{volid}; + $logfunc->('info', "removing backup '$volid'"); + eval { + my (undef, $volname) = parse_volume_id($volid); + my $archive_path = $class->filesystem_path($scfg, $volname); + PVE::Storage::archive_remove($archive_path); + }; + if (my $err = $@) { + $logfunc->('err', "error when removing backup '$volid' - $err\n"); + $failed = 1; + } + } + } + die "error pruning backups - check log\n" if $failed; + + return $prune_list; +} + # Import/Export interface: # Any path based storage is assumed to support 'raw' and 'tar' streams, so # the default implementations will return this if $scfg->{path} is set,