X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=PVE%2FStorage.pm;h=71d6ad78db1255286c64d48549f10894011b9094;hb=ac5c1af57cffb79ba18c6141b96131e5790c4b2d;hp=76d17c64d2a95ff1ad7877212c432be0d1a2e8f6;hpb=189e67ffa6c707a6609a310bd12c75da628082b0;p=pve-storage.git diff --git a/PVE/Storage.pm b/PVE/Storage.pm index 76d17c6..71d6ad7 100755 --- a/PVE/Storage.pm +++ b/PVE/Storage.pm @@ -37,15 +37,15 @@ use PVE::Storage::ISCSIDirectPlugin; use PVE::Storage::GlusterfsPlugin; use PVE::Storage::ZFSPoolPlugin; use PVE::Storage::ZFSPlugin; -use PVE::Storage::DRBDPlugin; use PVE::Storage::PBSPlugin; +use PVE::Storage::BTRFSPlugin; # Storage API version. Increment it on changes in storage API interface. -use constant APIVER => 8; +use constant APIVER => 9; # Age is the number of versions we're backward compatible with. # This is like having 'current=APIVER' and age='APIAGE' in libtool, # see https://www.gnu.org/software/libtool/manual/html_node/Libtool-versioning.html -use constant APIAGE => 7; +use constant APIAGE => 0; # load standard plugins PVE::Storage::DirPlugin->register(); @@ -60,8 +60,8 @@ PVE::Storage::ISCSIDirectPlugin->register(); PVE::Storage::GlusterfsPlugin->register(); PVE::Storage::ZFSPoolPlugin->register(); PVE::Storage::ZFSPlugin->register(); -PVE::Storage::DRBDPlugin->register(); PVE::Storage::PBSPlugin->register(); +PVE::Storage::BTRFSPlugin->register(); # load third-party plugins if ( -d '/usr/share/perl5/PVE/Storage/Custom' ) { @@ -75,10 +75,8 @@ if ( -d '/usr/share/perl5/PVE/Storage/Custom' ) { require $file; # Check perl interface: - die "not derived from PVE::Storage::Plugin\n" - if !$modname->isa('PVE::Storage::Plugin'); - die "does not provide an api() method\n" - if !$modname->can('api'); + die "not derived from PVE::Storage::Plugin\n" if !$modname->isa('PVE::Storage::Plugin'); + die "does not provide an api() method\n" if !$modname->can('api'); # Check storage API version and that file is really storage plugin. my $version = $modname->api(); die "implements an API version newer than current ($version > " . APIVER . ")\n" @@ -86,11 +84,11 @@ if ( -d '/usr/share/perl5/PVE/Storage/Custom' ) { my $min_version = (APIVER - APIAGE); die "API version too old, please update the plugin ($version < $min_version)\n" if $version < $min_version; + # all OK, do import and register (i.e., "use") import $file; $modname->register(); - # If we got this far and the API version is not the same, make some - # noise: + # If we got this far and the API version is not the same, make some noise: warn "Plugin \"$modname\" is implementing an older storage API, an upgrade is recommended\n" if $version != APIVER; }; @@ -103,10 +101,10 @@ if ( -d '/usr/share/perl5/PVE/Storage/Custom' ) { # initialize all plugins PVE::Storage::Plugin->init(); -my $UDEVADM = '/sbin/udevadm'; - our $iso_extension_re = qr/\.(?:iso|img)/i; +our $vztmpl_extension_re = qr/\.tar\.(gz|xz|zst)/i; + # PVE::Storage utility functions sub config { @@ -129,7 +127,7 @@ sub lock_storage_config { } } -# FIXME remove maxfiles for PVE 7.0 +# FIXME remove maxfiles for PVE 8.0 or PVE 9.0 my $convert_maxfiles_to_prune_backups = sub { my ($scfg) = @_; @@ -196,7 +194,7 @@ sub storage_check_enabled { # storage_can_replicate: # return true if storage supports replication -# (volumes alocated with vdisk_alloc() has replication feature) +# (volumes allocated with vdisk_alloc() has replication feature) sub storage_can_replicate { my ($cfg, $storeid, $format) = @_; @@ -498,8 +496,15 @@ sub check_volume_access { return undef; } -my $volume_is_base_and_used__no_lock = sub { - my ($scfg, $storeid, $plugin, $volname) = @_; +# NOTE: this check does not work for LVM-thin, where the clone -> base +# reference is not encoded in the volume ID. +# see note in PVE::Storage::LvmThinPlugin for details. +sub volume_is_base_and_used { + my ($cfg, $volid) = @_; + + my ($storeid, $volname) = parse_volume_id($volid); + my $scfg = storage_config($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); my ($vtype, $name, $vmid, undef, undef, $isBase, undef) = $plugin->parse_volname($volname); @@ -522,21 +527,6 @@ my $volume_is_base_and_used__no_lock = sub { } } return 0; -}; - -# NOTE: this check does not work for LVM-thin, where the clone -> base -# reference is not encoded in the volume ID. -# see note in PVE::Storage::LvmThinPlugin for details. -sub volume_is_base_and_used { - my ($cfg, $volid) = @_; - - my ($storeid, $volname) = parse_volume_id($volid); - my $scfg = storage_config($cfg, $storeid); - my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); - - $plugin->cluster_lock_storage($storeid, $scfg->{shared}, undef, sub { - return &$volume_is_base_and_used__no_lock($scfg, $storeid, $plugin, $volname); - }); } # try to map a filesystem path to a volume identifier @@ -587,7 +577,7 @@ sub path_to_volume_id { } elsif ($path =~ m!^$isodir/([^/]+$iso_extension_re)$!) { my $name = $1; return ('iso', "$sid:iso/$name"); - } elsif ($path =~ m!^$tmpldir/([^/]+\.tar\.gz)$!) { + } elsif ($path =~ m!^$tmpldir/([^/]+$vztmpl_extension_re)$!) { my $name = $1; return ('vztmpl', "$sid:vztmpl/$name"); } elsif ($path =~ m!^$privatedir/(\d+)$!) { @@ -619,22 +609,22 @@ sub path { } sub abs_filesystem_path { - my ($cfg, $volid) = @_; + my ($cfg, $volid, $allow_blockdev) = @_; my $path; if (parse_volume_id ($volid, 1)) { activate_volumes($cfg, [ $volid ]); $path = PVE::Storage::path($cfg, $volid); } else { - if (-f $volid) { + if (-f $volid || ($allow_blockdev && -b $volid)) { my $abspath = abs_path($volid); if ($abspath && $abspath =~ m|^(/.+)$|) { $path = $1; # untaint any path } } } - - die "can't find file '$volid'\n" if !($path && -f $path); + die "can't find file '$volid'\n" + if !($path && (-f $path || ($allow_blockdev && -b $path))); return $path; } @@ -702,7 +692,7 @@ sub storage_migrate { my $migration_snapshot; if (!defined($snapshot)) { - if ($scfg->{type} eq 'zfspool') { + if ($scfg->{type} eq 'zfspool' || $scfg->{type} eq 'btrfs') { $migration_snapshot = 1; $snapshot = '__migration__'; } @@ -726,7 +716,8 @@ sub storage_migrate { my $send = ['pvesm', 'export', $volid, $format, '-', '-with-snapshots', $with_snapshots]; my $recv = [@$ssh, '--', 'pvesm', 'import', $target_volid, $format, $import_fn, '-with-snapshots', $with_snapshots]; if (defined($snapshot)) { - push @$send, '-snapshot', $snapshot + push @$send, '-snapshot', $snapshot; + push @$recv, '-snapshot', $snapshot; } if ($migration_snapshot) { push @$recv, '-delete-snapshot', $snapshot; @@ -736,7 +727,7 @@ sub storage_migrate { if (defined($base_snapshot)) { # Check if the snapshot exists on the remote side: push @$send, '-base', $base_snapshot; - push @$recv, '-base', $base_snapshot; + push @$recv, '-base', $base_snapshot if $target_apiver >= 9; } my $new_volid; @@ -768,8 +759,14 @@ sub storage_migrate { or die "receive command failed: $!\n"; close($input); - my ($ip) = <$info> =~ /^($PVE::Tools::IPRE)$/ or die "no tunnel IP received\n"; - my ($port) = <$info> =~ /^(\d+)$/ or die "no tunnel port received\n"; + my $try_ip = <$info> // ''; + my ($ip) = $try_ip =~ /^($PVE::Tools::IPRE)$/ # untaint + or die "no tunnel IP received, got '$try_ip'\n"; + + my $try_port = <$info> // ''; + my ($port) = $try_port =~ /^(\d+)$/ # untaint + or die "no tunnel port received, got '$try_port'\n"; + my $socket = IO::Socket::IP->new(PeerHost => $ip, PeerPort => $port, Type => SOCK_STREAM) or die "failed to connect to tunnel at $ip:$port\n"; # we won't be reading from the socket @@ -920,7 +917,7 @@ sub vdisk_free { $plugin->cluster_lock_storage($storeid, $scfg->{shared}, undef, sub { # LVM-thin allows deletion of still referenced base volumes! die "base volume '$volname' is still in use by linked clones\n" - if &$volume_is_base_and_used__no_lock($scfg, $storeid, $plugin, $volname); + if volume_is_base_and_used($cfg, $volid); my (undef, undef, undef, undef, undef, $isBase, $format) = $plugin->parse_volname($volname); @@ -936,13 +933,13 @@ sub vdisk_free { } sub vdisk_list { - my ($cfg, $storeid, $vmid, $vollist) = @_; + my ($cfg, $storeid, $vmid, $vollist, $ctype) = @_; my $ids = $cfg->{ids}; storage_check_enabled($cfg, $storeid) if ($storeid); - my $res = {}; + my $res = $storeid ? { $storeid => [] } : {}; # prepare/activate/refresh all storages @@ -959,6 +956,7 @@ sub vdisk_list { next if $storeid && $storeid ne $sid; next if !storage_check_enabled($cfg, $sid, undef, 1); my $content = $ids->{$sid}->{content}; + next if defined($ctype) && !$content->{$ctype}; next if !($content->{rootdir} || $content->{images}); push @$storage_list, $sid; } @@ -968,9 +966,8 @@ sub vdisk_list { activate_storage_list($cfg, $storage_list, $cache); - foreach my $sid (keys %$ids) { + for my $sid ($storage_list->@*) { next if $storeid && $storeid ne $sid; - next if !storage_check_enabled($cfg, $sid, undef, 1); my $scfg = $ids->{$sid}; my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); @@ -1075,8 +1072,7 @@ sub activate_storage { # only call udevsettle if there are events if ($newseq > $cache->{uevent_seqnum}) { - my $timeout = 30; - system ("$UDEVADM settle --timeout=$timeout"); # ignore errors + system ("udevadm settle --timeout=30"); # ignore errors $cache->{uevent_seqnum} = $newseq; } @@ -1564,7 +1560,7 @@ sub extract_vzdump_config_vma { my $errstring; my $err = sub { my $output = shift; - if ($output =~ m/lzop: Broken pipe: / || $output =~ m/gzip: stdout: Broken pipe/ || $output =~ m/zstd: error 70 : Write error : Broken pipe/) { + if ($output =~ m/lzop: Broken pipe: / || $output =~ m/gzip: stdout: Broken pipe/ || $output =~ m/zstd: error 70 : Write error.*Broken pipe/) { $broken_pipe = 1; } elsif (!defined ($errstring) && $output !~ m/^\s*$/) { $errstring = "Failed to extract config from VMA archive: $output\n"; @@ -1631,6 +1627,8 @@ sub prune_backups { $keep = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'}); } + activate_storage($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); return $plugin->prune_backups($scfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc); } @@ -1668,10 +1666,9 @@ my $prune_mark = sub { sub prune_mark_backup_group { my ($backup_group, $keep) = @_; - my $keep_all = delete $keep->{'keep-all'}; + my @positive_opts = grep { $_ ne 'keep-all' && $keep->{$_} > 0 } keys $keep->%*; - if ($keep_all || !scalar(grep {$_ > 0} values %{$keep})) { - $keep = { 'keep-all' => 1 } if $keep_all; + if ($keep->{'keep-all'} || scalar(@positive_opts) == 0) { foreach my $prune_entry (@{$backup_group}) { $prune_entry->{mark} = 'keep'; } @@ -1717,7 +1714,7 @@ sub prune_mark_backup_group { } } -sub volume_export { +sub volume_export : prototype($$$$$$$) { my ($cfg, $fh, $volid, $format, $snapshot, $base_snapshot, $with_snapshots) = @_; my ($storeid, $volname) = parse_volume_id($volid, 1); @@ -1728,18 +1725,27 @@ sub volume_export { $snapshot, $base_snapshot, $with_snapshots); } -sub volume_import { - my ($cfg, $fh, $volid, $format, $base_snapshot, $with_snapshots, $allow_rename) = @_; +sub volume_import : prototype($$$$$$$$) { + my ($cfg, $fh, $volid, $format, $snapshot, $base_snapshot, $with_snapshots, $allow_rename) = @_; my ($storeid, $volname) = parse_volume_id($volid, 1); die "cannot import into volume '$volid'\n" if !$storeid; my $scfg = storage_config($cfg, $storeid); my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); - return $plugin->volume_import($scfg, $storeid, $fh, $volname, $format, - $base_snapshot, $with_snapshots, $allow_rename) // $volid; -} - -sub volume_export_formats { + return $plugin->volume_import( + $scfg, + $storeid, + $fh, + $volname, + $format, + $snapshot, + $base_snapshot, + $with_snapshots, + $allow_rename, + ) // $volid; +} + +sub volume_export_formats : prototype($$$$$) { my ($cfg, $volid, $snapshot, $base_snapshot, $with_snapshots) = @_; my ($storeid, $volname) = parse_volume_id($volid, 1); @@ -1751,21 +1757,27 @@ sub volume_export_formats { $with_snapshots); } -sub volume_import_formats { - my ($cfg, $volid, $base_snapshot, $with_snapshots) = @_; +sub volume_import_formats : prototype($$$$$) { + my ($cfg, $volid, $snapshot, $base_snapshot, $with_snapshots) = @_; my ($storeid, $volname) = parse_volume_id($volid, 1); return if !$storeid; my $scfg = storage_config($cfg, $storeid); my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); - return $plugin->volume_import_formats($scfg, $storeid, $volname, - $base_snapshot, $with_snapshots); + return $plugin->volume_import_formats( + $scfg, + $storeid, + $volname, + $snapshot, + $base_snapshot, + $with_snapshots, + ); } sub volume_transfer_formats { my ($cfg, $src_volid, $dst_volid, $snapshot, $base_snapshot, $with_snapshots) = @_; my @export_formats = volume_export_formats($cfg, $src_volid, $snapshot, $base_snapshot, $with_snapshots); - my @import_formats = volume_import_formats($cfg, $dst_volid, $base_snapshot, $with_snapshots); + my @import_formats = volume_import_formats($cfg, $dst_volid, $snapshot, $base_snapshot, $with_snapshots); my %import_hash = map { $_ => 1 } @import_formats; my @common = grep { $import_hash{$_} } @export_formats; return @common; @@ -1853,7 +1865,7 @@ sub get_bandwidth_limit { my ($operation, $storage_list, $override) = @_; # called for each limit (global, per-storage) with the 'default' and the - # $operation limit and should udpate $override for every limit affecting + # $operation limit and should update $override for every limit affecting # us. my $use_global_limits = 0; my $apply_limit = sub { @@ -1932,4 +1944,16 @@ sub assert_sid_unused { return undef; } +# removes leading/trailing spaces and (back)slashes completely +# substitutes every non-ASCII-alphanumerical char with '_', except '_.-' +sub normalize_content_filename { + my ($filename) = @_; + + chomp $filename; + $filename =~ s/^.*[\/\\]//; + $filename =~ s/[^a-zA-Z0-9_.-]/_/g; + + return $filename; +} + 1;