X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=PVE%2FStorage.pm;h=7a65624ee4c2d2d06db90c45ccb6dc788c700b2d;hb=ae36189d26ea64ab0cb174b9991f6ff442554bc7;hp=bb35b32a4376fb90a2906c391f6285854dfa5884;hpb=e83f7b40959f014a9df3f437d8c4206416e6ad0d;p=pve-storage.git diff --git a/PVE/Storage.pm b/PVE/Storage.pm index bb35b32..7a65624 100755 --- a/PVE/Storage.pm +++ b/PVE/Storage.pm @@ -7,12 +7,13 @@ use Data::Dumper; use POSIX; use IO::Select; use IO::File; +use IO::Socket::IP; use File::Basename; use File::Path; use Cwd 'abs_path'; use Socket; -use PVE::Tools qw(run_command file_read_firstline $IPV6RE); +use PVE::Tools qw(run_command file_read_firstline dir_glob_foreach $IPV6RE); use PVE::Cluster qw(cfs_read_file cfs_write_file cfs_lock_file); use PVE::Exception qw(raise_param_exc); use PVE::JSONSchema; @@ -33,7 +34,10 @@ use PVE::Storage::ZFSPoolPlugin; use PVE::Storage::ZFSPlugin; use PVE::Storage::DRBDPlugin; -# load and initialize all plugins +# Storage API version. Icrement it on changes in storage API interface. +use constant APIVER => 1; + +# load standard plugins PVE::Storage::DirPlugin->register(); PVE::Storage::LVMPlugin->register(); PVE::Storage::LvmThinPlugin->register(); @@ -46,6 +50,34 @@ PVE::Storage::GlusterfsPlugin->register(); PVE::Storage::ZFSPoolPlugin->register(); PVE::Storage::ZFSPlugin->register(); PVE::Storage::DRBDPlugin->register(); + +# load third-party plugins +if ( -d '/usr/share/perl5/PVE/Storage/Custom' ) { + dir_glob_foreach('/usr/share/perl5/PVE/Storage/Custom', '.*\.pm$', sub { + my ($file) = @_; + my $modname = 'PVE::Storage::Custom::' . $file; + $modname =~ s!\.pm$!!; + $file = 'PVE/Storage/Custom/' . $file; + + eval { + require $file; + }; + if ($@) { + warn $@; + # Check storage API version and that file is really storage plugin. + } elsif ($modname->isa('PVE::Storage::Plugin') && $modname->can('api') && $modname->api() == APIVER) { + eval { + import $file; + $modname->register(); + }; + warn $@ if $@; + } else { + warn "Error loading storage plugin \"$modname\" because of API version mismatch. Please, update it.\n" + } + }); +} + +# initialize all plugins PVE::Storage::Plugin->init(); my $UDEVADM = '/sbin/udevadm'; @@ -75,7 +107,7 @@ sub lock_storage_config { sub storage_config { my ($cfg, $storeid, $noerr) = @_; - die "no storage id specified\n" if !$storeid; + die "no storage ID specified\n" if !$storeid; my $scfg = $cfg->{ids}->{$storeid}; @@ -231,6 +263,22 @@ sub volume_has_feature { } } +sub volume_snapshot_list { + my ($cfg, $volid) = @_; + + my ($storeid, $volname) = parse_volume_id($volid, 1); + if ($storeid) { + my $scfg = storage_config($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + return $plugin->volume_snapshot_list($scfg, $storeid, $volname); + } elsif ($volid =~ m|^(/.+)$| && -e $volid) { + die "send file/device '$volid' is not possible\n"; + } else { + die "unable to parse volume ID '$volid'\n"; + } + # return an empty array if dataset does not exist. +} + sub get_image_dir { my ($cfg, $storeid, $vmid) = @_; @@ -290,6 +338,9 @@ sub parse_vmid { return int($vmid); } +# NOTE: basename and basevmid are always undef for LVM-thin, where the +# clone -> base reference is not encoded in the volume ID. +# see note in PVE::Storage::LvmThinPlugin for details. sub parse_volname { my ($cfg, $volid) = @_; @@ -310,25 +361,71 @@ sub parse_volume_id { return PVE::Storage::Plugin::parse_volume_id($volid, $noerr); } -sub volume_is_base { - my ($cfg, $volid) = @_; +# test if we have read access to volid +sub check_volume_access { + my ($rpcenv, $user, $cfg, $vmid, $volid) = @_; my ($sid, $volname) = parse_volume_id($volid, 1); - return 0 if !$sid; - - if (my $scfg = $cfg->{ids}->{$sid}) { - my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); - my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) = - $plugin->parse_volname($volname); - return $isBase ? 1 : 0; - } else { - # stale volid with undefined storage - so we can just guess - if ($volid =~ m/base-/) { - return 1; + if ($sid) { + my ($vtype, undef, $ownervm) = parse_volname($cfg, $volid); + if ($vtype eq 'iso' || $vtype eq 'vztmpl') { + # we simply allow access + } elsif (defined($ownervm) && defined($vmid) && ($ownervm == $vmid)) { + # we are owner - allow access + } elsif ($vtype eq 'backup' && $ownervm) { + $rpcenv->check($user, "/storage/$sid", ['Datastore.AllocateSpace']); + $rpcenv->check($user, "/vms/$ownervm", ['VM.Backup']); + } else { + # allow if we are Datastore administrator + $rpcenv->check($user, "/storage/$sid", ['Datastore.Allocate']); } + } else { + die "Only root can pass arbitrary filesystem paths." + if $user ne 'root@pam'; } + return undef; +} + +my $volume_is_base_and_used__no_lock = sub { + my ($scfg, $storeid, $plugin, $volname) = @_; + + my ($vtype, $name, $vmid, undef, undef, $isBase, undef) = + $plugin->parse_volname($volname); + + if ($isBase) { + my $vollist = $plugin->list_images($storeid, $scfg); + foreach my $info (@$vollist) { + my (undef, $tmpvolname) = parse_volume_id($info->{volid}); + my $basename = undef; + my $basevmid = undef; + + eval{ + (undef, undef, undef, $basename, $basevmid) = + $plugin->parse_volname($tmpvolname); + }; + + if ($basename && defined($basevmid) && $basevmid == $vmid && $basename eq $name) { + return 1; + } + } + } return 0; +}; + +# NOTE: this check does not work for LVM-thin, where the clone -> base +# reference is not encoded in the volume ID. +# see note in PVE::Storage::LvmThinPlugin for details. +sub volume_is_base_and_used { + my ($cfg, $volid) = @_; + + my ($storeid, $volname) = parse_volume_id($volid); + my $scfg = storage_config($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + + $plugin->cluster_lock_storage($storeid, $scfg->{shared}, undef, sub { + return &$volume_is_base_and_used__no_lock($scfg, $storeid, $plugin, $volname); + }); } # try to map a filesystem path to a volume identifier @@ -428,7 +525,7 @@ sub abs_filesystem_path { } sub storage_migrate { - my ($cfg, $volid, $target_host, $target_storeid, $target_volname) = @_; + my ($cfg, $volid, $target_sshinfo, $target_storeid, $target_volname, $base_snapshot, $snapshot, $ratelimit_bps, $insecure) = @_; my ($storeid, $volname) = parse_volume_id($volid); $target_volname = $volname if !$target_volname; @@ -442,17 +539,34 @@ sub storage_migrate { my $target_volid = "${target_storeid}:${target_volname}"; - my $errstr = "unable to migrate '$volid' to '${target_volid}' on host '$target_host'"; + my $target_ip = $target_sshinfo->{ip}; + my $errstr = "unable to migrate '$volid' to '${target_volid}' on host '$target_sshinfo->{name}'"; - my $sshoptions = "-o 'BatchMode=yes'"; - my $ssh = "/usr/bin/ssh $sshoptions"; + my $ssh = PVE::Cluster::ssh_info_to_command($target_sshinfo); + my $ssh_base = PVE::Cluster::ssh_info_to_command_base($target_sshinfo); + local $ENV{RSYNC_RSH} = PVE::Tools::cmd2string($ssh_base); - local $ENV{RSYNC_RSH} = $ssh; + my $no_incremental = sub { + my ($type) = @_; + die "incremental migration not supported on storage type $type\n" + if defined($base_snapshot); + }; + my $no_snapshot = sub { + my ($type) = @_; + # $snapshot is currently only used by replication + die "replicating storage migration not supported on storage type $type\n" + if defined($snapshot); + }; + + my @cstream = ([ '/usr/bin/cstream', '-t', $ratelimit_bps ]) + if defined($ratelimit_bps); # only implemented for file system based storage if ($scfg->{path}) { - if ($tcfg->{path}) { + $no_incremental->($scfg->{type}); + $no_snapshot->($scfg->{type}); + if ($tcfg->{path}) { my $src_plugin = PVE::Storage::Plugin->lookup($scfg->{type}); my $dst_plugin = PVE::Storage::Plugin->lookup($tcfg->{type}); my $src = $src_plugin->path($scfg, $volname, $storeid); @@ -467,27 +581,28 @@ sub storage_migrate { run_command(['/bin/cp', $src, $dst]); } else { - run_command(['/usr/bin/ssh', "root\@${target_host}", - '/bin/mkdir', '-p', $dirname]); + run_command([@$ssh, '/bin/mkdir', '-p', $dirname]); # we use rsync with --sparse, so we can't use --inplace, # so we remove file on the target if it already exists to # save space my ($size, $format) = PVE::Storage::Plugin::file_size_info($src); if ($format && ($format eq 'raw') && $size) { - run_command(['/usr/bin/ssh', "root\@${target_host}", - 'rm', '-f', $dst], + run_command([@$ssh, 'rm', '-f', $dst], outfunc => sub {}); } my $cmd; + my @bwlimit = ("--bwlimit=${ratelimit_bps}b") if defined($ratelimit_bps); if ($format eq 'subvol') { $cmd = ['/usr/bin/rsync', '--progress', '-X', '-A', '--numeric-ids', '-aH', '--delete', '--no-whole-file', '--inplace', - '--one-file-system', "$src/", "[root\@${target_host}]:$dst"]; + '--one-file-system', @bwlimit, + "$src/", "[root\@${target_ip}]:$dst"]; } else { $cmd = ['/usr/bin/rsync', '--progress', '--sparse', '--whole-file', - $src, "[root\@${target_host}]:$dst"]; + @bwlimit, + $src, "[root\@${target_ip}]:$dst"]; } my $percent = -1; @@ -515,35 +630,70 @@ sub storage_migrate { if ($tcfg->{type} eq 'zfspool') { - die "$errstr - pool on target has not same name as source!" + die "$errstr - pool on target does not have the same name as on source!" if $tcfg->{pool} ne $scfg->{pool}; - my (undef, $volname) = parse_volname($cfg, $volid); + my $migration_snapshot; + if (!defined($snapshot)) { + $migration_snapshot = 1; + $snapshot = '__migration__'; + } + my (undef, $volname) = parse_volname($cfg, $volid); my $zfspath = "$scfg->{pool}\/$volname"; - my $snap = ['zfs', 'snapshot', "$zfspath\@__migration__"]; + my @formats = volume_transfer_formats($cfg, $volid, $volid, $snapshot, $base_snapshot, 1); + die "cannot migrate from storage type '$scfg->{type}' to '$tcfg->{type}'\n" if !@formats; + my $format = $formats[0]; + + my @insecurecmd; + if ($insecure) { + @insecurecmd = ('pvecm', 'mtunnel', '-run-command', 1); + if (my $network = $target_sshinfo->{network}) { + push @insecurecmd, '-migration_network', $network; + } + } - my $send = [['zfs', 'send', '-Rpv', "$zfspath\@__migration__"], ['ssh', "root\@$target_host", - 'zfs', 'recv', $zfspath]]; + my $send = ['pvesm', 'export', $volid, $format, '-', '-snapshot', $snapshot, '-with-snapshots', '1']; + my $recv = [@$ssh, @insecurecmd, '--', 'pvesm', 'import', $volid, $format, '-', '-with-snapshots', '1']; + if ($migration_snapshot) { + push @$recv, '-delete-snapshot', $snapshot; + } - my $destroy_target = ['ssh', "root\@$target_host", 'zfs', 'destroy', "$zfspath\@__migration__"]; - run_command($snap); - eval{ - run_command($send); - }; - my $err; - if ($err = $@){ - run_command(['zfs', 'destroy', "$zfspath\@__migration__"]); - die $err; + if (defined($base_snapshot)) { + # Check if the snapshot exists on the remote side: + push @$send, '-base', $base_snapshot; + push @$recv, '-base', $base_snapshot; } - run_command($destroy_target); + volume_snapshot($cfg, $volid, $snapshot) if $migration_snapshot; + eval { + if ($insecure) { + my $pid = open(my $info, '-|', @$recv) + or die "receive command failed: $!\n"; + my ($ip) = <$info> =~ /^($PVE::Tools::IPRE)$/ or die "no tunnel IP received\n"; + my ($port) = <$info> =~ /^(\d+)$/ or die "no tunnel port received\n"; + my $socket = IO::Socket::IP->new(PeerHost => $ip, PeerPort => $port, Type => SOCK_STREAM) + or die "failed to connect to tunnel at $ip:$port\n"; + run_command([$send, @cstream], output => '>&'.fileno($socket)); + } else { + run_command([$send, @cstream, $recv]); + } + }; + my $err = $@; + warn "send/receive failed, cleaning up snapshot(s)..\n" if $err; + if ($migration_snapshot) { + eval { volume_snapshot_delete($cfg, $volid, $snapshot, 0) }; + warn "could not remove source snapshot: $@\n" if $@; + } + die $err if $err; } else { die "$errstr - target type $tcfg->{type} is not valid\n"; } } elsif ($scfg->{type} eq 'lvmthin' || $scfg->{type} eq 'lvm') { + $no_incremental->($scfg->{type}); + $no_snapshot->($scfg->{type}); if (($scfg->{type} eq $tcfg->{type}) && ($tcfg->{type} eq 'lvmthin' || $tcfg->{type} eq 'lvm')) { @@ -553,22 +703,22 @@ sub storage_migrate { my $src = path($cfg, $volid); my $dst = path($cfg, $target_volid); - run_command(['/usr/bin/ssh', "root\@${target_host}", + run_command([@$ssh, '--', 'pvesm', 'alloc', $target_storeid, $vmid, $target_volname, int($size/1024)]); eval { if ($tcfg->{type} eq 'lvmthin') { - run_command([["dd", "if=$src"],["/usr/bin/ssh", "root\@${target_host}", "-C", - "dd", 'conv=sparse', "of=$dst"]]); + run_command([["dd", "if=$src", "bs=4k"], @cstream, + [@$ssh, "dd", 'conv=sparse', "of=$dst", "bs=4k"]]); } else { - run_command([["dd", "if=$src"],["/usr/bin/ssh", "root\@${target_host}", "-C", - "dd", "of=$dst"]]); + run_command([["dd", "if=$src", "bs=4k"], @cstream, + [@$ssh, "dd", "of=$dst", "bs=4k"]]); } }; if (my $err = $@) { - run_command(['/usr/bin/ssh', "root\@${target_host}", - 'pvesm', 'free', $target_volid]); + run_command([@$ssh, 'pvesm', 'free', $target_volid]); + die $err; } } else { die "$errstr - migrate from source type '$scfg->{type}' to '$tcfg->{type}' not implemented\n"; @@ -617,7 +767,7 @@ sub vdisk_create_base { sub vdisk_alloc { my ($cfg, $storeid, $vmid, $fmt, $name, $size) = @_; - die "no storage id specified\n" if !$storeid; + die "no storage ID specified\n" if !$storeid; PVE::JSONSchema::parse_storage_id($storeid); @@ -650,9 +800,7 @@ sub vdisk_free { my ($cfg, $volid) = @_; my ($storeid, $volname) = parse_volume_id($volid); - my $scfg = storage_config($cfg, $storeid); - my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); activate_storage($cfg, $storeid); @@ -661,27 +809,12 @@ sub vdisk_free { # lock shared storage $plugin->cluster_lock_storage($storeid, $scfg->{shared}, undef, sub { + # LVM-thin allows deletion of still referenced base volumes! + die "base volume '$volname' is still in use by linked clones\n" + if &$volume_is_base_and_used__no_lock($scfg, $storeid, $plugin, $volname); - my ($vtype, $name, $vmid, undef, undef, $isBase, $format) = + my (undef, undef, undef, undef, undef, $isBase, $format) = $plugin->parse_volname($volname); - if ($isBase) { - my $vollist = $plugin->list_images($storeid, $scfg); - foreach my $info (@$vollist) { - my (undef, $tmpvolname) = parse_volume_id($info->{volid}); - my $basename = undef; - my $basevmid = undef; - - eval{ - (undef, undef, undef, $basename, $basevmid) = - $plugin->parse_volname($tmpvolname); - }; - - if ($basename && defined($basevmid) && $basevmid == $vmid && $basename eq $name) { - die "base volume '$volname' is still in use " . - "(use by '$tmpvolname')\n"; - } - } - } $cleanup_worker = $plugin->free_image($storeid, $scfg, $volname, $isBase, $format); }); @@ -960,7 +1093,7 @@ sub deactivate_volumes { } } - die "volume deativation failed: " . join(' ', @errlist) + die "volume deactivation failed: " . join(' ', @errlist) if scalar(@errlist); } @@ -970,9 +1103,9 @@ sub storage_info { my $ids = $cfg->{ids}; my $info = {}; - + my @ctypes = PVE::Tools::split_list($content); - + my $slist = []; foreach my $storeid (keys %$ids) { @@ -988,7 +1121,7 @@ sub storage_info { } next if !$want_ctype; } - + my $type = $ids->{$storeid}->{type}; $info->{$storeid} = { @@ -1021,9 +1154,9 @@ sub storage_info { eval { ($total, $avail, $used, $active) = $plugin->status($storeid, $scfg, $cache); }; warn $@ if $@; next if !$active; - $info->{$storeid}->{total} = $total; - $info->{$storeid}->{avail} = $avail; - $info->{$storeid}->{used} = $used; + $info->{$storeid}->{total} = int($total); + $info->{$storeid}->{avail} = int($avail); + $info->{$storeid}->{used} = int($used); $info->{$storeid}->{active} = $active; } @@ -1262,7 +1395,7 @@ sub extract_vzdump_config_tar { my $file; while (defined($file = <$fh>)) { - if ($file =~ m!$conf_re!) { + if ($file =~ $conf_re) { $file = $1; # untaint last; } @@ -1330,7 +1463,7 @@ sub extract_vzdump_config_vma { my $rerr = $@; # use exit code if no stderr output and not just broken pipe - if (!$errstring && !$broken_pipe && $rc > 0 && $rc != 141) { + if (!$errstring && !$broken_pipe && $rc != 0 && $rc != 141) { die "$rerr\n" if $rerr; die "config extraction failed with exit code $rc\n"; } @@ -1348,9 +1481,9 @@ sub extract_vzdump_config { my $archive = abs_filesystem_path($cfg, $volid); - if ($volid =~ /\/vzdump-(lxc|openvz)-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|(tar(\.(gz|lzo))?))$/) { - return extract_vzdump_config_tar($archive,'^(\./etc/vzdump/(pct|vps)\.conf)$'); - } elsif ($volid =~ /\/vzdump-qemu-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?))$/) { + if ($volid =~ /vzdump-(lxc|openvz)-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|(tar(\.(gz|lzo))?))$/) { + return extract_vzdump_config_tar($archive, qr!^(\./etc/vzdump/(pct|vps)\.conf)$!); + } elsif ($volid =~ /vzdump-qemu-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?))$/) { my $format; my $comp; if ($7 eq 'tgz') { @@ -1371,6 +1504,60 @@ sub extract_vzdump_config { } } +sub volume_export { + my ($cfg, $fh, $volid, $format, $snapshot, $base_snapshot, $with_snapshots) = @_; + + my ($storeid, $volname) = parse_volume_id($volid, 1); + die "cannot export volume '$volid'\n" if !$storeid; + my $scfg = storage_config($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + return $plugin->volume_export($scfg, $storeid, $fh, $volname, $format, + $snapshot, $base_snapshot, $with_snapshots); +} + +sub volume_import { + my ($cfg, $fh, $volid, $format, $base_snapshot, $with_snapshots) = @_; + + my ($storeid, $volname) = parse_volume_id($volid, 1); + die "cannot import into volume '$volid'\n" if !$storeid; + my $scfg = storage_config($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + return $plugin->volume_import($scfg, $storeid, $fh, $volname, $format, + $base_snapshot, $with_snapshots); +} + +sub volume_export_formats { + my ($cfg, $volid, $snapshot, $base_snapshot, $with_snapshots) = @_; + + my ($storeid, $volname) = parse_volume_id($volid, 1); + return if !$storeid; + my $scfg = storage_config($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + return $plugin->volume_export_formats($scfg, $storeid, $volname, + $snapshot, $base_snapshot, + $with_snapshots); +} + +sub volume_import_formats { + my ($cfg, $volid, $base_snapshot, $with_snapshots) = @_; + + my ($storeid, $volname) = parse_volume_id($volid, 1); + return if !$storeid; + my $scfg = storage_config($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + return $plugin->volume_import_formats($scfg, $storeid, $volname, + $base_snapshot, $with_snapshots); +} + +sub volume_transfer_formats { + my ($cfg, $src_volid, $dst_volid, $snapshot, $base_snapshot, $with_snapshots) = @_; + my @export_formats = volume_export_formats($cfg, $src_volid, $snapshot, $base_snapshot, $with_snapshots); + my @import_formats = volume_import_formats($cfg, $dst_volid, $base_snapshot, $with_snapshots); + my %import_hash = map { $_ => 1 } @import_formats; + my @common = grep { $import_hash{$_} } @export_formats; + return @common; +} + # bash completion helper sub complete_storage {