use strict;
use warnings;
-use File::Path;
use File::Basename;
-use PVE::Tools;
+use File::Path;
+use POSIX qw(ENOENT);
+
+use PVE::Cluster;
+use PVE::Exception qw(raise_param_exc);
use PVE::INotify;
-use PVE::Cluster qw(cfs_read_file);
-use PVE::Storage;
-use PVE::API2::Storage::Content;
+use PVE::JSONSchema qw(get_standard_option);
use PVE::RESTHandler;
use PVE::RPCEnvironment;
-use PVE::JSONSchema qw(get_standard_option);
-use PVE::Exception qw(raise_param_exc);
+use PVE::RRD;
+use PVE::Tools qw(run_command);
+
+use PVE::API2::Storage::Content;
+use PVE::API2::Storage::FileRestore;
+use PVE::API2::Storage::PruneBackups;
+use PVE::Storage;
use base qw(PVE::RESTHandler);
__PACKAGE__->register_method ({
- subclass => "PVE::API2::Storage::Content",
+ subclass => "PVE::API2::Storage::PruneBackups",
+ path => '{storage}/prunebackups',
+});
+
+__PACKAGE__->register_method ({
+ subclass => "PVE::API2::Storage::Content",
# set fragment delimiter (no subdirs) - we need that, because volume
- # IDs may contain a slash '/'
- fragmentDelimiter => '',
+ # IDs may contain a slash '/'
+ fragmentDelimiter => '',
path => '{storage}/content',
});
__PACKAGE__->register_method ({
- name => 'index',
+ subclass => "PVE::API2::Storage::FileRestore",
+ path => '{storage}/file-restore',
+});
+
+__PACKAGE__->register_method ({
+ name => 'index',
path => '',
method => 'GET',
description => "Get status for all datastores.",
- permissions => {
+ permissions => {
description => "Only list entries where you have 'Datastore.Audit' or 'Datastore.AllocateSpace' permissions on '/storage/<storage>'",
user => 'all',
},
additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
- storage => get_standard_option
- ('pve-storage-id', {
- description => "Only list status for specified storage",
- optional => 1,
- }),
- content => {
+ storage => get_standard_option('pve-storage-id', {
+ description => "Only list status for specified storage",
+ optional => 1,
+ completion => \&PVE::Storage::complete_storage_enabled,
+ }),
+ content => {
description => "Only list stores which support this content type.",
- type => 'string', format => 'pve-storage-content',
+ type => 'string', format => 'pve-storage-content-list',
+ optional => 1,
+ completion => \&PVE::Storage::complete_content_type,
+ },
+ enabled => {
+ description => "Only list stores which are enabled (not disabled in config).",
+ type => 'boolean',
optional => 1,
+ default => 0,
+ },
+ target => get_standard_option('pve-node', {
+ description => "If target is different to 'node', we only lists shared storages which " .
+ "content is accessible on this 'node' and the specified 'target' node.",
+ optional => 1,
+ completion => \&PVE::Cluster::get_nodelist,
+ }),
+ 'format' => {
+ description => "Include information about formats",
+ type => 'boolean',
+ optional => 1,
+ default => 0,
},
},
},
type => 'array',
items => {
type => "object",
- properties => { storage => { type => 'string' } },
+ properties => {
+ storage => get_standard_option('pve-storage-id'),
+ type => {
+ description => "Storage type.",
+ type => 'string',
+ },
+ content => {
+ description => "Allowed storage content types.",
+ type => 'string', format => 'pve-storage-content-list',
+ },
+ enabled => {
+ description => "Set when storage is enabled (not disabled).",
+ type => 'boolean',
+ optional => 1,
+ },
+ active => {
+ description => "Set when storage is accessible.",
+ type => 'boolean',
+ optional => 1,
+ },
+ shared => {
+ description => "Shared flag from storage configuration.",
+ type => 'boolean',
+ optional => 1,
+ },
+ total => {
+ description => "Total storage space in bytes.",
+ type => 'integer',
+ renderer => 'bytes',
+ optional => 1,
+ },
+ used => {
+ description => "Used storage space in bytes.",
+ type => 'integer',
+ renderer => 'bytes',
+ optional => 1,
+ },
+ avail => {
+ description => "Available storage space in bytes.",
+ type => 'integer',
+ renderer => 'bytes',
+ optional => 1,
+ },
+ used_fraction => {
+ description => "Used fraction (used/total).",
+ type => 'number',
+ renderer => 'fraction_as_percentage',
+ optional => 1,
+ },
+ },
},
links => [ { rel => 'child', href => "{storage}" } ],
},
my $rpcenv = PVE::RPCEnvironment::get();
my $authuser = $rpcenv->get_user();
- my $cfg = cfs_read_file("storage.cfg");
+ my $localnode = PVE::INotify::nodename();
- my $info = PVE::Storage::storage_info($cfg, $param->{content});
+ my $target = $param->{target};
+
+ undef $target if $target && ($target eq $localnode || $target eq 'localhost');
+
+ my $cfg = PVE::Storage::config();
+
+ my $info = PVE::Storage::storage_info($cfg, $param->{content}, $param->{format});
raise_param_exc({ storage => "No such storage." })
if $param->{storage} && !defined($info->{$param->{storage}});
-
+
my $res = {};
my @sids = PVE::Storage::storage_ids($cfg);
foreach my $storeid (@sids) {
+ my $data = $info->{$storeid};
+ next if !$data;
my $privs = [ 'Datastore.Audit', 'Datastore.AllocateSpace' ];
next if !$rpcenv->check_any($authuser, "/storage/$storeid", $privs, 1);
next if $param->{storage} && $param->{storage} ne $storeid;
- $res->{$storeid} = $info->{$storeid};
+
+ my $scfg = PVE::Storage::storage_config($cfg, $storeid);
+
+ next if $param->{enabled} && $scfg->{disable};
+
+ if ($target) {
+ # check if storage content is accessible on local node and specified target node
+ # we use this on the Clone GUI
+
+ next if !$scfg->{shared};
+ next if !PVE::Storage::storage_check_node($cfg, $storeid, undef, 1);
+ next if !PVE::Storage::storage_check_node($cfg, $storeid, $target, 1);
+ }
+
+ if ($data->{total}) {
+ $data->{used_fraction} = ($data->{used} // 0) / $data->{total};
+ }
+
+ $res->{$storeid} = $data;
}
return PVE::RESTHandler::hash_to_array($res, 'storage');
__PACKAGE__->register_method ({
name => 'diridx',
- path => '{storage}',
+ path => '{storage}',
method => 'GET',
description => "",
- permissions => {
+ permissions => {
check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
},
parameters => {
my ($param) = @_;
my $res = [
- { subdir => 'status' },
{ subdir => 'content' },
- { subdir => 'upload' },
+ { subdir => 'download-url' },
+ { subdir => 'file-restore' },
+ { subdir => 'prunebackups' },
{ subdir => 'rrd' },
{ subdir => 'rrddata' },
- ];
-
+ { subdir => 'status' },
+ { subdir => 'upload' },
+ ];
+
return $res;
}});
__PACKAGE__->register_method ({
name => 'read_status',
- path => '{storage}/status',
+ path => '{storage}/status',
method => 'GET',
description => "Read storage status.",
- permissions => {
+ permissions => {
check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
},
protected => 1,
code => sub {
my ($param) = @_;
- my $cfg = cfs_read_file("storage.cfg");
+ my $cfg = PVE::Storage::config();
my $info = PVE::Storage::storage_info($cfg, $param->{content});
raise_param_exc({ storage => "No such storage." })
if !defined($data);
-
+
return $data;
}});
__PACKAGE__->register_method ({
name => 'rrd',
- path => '{storage}/rrd',
+ path => '{storage}/rrd',
method => 'GET',
description => "Read storage RRD statistics (returns PNG).",
- permissions => {
+ permissions => {
check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
},
protected => 1,
code => sub {
my ($param) = @_;
- return PVE::Cluster::create_rrd_graph(
- "pve2-storage/$param->{node}/$param->{storage}",
+ return PVE::RRD::create_rrd_graph(
+ "pve2-storage/$param->{node}/$param->{storage}",
$param->{timeframe}, $param->{ds}, $param->{cf});
-
}});
__PACKAGE__->register_method ({
name => 'rrddata',
- path => '{storage}/rrddata',
+ path => '{storage}/rrddata',
method => 'GET',
description => "Read storage RRD statistics.",
- permissions => {
+ permissions => {
check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
},
protected => 1,
code => sub {
my ($param) = @_;
- return PVE::Cluster::create_rrd_data(
- "pve2-storage/$param->{node}/$param->{storage}",
- $param->{timeframe}, $param->{cf});
+ return PVE::RRD::create_rrd_data(
+ "pve2-storage/$param->{node}/$param->{storage}",
+ $param->{timeframe}, $param->{cf});
}});
+# makes no sense for big images and backup files (because it
+# create a copy of the file).
__PACKAGE__->register_method ({
name => 'upload',
- path => '{storage}/upload',
+ path => '{storage}/upload',
method => 'POST',
- description => "Upload file.",
- permissions => {
- check => ['perm', '/storage/{storage}', ['Datastore.AllocateSpace']],
+ description => "Upload templates and ISO images.",
+ permissions => {
+ check => ['perm', '/storage/{storage}', ['Datastore.AllocateTemplate']],
},
protected => 1,
parameters => {
properties => {
node => get_standard_option('pve-node'),
storage => get_standard_option('pve-storage-id'),
- content => {
+ content => {
description => "Content type.",
type => 'string', format => 'pve-storage-content',
+ enum => ['iso', 'vztmpl'],
},
- filename => {
- description => "The name of the file to create.",
+ filename => {
+ description => "The name of the file to create. Caution: This will be normalized!",
+ maxLength => 255,
type => 'string',
},
- tmpfilename => {
- description => "The source file name. This parameter is usually set by the REST handler. You can only overwrite it when connecting to the trustet port on localhost.",
+ tmpfilename => {
+ description => "The source file name. This parameter is usually set by the REST handler. You can only overwrite it when connecting to the trusted port on localhost.",
type => 'string',
optional => 1,
},
my $user = $rpcenv->get_user();
- my $cfg = cfs_read_file("storage.cfg");
+ my $cfg = PVE::Storage::config();
my $node = $param->{node};
my $scfg = PVE::Storage::storage_check_enabled($cfg, $param->{storage}, $node);
- die "cant upload to storage type '$scfg->{type}'"
- if !($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs');
+ die "can't upload to storage type '$scfg->{type}'\n"
+ if !defined($scfg->{path});
my $content = $param->{content};
die "missing temporary file name\n" if !$tmpfilename;
my $size = -s $tmpfilename;
- die "temporary file '$tmpfilename' does not exists\n" if !defined($size);
+ die "temporary file '$tmpfilename' does not exist\n" if !defined($size);
- my $filename = $param->{filename};
-
- chomp $filename;
- $filename =~ s/^.*[\/\\]//;
- $filename =~ s/\s/_/g;
+ my $filename = PVE::Storage::normalize_content_filename($param->{filename});
my $path;
if ($content eq 'iso') {
- if ($filename !~ m![^/]+\.[Ii][Ss][Oo]$!) {
- raise_param_exc({ filename => "missing '.iso' extension" });
+ if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
+ raise_param_exc({ filename => "wrong file extension" });
}
$path = PVE::Storage::get_iso_dir($cfg, $param->{storage});
} elsif ($content eq 'vztmpl') {
- if ($filename !~ m![^/]+\.tar\.gz$!) {
- raise_param_exc({ filename => "missing '.tar.gz' extension" });
+ if ($filename !~ m![^/]+$PVE::Storage::vztmpl_extension_re$!) {
+ raise_param_exc({ filename => "wrong file extension" });
}
$path = PVE::Storage::get_vztmpl_dir($cfg, $param->{storage});
- } elsif ($content eq 'backup') {
- if ($filename !~ m!/([^/]+\.(tar|tgz))$!) {
- raise_param_exc({ filename => "missing '.(tar|tgz)' extension" });
- }
- $path = PVE::Storage::get_backup_dir($cfg, $param->{storage});
} else {
- raise_param_exc({ content => "upload content type '$content' not implemented" });
+ raise_param_exc({ content => "upload content type '$content' not allowed" });
}
- die "storage '$param->{storage}' does not support '$content' content\n"
+ die "storage '$param->{storage}' does not support '$content' content\n"
if !$scfg->{content}->{$content};
my $dest = "$path/$filename";
my $dirname = dirname($dest);
- # we simply overwrite when destination when file already exists
+ # best effort to match apl_download behaviour
+ chmod 0644, $tmpfilename;
+
+ my $err_cleanup = sub { unlink $dest, $tmpfilename; die "cleanup failed: $!" if $! && $! != ENOENT };
my $cmd;
if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
my $remip = PVE::Cluster::remote_node_ip($node);
- my @ssh_options = ('-o', 'BatchMode=yes', '-c', 'blowfish-cbc');
+ my @ssh_options = ('-o', 'BatchMode=yes');
- my @remcmd = ('/usr/bin/ssh', @ssh_options, $remip);
+ my @remcmd = ('/usr/bin/ssh', @ssh_options, $remip, '--');
- eval {
- # activate remote storage
- PVE::Tools::run_command([@remcmd, '/usr/sbin/pvesm', 'status',
- '--storage', $param->{storage}]);
+ eval { # activate remote storage
+ run_command([@remcmd, '/usr/sbin/pvesm', 'status', '--storage', $param->{storage}]);
};
- die "can't activate storage '$param->{storage}' on node '$node'\n" if $@;
+ die "can't activate storage '$param->{storage}' on node '$node': $@\n" if $@;
- PVE::Tools::run_command([@remcmd, '/bin/mkdir', '-p', $dirname],
- errmsg => "mkdir failed");
+ run_command(
+ [@remcmd, '/bin/mkdir', '-p', '--', PVE::Tools::shell_quote($dirname)],
+ errmsg => "mkdir failed",
+ );
- $cmd = ['/usr/bin/scp', @ssh_options, $tmpfilename, "$remip:$dest"];
+ $cmd = ['/usr/bin/scp', @ssh_options, '-p', '--', $tmpfilename, "[$remip]:" . PVE::Tools::shell_quote($dest)];
+
+ $err_cleanup = sub { run_command([@remcmd, 'rm', '-f', '--', $dest, $tmpfilename]) };
} else {
PVE::Storage::activate_storage($cfg, $param->{storage});
File::Path::make_path($dirname);
- $cmd = ['cp', $tmpfilename, $dest];
+ $cmd = ['cp', '--', $tmpfilename, $dest];
}
- my $worker = sub {
+ # NOTE: we simply overwrite the destination file if it already exists
+ my $worker = sub {
my $upid = shift;
-
+
print "starting file import from: $tmpfilename\n";
print "target node: $node\n";
print "target file: $dest\n";
print "file size is: $size\n";
print "command: " . join(' ', @$cmd) . "\n";
- eval { PVE::Tools::run_command($cmd, errmsg => 'import failed'); };
+ eval { run_command($cmd, errmsg => 'import failed'); };
+
+ unlink $tmpfilename; # the temporary file got only uploaded locally, no need to rm remote
+ warn "unable to clean up temporary file '$tmpfilename' - $!\n" if $! && $! != ENOENT;
+
if (my $err = $@) {
- unlink $dest;
+ eval { $err_cleanup->() };
+ warn "$@" if $@;
die $err;
}
print "finished file import successfully\n";
};
- return $rpcenv->fork_worker('imgcopy', undef, $user, $worker);
+ my $upid = $rpcenv->fork_worker('imgcopy', undef, $user, $worker);
+
+ # apache removes the temporary file on return, so we need
+ # to wait here to make sure the worker process starts and
+ # opens the file before it gets removed.
+ sleep(1);
+
+ return $upid;
}});
-
+
+__PACKAGE__->register_method({
+ name => 'download_url',
+ path => '{storage}/download-url',
+ method => 'POST',
+ description => "Download templates and ISO images by using an URL.",
+ proxyto => 'node',
+ permissions => {
+ check => [ 'and',
+ ['perm', '/storage/{storage}', [ 'Datastore.AllocateTemplate' ]],
+ ['perm', '/', [ 'Sys.Audit', 'Sys.Modify' ]],
+ ],
+ },
+ protected => 1,
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ storage => get_standard_option('pve-storage-id'),
+ url => {
+ description => "The URL to download the file from.",
+ type => 'string',
+ pattern => 'https?://.*',
+ },
+ content => {
+ description => "Content type.", # TODO: could be optional & detected in most cases
+ type => 'string', format => 'pve-storage-content',
+ enum => ['iso', 'vztmpl'],
+ },
+ filename => {
+ description => "The name of the file to create. Caution: This will be normalized!",
+ maxLength => 255,
+ type => 'string',
+ },
+ checksum => {
+ description => "The expected checksum of the file.",
+ type => 'string',
+ requires => 'checksum-algorithm',
+ optional => 1,
+ },
+ 'checksum-algorithm' => {
+ description => "The algorithm to calculate the checksum of the file.",
+ type => 'string',
+ enum => ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'],
+ requires => 'checksum',
+ optional => 1,
+ },
+ 'verify-certificates' => {
+ description => "If false, no SSL/TLS certificates will be verified.",
+ type => 'boolean',
+ optional => 1,
+ default => 1,
+ },
+ },
+ },
+ returns => {
+ type => "string"
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $user = $rpcenv->get_user();
+
+ my $cfg = PVE::Storage::config();
+
+ my ($node, $storage) = $param->@{'node', 'storage'};
+ my $scfg = PVE::Storage::storage_check_enabled($cfg, $storage, $node);
+
+ die "can't upload to storage type '$scfg->{type}', not a file based storage!\n"
+ if !defined($scfg->{path});
+
+ my ($content, $url) = $param->@{'content', 'url'};
+
+ die "storage '$storage' is not configured for content-type '$content'\n"
+ if !$scfg->{content}->{$content};
+
+ my $filename = PVE::Storage::normalize_content_filename($param->{filename});
+
+ my $path;
+ if ($content eq 'iso') {
+ if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
+ raise_param_exc({ filename => "wrong file extension" });
+ }
+ $path = PVE::Storage::get_iso_dir($cfg, $storage);
+ } elsif ($content eq 'vztmpl') {
+ if ($filename !~ m![^/]+$PVE::Storage::vztmpl_extension_re$!) {
+ raise_param_exc({ filename => "wrong file extension" });
+ }
+ $path = PVE::Storage::get_vztmpl_dir($cfg, $storage);
+ } else {
+ raise_param_exc({ content => "upload content-type '$content' is not allowed" });
+ }
+
+ PVE::Storage::activate_storage($cfg, $storage);
+ File::Path::make_path($path);
+
+ my $dccfg = PVE::Cluster::cfs_read_file('datacenter.cfg');
+ my $opts = {
+ hash_required => 0,
+ verify_certificates => $param->{'verify-certificates'} // 1,
+ http_proxy => $dccfg->{http_proxy},
+ };
+
+ my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm'};
+ if ($checksum) {
+ $opts->{"${checksum_algorithm}sum"} = $checksum;
+ $opts->{hash_required} = 1;
+ }
+
+ my $worker = sub {
+ PVE::Tools::download_file_from_url("$path/$filename", $url, $opts);
+ };
+
+ my $worker_id = PVE::Tools::encode_text($filename); # must not pass : or the like as w-ID
+
+ return $rpcenv->fork_worker('download', $worker_id, $user, $worker);
+ }});
+
1;
projects / pve-storage.git / blobdiff