[pve-storage.git] / PVE / API2 / Storage / Status.pm

package PVE::API2::Storage::Status;

use strict;
use warnings;

use File::Basename;
use File::Path;
use POSIX qw(ENOENT);

use PVE::Cluster;
use PVE::Exception qw(raise_param_exc);
use PVE::INotify;
use PVE::JSONSchema qw(get_standard_option);
use PVE::RESTHandler;
use PVE::RPCEnvironment;
use PVE::RRD;
use PVE::Tools qw(run_command);

use PVE::API2::Storage::Content;
use PVE::API2::Storage::FileRestore;
use PVE::API2::Storage::PruneBackups;
use PVE::Storage;

use base qw(PVE::RESTHandler);

__PACKAGE__->register_method ({
    subclass => "PVE::API2::Storage::PruneBackups",
    path => '{storage}/prunebackups',
});

__PACKAGE__->register_method ({
    subclass => "PVE::API2::Storage::Content",
    # set fragment delimiter (no subdirs) - we need that, because volume
    # IDs may contain a slash '/'
    fragmentDelimiter => '',
    path => '{storage}/content',
});

__PACKAGE__->register_method ({
   subclass => "PVE::API2::Storage::FileRestore",
   path => '{storage}/file-restore',
});

__PACKAGE__->register_method ({
    name => 'index',
    path => '',
    method => 'GET',
    description => "Get status for all datastores.",
    permissions => {
	description => "Only list entries where you have 'Datastore.Audit' or 'Datastore.AllocateSpace' permissions on '/storage/<storage>'",
	user => 'all',
    },
    protected => 1,
    proxyto => 'node',
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    storage => get_standard_option('pve-storage-id', {
		description => "Only list status for  specified storage",
		optional => 1,
		completion => \&PVE::Storage::complete_storage_enabled,
	    }),
	    content => {
		description => "Only list stores which support this content type.",
		type => 'string', format => 'pve-storage-content-list',
		optional => 1,
		completion => \&PVE::Storage::complete_content_type,
	    },
	    enabled => {
		description => "Only list stores which are enabled (not disabled in config).",
		type => 'boolean',
		optional => 1,
		default => 0,
	    },
	    target => get_standard_option('pve-node', {
		description => "If target is different to 'node', we only lists shared storages which " .
		    "content is accessible on this 'node' and the specified 'target' node.",
		optional => 1,
		completion => \&PVE::Cluster::get_nodelist,
	    }),
	    'format' => {
		description => "Include information about formats",
		type => 'boolean',
		optional => 1,
		default => 0,
	    },
	},
    },
    returns => {
	type => 'array',
	items => {
	    type => "object",
	    properties => {
		storage => get_standard_option('pve-storage-id'),
		type => {
		    description => "Storage type.",
		    type => 'string',
		},
		content => {
		    description => "Allowed storage content types.",
		    type => 'string', format => 'pve-storage-content-list',
		},
		enabled => {
		    description => "Set when storage is enabled (not disabled).",
		    type => 'boolean',
		    optional => 1,
		},
		active => {
		    description => "Set when storage is accessible.",
		    type => 'boolean',
		    optional => 1,
		},
		shared => {
		    description => "Shared flag from storage configuration.",
		    type => 'boolean',
		    optional => 1,
		},
		total => {
		    description => "Total storage space in bytes.",
		    type => 'integer',
		    renderer => 'bytes',
		    optional => 1,
		},
		used => {
		    description => "Used storage space in bytes.",
		    type => 'integer',
		    renderer => 'bytes',
		    optional => 1,
		},
		avail => {
		    description => "Available storage space in bytes.",
		    type => 'integer',
		    renderer => 'bytes',
		    optional => 1,
		},
		used_fraction => {
		    description => "Used fraction (used/total).",
		    type => 'number',
		    renderer => 'fraction_as_percentage',
		    optional => 1,
		},
	    },
	},
	links => [ { rel => 'child', href => "{storage}" } ],
    },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();
	my $authuser = $rpcenv->get_user();

	my $localnode = PVE::INotify::nodename();

	my $target = $param->{target};

	undef $target if $target && ($target eq $localnode || $target eq 'localhost');

	my $cfg = PVE::Storage::config();

	my $info = PVE::Storage::storage_info($cfg, $param->{content}, $param->{format});

	raise_param_exc({ storage => "No such storage." })
	    if $param->{storage} && !defined($info->{$param->{storage}});

	my $res = {};
	my @sids = PVE::Storage::storage_ids($cfg);
	foreach my $storeid (@sids) {
	    my $data = $info->{$storeid};
	    next if !$data;
	    my $privs = [ 'Datastore.Audit', 'Datastore.AllocateSpace' ];
	    next if !$rpcenv->check_any($authuser, "/storage/$storeid", $privs, 1);
	    next if $param->{storage} && $param->{storage} ne $storeid;

	    my $scfg = PVE::Storage::storage_config($cfg, $storeid);

	    next if $param->{enabled} && $scfg->{disable};

	    if ($target) {
		# check if storage content is accessible on local node and specified target node
		# we use this on the Clone GUI

		next if !$scfg->{shared};
		next if !PVE::Storage::storage_check_node($cfg, $storeid, undef, 1);
		next if !PVE::Storage::storage_check_node($cfg, $storeid, $target, 1);
	    }

	    if ($data->{total}) {
		$data->{used_fraction} = ($data->{used} // 0) / $data->{total};
	    }

	    $res->{$storeid} = $data;
	}

	return PVE::RESTHandler::hash_to_array($res, 'storage');
    }});

__PACKAGE__->register_method ({
    name => 'diridx',
    path => '{storage}',
    method => 'GET',
    description => "",
    permissions => {
	check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
    },
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    storage => get_standard_option('pve-storage-id'),
	},
    },
    returns => {
	type => 'array',
	items => {
	    type => "object",
	    properties => {
		subdir => { type => 'string' },
	    },
	},
	links => [ { rel => 'child', href => "{subdir}" } ],
    },
    code => sub {
	my ($param) = @_;

	my $res = [
	    { subdir => 'content' },
	    { subdir => 'download-url' },
	    { subdir => 'file-restore' },
	    { subdir => 'prunebackups' },
	    { subdir => 'rrd' },
	    { subdir => 'rrddata' },
	    { subdir => 'status' },
	    { subdir => 'upload' },
	];

	return $res;
    }});

__PACKAGE__->register_method ({
    name => 'read_status',
    path => '{storage}/status',
    method => 'GET',
    description => "Read storage status.",
    permissions => {
	check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
    },
    protected => 1,
    proxyto => 'node',
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    storage => get_standard_option('pve-storage-id'),
	},
    },
    returns => {
	type => "object",
	properties => {},
    },
    code => sub {
	my ($param) = @_;

	my $cfg = PVE::Storage::config();

	my $info = PVE::Storage::storage_info($cfg, $param->{content});

	my $data = $info->{$param->{storage}};

	raise_param_exc({ storage => "No such storage." })
	    if !defined($data);

	return $data;
    }});

__PACKAGE__->register_method ({
    name => 'rrd',
    path => '{storage}/rrd',
    method => 'GET',
    description => "Read storage RRD statistics (returns PNG).",
    permissions => {
	check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
    },
    protected => 1,
    proxyto => 'node',
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    storage => get_standard_option('pve-storage-id'),
	    timeframe => {
		description => "Specify the time frame you are interested in.",
		type => 'string',
		enum => [ 'hour', 'day', 'week', 'month', 'year' ],
	    },
	    ds => {
		description => "The list of datasources you want to display.",
 		type => 'string', format => 'pve-configid-list',
	    },
	    cf => {
		description => "The RRD consolidation function",
 		type => 'string',
		enum => [ 'AVERAGE', 'MAX' ],
		optional => 1,
	    },
	},
    },
    returns => {
	type => "object",
	properties => {
	    filename => { type => 'string' },
	},
    },
    code => sub {
	my ($param) = @_;

	return PVE::RRD::create_rrd_graph(
	    "pve2-storage/$param->{node}/$param->{storage}",
	    $param->{timeframe}, $param->{ds}, $param->{cf});
    }});

__PACKAGE__->register_method ({
    name => 'rrddata',
    path => '{storage}/rrddata',
    method => 'GET',
    description => "Read storage RRD statistics.",
    permissions => {
	check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
    },
    protected => 1,
    proxyto => 'node',
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    storage => get_standard_option('pve-storage-id'),
	    timeframe => {
		description => "Specify the time frame you are interested in.",
		type => 'string',
		enum => [ 'hour', 'day', 'week', 'month', 'year' ],
	    },
	    cf => {
		description => "The RRD consolidation function",
 		type => 'string',
		enum => [ 'AVERAGE', 'MAX' ],
		optional => 1,
	    },
	},
    },
    returns => {
	type => "array",
	items => {
	    type => "object",
	    properties => {},
	},
    },
    code => sub {
	my ($param) = @_;

	return PVE::RRD::create_rrd_data(
	    "pve2-storage/$param->{node}/$param->{storage}",
	    $param->{timeframe}, $param->{cf});
    }});

# makes no sense for big images and backup files (because it
# create a copy of the file).
__PACKAGE__->register_method ({
    name => 'upload',
    path => '{storage}/upload',
    method => 'POST',
    description => "Upload templates and ISO images.",
    permissions => {
	check => ['perm', '/storage/{storage}', ['Datastore.AllocateTemplate']],
    },
    protected => 1,
    parameters => {
    	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    storage => get_standard_option('pve-storage-id'),
	    content => {
		description => "Content type.",
		type => 'string', format => 'pve-storage-content',
		enum => ['iso', 'vztmpl'],
	    },
	    filename => {
		description => "The name of the file to create. Caution: This will be normalized!",
		maxLength => 255,
		type => 'string',
	    },
	    tmpfilename => {
		description => "The source file name. This parameter is usually set by the REST handler. You can only overwrite it when connecting to the trusted port on localhost.",
		type => 'string',
		optional => 1,
	    },
	},
    },
    returns => { type => "string" },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();

	my $user = $rpcenv->get_user();

	my $cfg = PVE::Storage::config();

	my $node = $param->{node};
	my $scfg = PVE::Storage::storage_check_enabled($cfg, $param->{storage}, $node);

	die "can't upload to storage type '$scfg->{type}'\n"
	    if !defined($scfg->{path});

	my $content = $param->{content};

	my $tmpfilename = $param->{tmpfilename};
	die "missing temporary file name\n" if !$tmpfilename;

	my $size = -s $tmpfilename;
	die "temporary file '$tmpfilename' does not exist\n" if !defined($size);

	my $filename = PVE::Storage::normalize_content_filename($param->{filename});

	my $path;

	if ($content eq 'iso') {
	    if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
		raise_param_exc({ filename => "wrong file extension" });
	    }
	    $path = PVE::Storage::get_iso_dir($cfg, $param->{storage});
	} elsif ($content eq 'vztmpl') {
	    if ($filename !~ m![^/]+$PVE::Storage::vztmpl_extension_re$!) {
		raise_param_exc({ filename => "wrong file extension" });
	    }
	    $path = PVE::Storage::get_vztmpl_dir($cfg, $param->{storage});
	} else {
	    raise_param_exc({ content => "upload content type '$content' not allowed" });
	}

	die "storage '$param->{storage}' does not support '$content' content\n"
	    if !$scfg->{content}->{$content};

	my $dest = "$path/$filename";
	my $dirname = dirname($dest);

	# best effort to match apl_download behaviour
	chmod 0644, $tmpfilename;

	my $err_cleanup = sub { unlink $dest, $tmpfilename; die "cleanup failed: $!" if $! && $! != ENOENT };

	my $cmd;
	if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
	    my $remip = PVE::Cluster::remote_node_ip($node);

	    my @ssh_options = ('-o', 'BatchMode=yes');

	    my @remcmd = ('/usr/bin/ssh', @ssh_options, $remip, '--');

	    eval { # activate remote storage
		run_command([@remcmd, '/usr/sbin/pvesm', 'status', '--storage', $param->{storage}]);
	    };
	    die "can't activate storage '$param->{storage}' on node '$node': $@\n" if $@;

	    run_command(
		[@remcmd, '/bin/mkdir', '-p', '--', PVE::Tools::shell_quote($dirname)],
		errmsg => "mkdir failed",
	    );
 
	    $cmd = ['/usr/bin/scp', @ssh_options, '-p', '--', $tmpfilename, "[$remip]:" . PVE::Tools::shell_quote($dest)];

	    $err_cleanup = sub { run_command([@remcmd, 'rm', '-f', '--', $dest, $tmpfilename]) };
	} else {
	    PVE::Storage::activate_storage($cfg, $param->{storage});
	    File::Path::make_path($dirname);
	    $cmd = ['cp', '--', $tmpfilename, $dest];
	}

	# NOTE: we simply overwrite the destination file if it already exists
	my $worker = sub {
	    my $upid = shift;

	    print "starting file import from: $tmpfilename\n";
	    print "target node: $node\n";
	    print "target file: $dest\n";
	    print "file size is: $size\n";
	    print "command: " . join(' ', @$cmd) . "\n";

	    eval { run_command($cmd, errmsg => 'import failed'); };

	    unlink $tmpfilename; # the temporary file got only uploaded locally, no need to rm remote
	    warn "unable to clean up temporary file '$tmpfilename' - $!\n" if $! && $! != ENOENT;

	    if (my $err = $@) {
		eval { $err_cleanup->() };
		warn "$@" if $@;
		die $err;
	    }
	    print "finished file import successfully\n";
	};

	my $upid = $rpcenv->fork_worker('imgcopy', undef, $user, $worker);

	# apache removes the temporary file on return, so we need
	# to wait here to make sure the worker process starts and
	# opens the file before it gets removed.
	sleep(1);

	return $upid;
   }});

__PACKAGE__->register_method({
    name => 'download_url',
    path => '{storage}/download-url',
    method => 'POST',
    description => "Download templates and ISO images by using an URL.",
    proxyto => 'node',
    permissions => {
	check => [ 'and',
	    ['perm', '/storage/{storage}', [ 'Datastore.AllocateTemplate' ]],
	    ['perm', '/', [ 'Sys.Audit', 'Sys.Modify' ]],
	],
    },
    protected => 1,
    parameters => {
	additionalProperties => 0,
	properties => {
	    node => get_standard_option('pve-node'),
	    storage => get_standard_option('pve-storage-id'),
	    url => {
		description => "The URL to download the file from.",
		type => 'string',
		pattern => 'https?://.*',
	    },
	    content => {
		description => "Content type.", # TODO: could be optional & detected in most cases
		type => 'string', format => 'pve-storage-content',
		enum => ['iso', 'vztmpl'],
	    },
	    filename => {
		description => "The name of the file to create. Caution: This will be normalized!",
		maxLength => 255,
		type => 'string',
	    },
	    checksum => {
		description => "The expected checksum of the file.",
		type => 'string',
		requires => 'checksum-algorithm',
		optional => 1,
	    },
	    'checksum-algorithm' => {
		description => "The algorithm to calculate the checksum of the file.",
		type => 'string',
		enum => ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'],
		requires => 'checksum',
		optional => 1,
	    },
	    'verify-certificates' => {
		description => "If false, no SSL/TLS certificates will be verified.",
		type => 'boolean',
		optional => 1,
		default => 1,
	    },
	},
    },
    returns => {
	type => "string"
    },
    code => sub {
	my ($param) = @_;

	my $rpcenv = PVE::RPCEnvironment::get();
	my $user = $rpcenv->get_user();

	my $cfg = PVE::Storage::config();

	my ($node, $storage) = $param->@{'node', 'storage'};
	my $scfg = PVE::Storage::storage_check_enabled($cfg, $storage, $node);

	die "can't upload to storage type '$scfg->{type}', not a file based storage!\n"
	    if !defined($scfg->{path});

	my ($content, $url) = $param->@{'content', 'url'};

	die "storage '$storage' is not configured for content-type '$content'\n"
	    if !$scfg->{content}->{$content};

	my $filename = PVE::Storage::normalize_content_filename($param->{filename});

	my $path;
	if ($content eq 'iso') {
	    if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
		raise_param_exc({ filename => "wrong file extension" });
	    }
	    $path = PVE::Storage::get_iso_dir($cfg, $storage);
	} elsif ($content eq 'vztmpl') {
	    if ($filename !~ m![^/]+$PVE::Storage::vztmpl_extension_re$!) {
		raise_param_exc({ filename => "wrong file extension" });
	    }
	    $path = PVE::Storage::get_vztmpl_dir($cfg, $storage);
	} else {
	    raise_param_exc({ content => "upload content-type '$content' is not allowed" });
	}

	PVE::Storage::activate_storage($cfg, $storage);
	File::Path::make_path($path);

	my $dccfg = PVE::Cluster::cfs_read_file('datacenter.cfg');
	my $opts = {
	    hash_required => 0,
	    verify_certificates => $param->{'verify-certificates'} // 1,
	    http_proxy => $dccfg->{http_proxy},
	};

	my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm'};
	if ($checksum) {
	    $opts->{"${checksum_algorithm}sum"} = $checksum;
	    $opts->{hash_required} = 1;
	}

	my $worker = sub {
	    PVE::Tools::download_file_from_url("$path/$filename", $url, $opts);
	};

	my $worker_id = PVE::Tools::encode_text($filename); # must not pass : or the like as w-ID

	return $rpcenv->fork_worker('download', $worker_id, $user, $worker);
    }});

1;
Commit	Line	Data
b6cf0a66 DM	1	package PVE::API2::Storage::Status;
	2
	3	use strict;
	4	use warnings;
	5
7814e05f	6	use File::Basename;
0a65c237	7	use File::Path;
b11d054b	8	use POSIX qw(ENOENT);
0a65c237	9
83d7192f	10	use PVE::Cluster;
0a65c237 TL	11	use PVE::Exception qw(raise_param_exc);
	12	use PVE::INotify;
	13	use PVE::JSONSchema qw(get_standard_option);
	14	use PVE::RESTHandler;
	15	use PVE::RPCEnvironment;
0ce8cadd	16	use PVE::RRD;
a84804c4	17	use PVE::Tools qw(run_command);
0a65c237	18
b6cf0a66	19	use PVE::API2::Storage::Content;
f1a3ce3b	20	use PVE::API2::Storage::FileRestore;
0a65c237 TL	21	use PVE::API2::Storage::PruneBackups;
0a65c237 TL	22	use PVE::Storage;
b6cf0a66 DM	23
	24	use base qw(PVE::RESTHandler);
	25
25a95836 FE	26	__PACKAGE__->register_method ({
	27	subclass => "PVE::API2::Storage::PruneBackups",
	28	path => '{storage}/prunebackups',
	29	});
	30
b6cf0a66	31	__PACKAGE__->register_method ({
7dd31e68	32	subclass => "PVE::API2::Storage::Content",
b6cf0a66	33	# set fragment delimiter (no subdirs) - we need that, because volume
7dd31e68 FE	34	# IDs may contain a slash '/'
7dd31e68 FE	35	fragmentDelimiter => '',
b6cf0a66 DM	36	path => '{storage}/content',
	37	});
	38
f1a3ce3b SR	39	__PACKAGE__->register_method ({
	40	subclass => "PVE::API2::Storage::FileRestore",
	41	path => '{storage}/file-restore',
	42	});
	43
b6cf0a66	44	__PACKAGE__->register_method ({
7dd31e68	45	name => 'index',
b6cf0a66 DM	46	path => '',
	47	method => 'GET',
	48	description => "Get status for all datastores.",
7dd31e68	49	permissions => {
5f642f73 DM	50	description => "Only list entries where you have 'Datastore.Audit' or 'Datastore.AllocateSpace' permissions on '/storage/<storage>'",
	51	user => 'all',
	52	},
b6cf0a66 DM	53	protected => 1,
	54	proxyto => 'node',
	55	parameters => {
	56	additionalProperties => 0,
	57	properties => {
	58	node => get_standard_option('pve-node'),
f3bd890d DM	59	storage => get_standard_option('pve-storage-id', {
	60	description => "Only list status for specified storage",
	61	optional => 1,
	62	completion => \&PVE::Storage::complete_storage_enabled,
	63	}),
7dd31e68	64	content => {
b6cf0a66	65	description => "Only list stores which support this content type.",
583c2802	66	type => 'string', format => 'pve-storage-content-list',
b6cf0a66	67	optional => 1,
98437f4c	68	completion => \&PVE::Storage::complete_content_type,
b6cf0a66	69	},
283608f3 DM	70	enabled => {
	71	description => "Only list stores which are enabled (not disabled in config).",
	72	type => 'boolean',
12c2fe32 DM	73	optional => 1,
12c2fe32 DM	74	default => 0,
283608f3 DM	75	},
	76	target => get_standard_option('pve-node', {
	77	description => "If target is different to 'node', we only lists shared storages which " .
	78	"content is accessible on this 'node' and the specified 'target' node.",
	79	optional => 1,
f3bd890d	80	completion => \&PVE::Cluster::get_nodelist,
283608f3	81	}),
d347322a	82	'format' => {
856c54bd DC	83	description => "Include information about formats",
	84	type => 'boolean',
	85	optional => 1,
	86	default => 0,
	87	},
b6cf0a66 DM	88	},
	89	},
	90	returns => {
	91	type => 'array',
	92	items => {
	93	type => "object",
d347322a DM	94	properties => {
	95	storage => get_standard_option('pve-storage-id'),
	96	type => {
	97	description => "Storage type.",
	98	type => 'string',
	99	},
	100	content => {
	101	description => "Allowed storage content types.",
	102	type => 'string', format => 'pve-storage-content-list',
	103	},
	104	enabled => {
	105	description => "Set when storage is enabled (not disabled).",
	106	type => 'boolean',
	107	optional => 1,
	108	},
	109	active => {
	110	description => "Set when storage is accessible.",
	111	type => 'boolean',
	112	optional => 1,
	113	},
	114	shared => {
	115	description => "Shared flag from storage configuration.",
	116	type => 'boolean',
	117	optional => 1,
	118	},
	119	total => {
	120	description => "Total storage space in bytes.",
	121	type => 'integer',
	122	renderer => 'bytes',
	123	optional => 1,
	124	},
	125	used => {
	126	description => "Used storage space in bytes.",
	127	type => 'integer',
	128	renderer => 'bytes',
	129	optional => 1,
	130	},
	131	avail => {
	132	description => "Available storage space in bytes.",
	133	type => 'integer',
	134	renderer => 'bytes',
	135	optional => 1,
	136	},
	137	used_fraction => {
	138	description => "Used fraction (used/total).",
	139	type => 'number',
	140	renderer => 'fraction_as_percentage',
	141	optional => 1,
	142	},
	143	},
b6cf0a66 DM	144	},
	145	links => [ { rel => 'child', href => "{storage}" } ],
	146	},
	147	code => sub {
	148	my ($param) = @_;
	149
5f642f73 DM	150	my $rpcenv = PVE::RPCEnvironment::get();
	151	my $authuser = $rpcenv->get_user();
	152
283608f3 DM	153	my $localnode = PVE::INotify::nodename();
	154
	155	my $target = $param->{target};
	156
	157	undef $target if $target && ($target eq $localnode \|\| $target eq 'localhost');
f6c03d36	158
83d7192f	159	my $cfg = PVE::Storage::config();
b6cf0a66	160
856c54bd	161	my $info = PVE::Storage::storage_info($cfg, $param->{content}, $param->{format});
b6cf0a66	162
5f642f73 DM	163	raise_param_exc({ storage => "No such storage." })
5f642f73 DM	164	if $param->{storage} && !defined($info->{$param->{storage}});
f6c03d36	165
5f642f73 DM	166	my $res = {};
	167	my @sids = PVE::Storage::storage_ids($cfg);
	168	foreach my $storeid (@sids) {
d347322a DM	169	my $data = $info->{$storeid};
d347322a DM	170	next if !$data;
5f642f73 DM	171	my $privs = [ 'Datastore.Audit', 'Datastore.AllocateSpace' ];
	172	next if !$rpcenv->check_any($authuser, "/storage/$storeid", $privs, 1);
	173	next if $param->{storage} && $param->{storage} ne $storeid;
283608f3 DM	174
	175	my $scfg = PVE::Storage::storage_config($cfg, $storeid);
	176
	177	next if $param->{enabled} && $scfg->{disable};
f6c03d36	178
283608f3 DM	179	if ($target) {
	180	# check if storage content is accessible on local node and specified target node
	181	# we use this on the Clone GUI
	182
	183	next if !$scfg->{shared};
	184	next if !PVE::Storage::storage_check_node($cfg, $storeid, undef, 1);
	185	next if !PVE::Storage::storage_check_node($cfg, $storeid, $target, 1);
	186	}
	187
d347322a DM	188	if ($data->{total}) {
	189	$data->{used_fraction} = ($data->{used} // 0) / $data->{total};
	190	}
	191
	192	$res->{$storeid} = $data;
b6cf0a66	193	}
5f642f73 DM	194
5f642f73 DM	195	return PVE::RESTHandler::hash_to_array($res, 'storage');
b6cf0a66 DM	196	}});
	197
	198	__PACKAGE__->register_method ({
	199	name => 'diridx',
7dd31e68	200	path => '{storage}',
b6cf0a66 DM	201	method => 'GET',
b6cf0a66 DM	202	description => "",
7dd31e68	203	permissions => {
5f642f73 DM	204	check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
5f642f73 DM	205	},
b6cf0a66 DM	206	parameters => {
	207	additionalProperties => 0,
	208	properties => {
	209	node => get_standard_option('pve-node'),
	210	storage => get_standard_option('pve-storage-id'),
	211	},
	212	},
	213	returns => {
	214	type => 'array',
	215	items => {
	216	type => "object",
	217	properties => {
	218	subdir => { type => 'string' },
	219	},
	220	},
	221	links => [ { rel => 'child', href => "{subdir}" } ],
	222	},
	223	code => sub {
	224	my ($param) = @_;
	225
	226	my $res = [
b6cf0a66	227	{ subdir => 'content' },
837b1942	228	{ subdir => 'download-url' },
ead6be93 TL	229	{ subdir => 'file-restore' },
ead6be93 TL	230	{ subdir => 'prunebackups' },
b6cf0a66 DM	231	{ subdir => 'rrd' },
b6cf0a66 DM	232	{ subdir => 'rrddata' },
ead6be93 TL	233	{ subdir => 'status' },
ead6be93 TL	234	{ subdir => 'upload' },
7dd31e68 FE	235	];
7dd31e68 FE	236
b6cf0a66 DM	237	return $res;
	238	}});
	239
	240	__PACKAGE__->register_method ({
	241	name => 'read_status',
7dd31e68	242	path => '{storage}/status',
b6cf0a66 DM	243	method => 'GET',
b6cf0a66 DM	244	description => "Read storage status.",
7dd31e68	245	permissions => {
5f642f73 DM	246	check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
5f642f73 DM	247	},
b6cf0a66 DM	248	protected => 1,
	249	proxyto => 'node',
	250	parameters => {
	251	additionalProperties => 0,
	252	properties => {
	253	node => get_standard_option('pve-node'),
	254	storage => get_standard_option('pve-storage-id'),
	255	},
	256	},
	257	returns => {
	258	type => "object",
	259	properties => {},
	260	},
	261	code => sub {
	262	my ($param) = @_;
	263
83d7192f	264	my $cfg = PVE::Storage::config();
b6cf0a66 DM	265
	266	my $info = PVE::Storage::storage_info($cfg, $param->{content});
	267
	268	my $data = $info->{$param->{storage}};
	269
	270	raise_param_exc({ storage => "No such storage." })
	271	if !defined($data);
7dd31e68	272
b6cf0a66 DM	273	return $data;
	274	}});
	275
	276	__PACKAGE__->register_method ({
	277	name => 'rrd',
7dd31e68	278	path => '{storage}/rrd',
b6cf0a66 DM	279	method => 'GET',
b6cf0a66 DM	280	description => "Read storage RRD statistics (returns PNG).",
7dd31e68	281	permissions => {
5f642f73 DM	282	check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
5f642f73 DM	283	},
b6cf0a66 DM	284	protected => 1,
	285	proxyto => 'node',
	286	parameters => {
	287	additionalProperties => 0,
	288	properties => {
	289	node => get_standard_option('pve-node'),
	290	storage => get_standard_option('pve-storage-id'),
	291	timeframe => {
	292	description => "Specify the time frame you are interested in.",
	293	type => 'string',
	294	enum => [ 'hour', 'day', 'week', 'month', 'year' ],
	295	},
	296	ds => {
	297	description => "The list of datasources you want to display.",
	298	type => 'string', format => 'pve-configid-list',
	299	},
	300	cf => {
	301	description => "The RRD consolidation function",
	302	type => 'string',
	303	enum => [ 'AVERAGE', 'MAX' ],
	304	optional => 1,
	305	},
	306	},
	307	},
	308	returns => {
	309	type => "object",
	310	properties => {
	311	filename => { type => 'string' },
	312	},
	313	},
	314	code => sub {
	315	my ($param) = @_;
	316
0ce8cadd	317	return PVE::RRD::create_rrd_graph(
7dd31e68	318	"pve2-storage/$param->{node}/$param->{storage}",
b6cf0a66	319	$param->{timeframe}, $param->{ds}, $param->{cf});
b6cf0a66 DM	320	}});
	321
	322	__PACKAGE__->register_method ({
	323	name => 'rrddata',
7dd31e68	324	path => '{storage}/rrddata',
b6cf0a66 DM	325	method => 'GET',
b6cf0a66 DM	326	description => "Read storage RRD statistics.",
7dd31e68	327	permissions => {
5f642f73 DM	328	check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1],
5f642f73 DM	329	},
b6cf0a66 DM	330	protected => 1,
	331	proxyto => 'node',
	332	parameters => {
	333	additionalProperties => 0,
	334	properties => {
	335	node => get_standard_option('pve-node'),
	336	storage => get_standard_option('pve-storage-id'),
	337	timeframe => {
	338	description => "Specify the time frame you are interested in.",
	339	type => 'string',
	340	enum => [ 'hour', 'day', 'week', 'month', 'year' ],
	341	},
	342	cf => {
	343	description => "The RRD consolidation function",
	344	type => 'string',
	345	enum => [ 'AVERAGE', 'MAX' ],
	346	optional => 1,
	347	},
	348	},
	349	},
	350	returns => {
	351	type => "array",
	352	items => {
	353	type => "object",
	354	properties => {},
	355	},
	356	},
	357	code => sub {
	358	my ($param) = @_;
	359
0ce8cadd	360	return PVE::RRD::create_rrd_data(
7dd31e68 FE	361	"pve2-storage/$param->{node}/$param->{storage}",
7dd31e68 FE	362	$param->{timeframe}, $param->{cf});
b6cf0a66	363	}});
7814e05f	364
7dd31e68	365	# makes no sense for big images and backup files (because it
1f6610f3	366	# create a copy of the file).
7814e05f DM	367	__PACKAGE__->register_method ({
7814e05f DM	368	name => 'upload',
7dd31e68	369	path => '{storage}/upload',
7814e05f	370	method => 'POST',
1f6610f3	371	description => "Upload templates and ISO images.",
7dd31e68	372	permissions => {
1f6610f3	373	check => ['perm', '/storage/{storage}', ['Datastore.AllocateTemplate']],
5f642f73	374	},
7814e05f DM	375	protected => 1,
	376	parameters => {
	377	additionalProperties => 0,
	378	properties => {
	379	node => get_standard_option('pve-node'),
	380	storage => get_standard_option('pve-storage-id'),
7dd31e68	381	content => {
7814e05f DM	382	description => "Content type.",
7814e05f DM	383	type => 'string', format => 'pve-storage-content',
1e96ffc9	384	enum => ['iso', 'vztmpl'],
7814e05f	385	},
7dd31e68	386	filename => {
ca8c8658 LS	387	description => "The name of the file to create. Caution: This will be normalized!",
ca8c8658 LS	388	maxLength => 255,
7814e05f DM	389	type => 'string',
7814e05f DM	390	},
7dd31e68	391	tmpfilename => {
5c0720d6	392	description => "The source file name. This parameter is usually set by the REST handler. You can only overwrite it when connecting to the trusted port on localhost.",
7814e05f DM	393	type => 'string',
	394	optional => 1,
	395	},
	396	},
	397	},
	398	returns => { type => "string" },
	399	code => sub {
	400	my ($param) = @_;
	401
	402	my $rpcenv = PVE::RPCEnvironment::get();
	403
	404	my $user = $rpcenv->get_user();
	405
83d7192f	406	my $cfg = PVE::Storage::config();
7814e05f DM	407
	408	my $node = $param->{node};
	409	my $scfg = PVE::Storage::storage_check_enabled($cfg, $param->{storage}, $node);
	410
d68c7bca	411	die "can't upload to storage type '$scfg->{type}'\n"
ee8c176d	412	if !defined($scfg->{path});
7814e05f DM	413
	414	my $content = $param->{content};
	415
	416	my $tmpfilename = $param->{tmpfilename};
	417	die "missing temporary file name\n" if !$tmpfilename;
	418
	419	my $size = -s $tmpfilename;
481f6177	420	die "temporary file '$tmpfilename' does not exist\n" if !defined($size);
7814e05f	421
edda43ed	422	my $filename = PVE::Storage::normalize_content_filename($param->{filename});
7814e05f DM	423
	424	my $path;
	425
	426	if ($content eq 'iso') {
4c693491	427	if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
bba10cf4	428	raise_param_exc({ filename => "wrong file extension" });
7814e05f DM	429	}
	430	$path = PVE::Storage::get_iso_dir($cfg, $param->{storage});
	431	} elsif ($content eq 'vztmpl') {
bba10cf4 LS	432	if ($filename !~ m![^/]+$PVE::Storage::vztmpl_extension_re$!) {
bba10cf4 LS	433	raise_param_exc({ filename => "wrong file extension" });
7814e05f	434	}
4ea5bca4	435	$path = PVE::Storage::get_vztmpl_dir($cfg, $param->{storage});
7814e05f	436	} else {
1f6610f3	437	raise_param_exc({ content => "upload content type '$content' not allowed" });
7814e05f DM	438	}
7814e05f DM	439
d68c7bca	440	die "storage '$param->{storage}' does not support '$content' content\n"
7814e05f DM	441	if !$scfg->{content}->{$content};
	442
	443	my $dest = "$path/$filename";
	444	my $dirname = dirname($dest);
	445
5c0720d6 SR	446	# best effort to match apl_download behaviour
	447	chmod 0644, $tmpfilename;
	448
b11d054b	449	my $err_cleanup = sub { unlink $dest, $tmpfilename; die "cleanup failed: $!" if $! && $! != ENOENT };
7814e05f DM	450
	451	my $cmd;
	452	if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
	453	my $remip = PVE::Cluster::remote_node_ip($node);
	454
45c2ee35	455	my @ssh_options = ('-o', 'BatchMode=yes');
7814e05f	456
53ec90e2	457	my @remcmd = ('/usr/bin/ssh', @ssh_options, $remip, '--');
7814e05f	458
a84804c4 TL	459	eval { # activate remote storage
a84804c4 TL	460	run_command([@remcmd, '/usr/sbin/pvesm', 'status', '--storage', $param->{storage}]);
7814e05f	461	};
4af77132	462	die "can't activate storage '$param->{storage}' on node '$node': $@\n" if $@;
7814e05f	463
a84804c4 TL	464	run_command(
	465	[@remcmd, '/bin/mkdir', '-p', '--', PVE::Tools::shell_quote($dirname)],
	466	errmsg => "mkdir failed",
	467	);
7814e05f	468
5c0720d6	469	$cmd = ['/usr/bin/scp', @ssh_options, '-p', '--', $tmpfilename, "[$remip]:" . PVE::Tools::shell_quote($dest)];
b11d054b TL	470
b11d054b TL	471	$err_cleanup = sub { run_command([@remcmd, 'rm', '-f', '--', $dest, $tmpfilename]) };
7814e05f DM	472	} else {
7814e05f DM	473	PVE::Storage::activate_storage($cfg, $param->{storage});
4ea5bca4	474	File::Path::make_path($dirname);
53ec90e2	475	$cmd = ['cp', '--', $tmpfilename, $dest];
7814e05f DM	476	}
7814e05f DM	477
b11d054b	478	# NOTE: we simply overwrite the destination file if it already exists
7dd31e68	479	my $worker = sub {
7814e05f	480	my $upid = shift;
7dd31e68	481
7814e05f DM	482	print "starting file import from: $tmpfilename\n";
	483	print "target node: $node\n";
	484	print "target file: $dest\n";
	485	print "file size is: $size\n";
	486	print "command: " . join(' ', @$cmd) . "\n";
	487
a84804c4	488	eval { run_command($cmd, errmsg => 'import failed'); };
f6aeefff	489
3de1c50a TL	490	unlink $tmpfilename; # the temporary file got only uploaded locally, no need to rm remote
3de1c50a TL	491	warn "unable to clean up temporary file '$tmpfilename' - $!\n" if $! && $! != ENOENT;
f6aeefff	492
7814e05f	493	if (my $err = $@) {
b11d054b TL	494	eval { $err_cleanup->() };
b11d054b TL	495	warn "$@" if $@;
7814e05f DM	496	die $err;
	497	}
	498	print "finished file import successfully\n";
	499	};
	500
d6c9dc34 DM	501	my $upid = $rpcenv->fork_worker('imgcopy', undef, $user, $worker);
	502
	503	# apache removes the temporary file on return, so we need
	504	# to wait here to make sure the worker process starts and
	505	# opens the file before it gets removed.
	506	sleep(1);
	507
	508	return $upid;
7814e05f	509	}});
7dd31e68	510
837b1942 LS	511	__PACKAGE__->register_method({
	512	name => 'download_url',
	513	path => '{storage}/download-url',
	514	method => 'POST',
	515	description => "Download templates and ISO images by using an URL.",
	516	proxyto => 'node',
	517	permissions => {
	518	check => [ 'and',
	519	['perm', '/storage/{storage}', [ 'Datastore.AllocateTemplate' ]],
	520	['perm', '/', [ 'Sys.Audit', 'Sys.Modify' ]],
	521	],
	522	},
	523	protected => 1,
	524	parameters => {
	525	additionalProperties => 0,
	526	properties => {
	527	node => get_standard_option('pve-node'),
	528	storage => get_standard_option('pve-storage-id'),
	529	url => {
	530	description => "The URL to download the file from.",
	531	type => 'string',
	532	pattern => 'https?://.*',
	533	},
	534	content => {
d0a3db1b	535	description => "Content type.", # TODO: could be optional & detected in most cases
837b1942 LS	536	type => 'string', format => 'pve-storage-content',
	537	enum => ['iso', 'vztmpl'],
	538	},
	539	filename => {
	540	description => "The name of the file to create. Caution: This will be normalized!",
fc015f3d	541	maxLength => 255,
837b1942 LS	542	type => 'string',
	543	},
	544	checksum => {
	545	description => "The expected checksum of the file.",
	546	type => 'string',
	547	requires => 'checksum-algorithm',
	548	optional => 1,
	549	},
	550	'checksum-algorithm' => {
	551	description => "The algorithm to calculate the checksum of the file.",
	552	type => 'string',
	553	enum => ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'],
	554	requires => 'checksum',
	555	optional => 1,
	556	},
	557	'verify-certificates' => {
	558	description => "If false, no SSL/TLS certificates will be verified.",
	559	type => 'boolean',
	560	optional => 1,
	561	default => 1,
d0a3db1b	562	},
837b1942 LS	563	},
	564	},
	565	returns => {
	566	type => "string"
	567	},
	568	code => sub {
	569	my ($param) = @_;
	570
	571	my $rpcenv = PVE::RPCEnvironment::get();
	572	my $user = $rpcenv->get_user();
	573
	574	my $cfg = PVE::Storage::config();
	575
	576	my ($node, $storage) = $param->@{'node', 'storage'};
	577	my $scfg = PVE::Storage::storage_check_enabled($cfg, $storage, $node);
	578
	579	die "can't upload to storage type '$scfg->{type}', not a file based storage!\n"
	580	if !defined($scfg->{path});
	581
	582	my ($content, $url) = $param->@{'content', 'url'};
	583
	584	die "storage '$storage' is not configured for content-type '$content'\n"
	585	if !$scfg->{content}->{$content};
	586
	587	my $filename = PVE::Storage::normalize_content_filename($param->{filename});
	588
	589	my $path;
	590	if ($content eq 'iso') {
	591	if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
	592	raise_param_exc({ filename => "wrong file extension" });
	593	}
	594	$path = PVE::Storage::get_iso_dir($cfg, $storage);
	595	} elsif ($content eq 'vztmpl') {
	596	if ($filename !~ m![^/]+$PVE::Storage::vztmpl_extension_re$!) {
	597	raise_param_exc({ filename => "wrong file extension" });
	598	}
	599	$path = PVE::Storage::get_vztmpl_dir($cfg, $storage);
	600	} else {
	601	raise_param_exc({ content => "upload content-type '$content' is not allowed" });
	602	}
	603
	604	PVE::Storage::activate_storage($cfg, $storage);
	605	File::Path::make_path($path);
	606
	607	my $dccfg = PVE::Cluster::cfs_read_file('datacenter.cfg');
	608	my $opts = {
	609	hash_required => 0,
	610	verify_certificates => $param->{'verify-certificates'} // 1,
	611	http_proxy => $dccfg->{http_proxy},
	612	};
	613
	614	my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm'};
	615	if ($checksum) {
	616	$opts->{"${checksum_algorithm}sum"} = $checksum;
	617	$opts->{hash_required} = 1;
	618	}
	619
	620	my $worker = sub {
	621	PVE::Tools::download_file_from_url("$path/$filename", $url, $opts);
	622	};
	623
fc015f3d TL	624	my $worker_id = PVE::Tools::encode_text($filename); # must not pass : or the like as w-ID
	625
	626	return $rpcenv->fork_worker('download', $worker_id, $user, $worker);
837b1942 LS	627	}});
837b1942 LS	628
b6cf0a66	629	1;