use POSIX qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC);
use Fcntl ':flock';
use File::Path;
+use MIME::Base64 qw(encode_base64);
+
+use IO::Socket::IP;
+use IO::Socket::UNIX;
+use Socket qw(SOCK_STREAM);
use PVE::SafeSyslog;
use PVE::Cluster;
use PVE::INotify;
use PVE::RPCEnvironment;
use PVE::Storage;
+use PVE::Tools qw(extract_param);
use PVE::API2::Storage::Config;
use PVE::API2::Storage::Content;
+use PVE::API2::Storage::PruneBackups;
+use PVE::API2::Storage::Scan;
use PVE::API2::Storage::Status;
use PVE::JSONSchema qw(get_standard_option);
use PVE::PTY;
use base qw(PVE::CLIHandler);
-my $KNOWN_EXPORT_FORMATS = ['raw+size', 'tar+size', 'qcow2+size', 'vmdk+size', 'zfs'];
+my $KNOWN_EXPORT_FORMATS = ['raw+size', 'tar+size', 'qcow2+size', 'vmdk+size', 'zfs', 'btrfs'];
my $nodename = PVE::INotify::nodename();
}
};
+ my $master_key_map = {
+ name => 'master-pubkey',
+ desc => 'a file containing a PEM-formatted master public key',
+ func => sub {
+ my ($value) = @_;
+ return encode_base64(PVE::Tools::file_get_contents($value), '');
+ }
+ };
+
my $mapping = {
'cifsscan' => [ $password_map ],
- 'create' => [ $password_map, $enc_key_map ],
- 'update' => [ $password_map, $enc_key_map ],
+ 'cifs' => [ $password_map ],
+ 'pbs' => [ $password_map ],
+ 'create' => [ $password_map, $enc_key_map, $master_key_map ],
+ 'update' => [ $password_map, $enc_key_map, $master_key_map ],
};
return $mapping->{$name};
}
base => {
description => "Snapshot to start an incremental stream from",
type => 'string',
- pattern => qr/[a-z0-9_\-]{1,40}/,
+ pattern => qr/[a-z0-9_\-]{1,40}/i,
maxLength => 40,
optional => 1,
},
snapshot => {
description => "Snapshot to export",
type => 'string',
- pattern => qr/[a-z0-9_\-]{1,40}/,
+ pattern => qr/[a-z0-9_\-]{1,40}/i,
maxLength => 40,
optional => 1,
},
optional => 1,
default => 0,
},
+ 'snapshot-list' => {
+ description => "Ordered list of snapshots to transfer",
+ type => 'string',
+ format => 'string-list',
+ optional => 1,
+ },
},
},
returns => { type => 'null' },
code => sub {
my ($param) = @_;
+ my $with_snapshots = $param->{'with-snapshots'};
+ if (defined(my $list = $param->{'snapshot-list'})) {
+ $with_snapshots = PVE::Tools::split_list($list);
+ }
+
my $filename = $param->{filename};
my $outfh;
eval {
my $cfg = PVE::Storage::config();
PVE::Storage::volume_export($cfg, $outfh, $param->{volume}, $param->{format},
- $param->{snapshot}, $param->{base}, $param->{'with-snapshots'});
+ $param->{snapshot}, $param->{base}, $with_snapshots);
};
my $err = $@;
if ($filename ne '-') {
},
filename => {
description => "Source file name. For '-' stdin is used, the " .
- "tcp://<IP-or-CIDR> format allows to use a TCP connection as input. " .
+ "tcp://<IP-or-CIDR> format allows to use a TCP connection, " .
+ "the unix://PATH-TO-SOCKET format a UNIX socket as input." .
"Else, the file is treated as common file.",
type => 'string',
},
base => {
description => "Base snapshot of an incremental stream",
type => 'string',
- pattern => qr/[a-z0-9_\-]{1,40}/,
+ pattern => qr/[a-z0-9_\-]{1,40}/i,
maxLength => 40,
optional => 1,
},
'delete-snapshot' => {
description => "A snapshot to delete on success",
type => 'string',
- pattern => qr/[a-z0-9_\-]{1,80}/,
+ pattern => qr/[a-z0-9_\-]{1,80}/i,
maxLength => 80,
optional => 1,
},
optional => 1,
default => 0,
},
+ snapshot => {
+ description => "The current-state snapshot if the stream contains snapshots",
+ type => 'string',
+ pattern => qr/[a-z0-9_\-]{1,40}/i,
+ maxLength => 40,
+ optional => 1,
+ },
},
},
returns => { type => 'string' },
alarm $prev_alarm;
close($socket);
+ $infh = \*$client;
+ } elsif ($filename =~ m!^unix://(.*)$!) {
+ my $socket_path = $1;
+ my $socket = IO::Socket::UNIX->new(
+ Type => SOCK_STREAM(),
+ Local => $socket_path,
+ Listen => 1,
+ ) or die "failed to open socket: $!\n";
+
+ print "ready\n";
+ *STDOUT->flush();
+
+ my $prev_alarm = alarm 0;
+ local $SIG{ALRM} = sub { die "timed out waiting for client\n" };
+ alarm 30;
+ my $client = $socket->accept; # Wait for a client
+ alarm $prev_alarm;
+ close($socket);
+
$infh = \*$client;
} else {
sysopen($infh, $filename, O_RDONLY)
my $volume = $param->{volume};
my $delete = $param->{'delete-snapshot'};
my $imported_volid = PVE::Storage::volume_import($cfg, $infh, $volume, $param->{format},
- $param->{base}, $param->{'with-snapshots'}, $param->{'allow-rename'});
+ $param->{snapshot}, $param->{base}, $param->{'with-snapshots'},
+ $param->{'allow-rename'});
PVE::Storage::volume_snapshot_delete($cfg, $imported_volid, $delete)
if defined($delete);
return $imported_volid;
});
__PACKAGE__->register_method ({
- name => 'nfsscan',
- path => 'nfs',
+ name => 'prunebackups',
+ path => 'prunebackups',
method => 'GET',
- description => "Scan remote NFS server.",
+ description => "Prune backups. Only those using the standard naming scheme are considered. " .
+ "If no keep options are specified, those from the storage configuration are used.",
protected => 1,
- proxyto => "node",
- permissions => {
- check => ['perm', '/storage', ['Datastore.Allocate']],
- },
+ proxyto => 'node',
parameters => {
additionalProperties => 0,
properties => {
- node => get_standard_option('pve-node'),
- server => {
- description => "The server address (name or IP).",
- type => 'string', format => 'pve-storage-server',
- },
- },
- },
- returns => {
- type => 'array',
- items => {
- type => "object",
- properties => {
- path => {
- description => "The exported path.",
- type => 'string',
- },
- options => {
- description => "NFS export options.",
- type => 'string',
- },
- },
- },
- },
- code => sub {
- my ($param) = @_;
-
- my $server = $param->{server};
- my $res = PVE::Storage::scan_nfs($server);
-
- my $data = [];
- foreach my $k (sort keys %$res) {
- push @$data, { path => $k, options => $res->{$k} };
- }
- return $data;
- }});
-
-__PACKAGE__->register_method ({
- name => 'cifsscan',
- path => 'cifs',
- method => 'GET',
- description => "Scan remote CIFS server.",
- protected => 1,
- proxyto => "node",
- permissions => {
- check => ['perm', '/storage', ['Datastore.Allocate']],
- },
- parameters => {
- additionalProperties => 0,
- properties => {
- node => get_standard_option('pve-node'),
- server => {
- description => "The server address (name or IP).",
- type => 'string', format => 'pve-storage-server',
- },
- username => {
- description => "User name.",
- type => 'string',
+ 'dry-run' => {
+ description => "Only show what would be pruned, don't delete anything.",
+ type => 'boolean',
optional => 1,
},
- password => {
- description => "User password.",
+ node => get_standard_option('pve-node'),
+ storage => get_standard_option('pve-storage-id', {
+ completion => \&PVE::Storage::complete_storage_enabled,
+ }),
+ %{$PVE::Storage::Plugin::prune_backups_format},
+ type => {
+ description => "Either 'qemu' or 'lxc'. Only consider backups for guests of this type.",
type => 'string',
optional => 1,
+ enum => ['qemu', 'lxc'],
},
- domain => {
- description => "SMB domain (Workgroup).",
- type => 'string',
+ vmid => get_standard_option('pve-vmid', {
+ description => "Only consider backups for this guest.",
optional => 1,
- },
+ completion => \&PVE::Cluster::complete_vmid,
+ }),
},
},
returns => {
- type => 'array',
- items => {
- type => "object",
- properties => {
- share => {
- description => "The cifs share name.",
- type => 'string',
- },
- description => {
- description => "Descriptive text from server.",
- type => 'string',
- },
- },
- },
- },
- code => sub {
- my ($param) = @_;
-
- my $server = $param->{server};
-
- my $username = $param->{username};
- my $password = $param->{password};
- my $domain = $param->{domain};
-
- my $res = PVE::Storage::scan_cifs($server, $username, $password, $domain);
-
- my $data = [];
- foreach my $k (sort keys %$res) {
- push @$data, { share => $k, description => $res->{$k} };
- }
-
- return $data;
- }});
-
-# Note: GlusterFS currently does not have an equivalent of showmount.
-# As workaround, we simply use nfs showmount.
-# see http://www.gluster.org/category/volumes/
-
-__PACKAGE__->register_method ({
- name => 'glusterfsscan',
- path => 'glusterfs',
- method => 'GET',
- description => "Scan remote GlusterFS server.",
- protected => 1,
- proxyto => "node",
- permissions => {
- check => ['perm', '/storage', ['Datastore.Allocate']],
- },
- parameters => {
- additionalProperties => 0,
+ type => 'object',
properties => {
- node => get_standard_option('pve-node'),
- server => {
- description => "The server address (name or IP).",
- type => 'string', format => 'pve-storage-server',
+ dryrun => {
+ description => 'If it was a dry run or not. The list will only be defined in that case.',
+ type => 'boolean',
},
- },
- },
- returns => {
- type => 'array',
- items => {
- type => "object",
- properties => {
- volname => {
- description => "The volume name.",
- type => 'string',
+ list => {
+ type => 'array',
+ items => {
+ type => 'object',
+ properties => {
+ volid => {
+ description => "Backup volume ID.",
+ type => 'string',
+ },
+ 'ctime' => {
+ description => "Creation time of the backup (seconds since the UNIX epoch).",
+ type => 'integer',
+ },
+ 'mark' => {
+ description => "Whether the backup would be kept or removed. For backups that don't " .
+ "use the standard naming scheme, it's 'protected'.",
+ type => 'string',
+ },
+ type => {
+ description => "One of 'qemu', 'lxc', 'openvz' or 'unknown'.",
+ type => 'string',
+ },
+ 'vmid' => {
+ description => "The VM the backup belongs to.",
+ type => 'integer',
+ optional => 1,
+ },
+ },
},
},
},
code => sub {
my ($param) = @_;
- my $server = $param->{server};
- my $res = PVE::Storage::scan_nfs($server);
+ my $dryrun = extract_param($param, 'dry-run') ? 1 : 0;
- my $data = [];
- foreach my $path (sort keys %$res) {
- if ($path =~ m!^/([^\s/]+)$!) {
- push @$data, { volname => $1 };
- }
+ my $keep_opts;
+ foreach my $keep (keys %{$PVE::Storage::Plugin::prune_backups_format}) {
+ $keep_opts->{$keep} = extract_param($param, $keep) if defined($param->{$keep});
}
- return $data;
- }});
-
-__PACKAGE__->register_method ({
- name => 'iscsiscan',
- path => 'iscsi',
- method => 'GET',
- description => "Scan remote iSCSI server.",
- protected => 1,
- proxyto => "node",
- permissions => {
- check => ['perm', '/storage', ['Datastore.Allocate']],
- },
- parameters => {
- additionalProperties => 0,
- properties => {
- node => get_standard_option('pve-node'),
- portal => {
- description => "The iSCSI portal (IP or DNS name with optional port).",
- type => 'string', format => 'pve-storage-portal-dns',
- },
- },
- },
- returns => {
- type => 'array',
- items => {
- type => "object",
- properties => {
- target => {
- description => "The iSCSI target name.",
- type => 'string',
- },
- portal => {
- description => "The iSCSI portal name.",
- type => 'string',
- },
- },
- },
- },
- code => sub {
- my ($param) = @_;
-
- my $res = PVE::Storage::scan_iscsi($param->{portal});
+ $param->{'prune-backups'} = PVE::JSONSchema::print_property_string(
+ $keep_opts, $PVE::Storage::Plugin::prune_backups_format) if $keep_opts;
- my $data = [];
- foreach my $k (sort keys %$res) {
- push @$data, { target => $k, portal => join(',', @{$res->{$k}}) };
+ my $list = [];
+ if ($dryrun) {
+ $list = PVE::API2::Storage::PruneBackups->dryrun($param);
+ } else {
+ PVE::API2::Storage::PruneBackups->delete($param);
}
- return $data;
- }});
-
-__PACKAGE__->register_method ({
- name => 'lvmscan',
- path => 'lvm',
- method => 'GET',
- description => "List local LVM volume groups.",
- protected => 1,
- proxyto => "node",
- permissions => {
- check => ['perm', '/storage', ['Datastore.Allocate']],
- },
- parameters => {
- additionalProperties => 0,
- properties => {
- node => get_standard_option('pve-node'),
- },
- },
- returns => {
- type => 'array',
- items => {
- type => "object",
- properties => {
- vg => {
- description => "The LVM logical volume group name.",
- type => 'string',
- },
- },
- },
- },
- code => sub {
- my ($param) = @_;
-
- my $res = PVE::Storage::LVMPlugin::lvm_vgs();
- return PVE::RESTHandler::hash_to_array($res, 'vg');
- }});
-
-__PACKAGE__->register_method ({
- name => 'lvmthinscan',
- path => 'lvmthin',
- method => 'GET',
- description => "List local LVM Thin Pools.",
- protected => 1,
- proxyto => "node",
- permissions => {
- check => ['perm', '/storage', ['Datastore.Allocate']],
- },
- parameters => {
- additionalProperties => 0,
- properties => {
- node => get_standard_option('pve-node'),
- vg => {
- type => 'string',
- pattern => '[a-zA-Z0-9\.\+\_][a-zA-Z0-9\.\+\_\-]+', # see lvm(8) manpage
- maxLength => 100,
- },
- },
- },
- returns => {
- type => 'array',
- items => {
- type => "object",
- properties => {
- lv => {
- description => "The LVM Thin Pool name (LVM logical volume).",
- type => 'string',
- },
- },
- },
- },
- code => sub {
- my ($param) = @_;
-
- return PVE::Storage::LvmThinPlugin::list_thinpools($param->{vg});
+ return {
+ dryrun => $dryrun,
+ list => $list,
+ };
}});
-__PACKAGE__->register_method ({
- name => 'zfsscan',
- path => 'zfs',
- method => 'GET',
- description => "Scan zfs pool list on local node.",
- protected => 1,
- proxyto => "node",
- permissions => {
- check => ['perm', '/storage', ['Datastore.Allocate']],
- },
- parameters => {
- additionalProperties => 0,
- properties => {
- node => get_standard_option('pve-node'),
- },
- },
- returns => {
- type => 'array',
- items => {
- type => "object",
- properties => {
- pool => {
- description => "ZFS pool name.",
- type => 'string',
- },
- },
- },
- },
- code => sub {
- my ($param) = @_;
-
- return PVE::Storage::scan_zfs();
- }});
+my $print_api_result = sub {
+ my ($data, $schema, $options) = @_;
+ PVE::CLIFormatter::print_api_result($data, $schema, undef, $options);
+};
our $cmddef = {
add => [ "PVE::API2::Storage::Config", 'create', ['type', 'storage'] ],
free => [ "PVE::API2::Storage::Content", 'delete', ['volume'],
{ node => $nodename } ],
scan => {
- nfs => [ __PACKAGE__, 'nfsscan', ['server'], { node => $nodename }, sub {
+ nfs => [ "PVE::API2::Storage::Scan", 'nfsscan', ['server'], { node => $nodename }, sub {
my $res = shift;
my $maxlen = 0;
printf "%-${maxlen}s %s\n", $rec->{path}, $rec->{options};
}
}],
- cifs => [ __PACKAGE__, 'cifsscan', ['server'], { node => $nodename }, sub {
+ cifs => [ "PVE::API2::Storage::Scan", 'cifsscan', ['server'], { node => $nodename }, sub {
my $res = shift;
my $maxlen = 0;
printf "%-${maxlen}s %s\n", $rec->{share}, $rec->{description};
}
}],
- glusterfs => [ __PACKAGE__, 'glusterfsscan', ['server'], { node => $nodename }, sub {
+ glusterfs => [ "PVE::API2::Storage::Scan", 'glusterfsscan', ['server'], { node => $nodename }, sub {
my $res = shift;
foreach my $rec (@$res) {
printf "%s\n", $rec->{volname};
}
}],
- iscsi => [ __PACKAGE__, 'iscsiscan', ['portal'], { node => $nodename }, sub {
+ iscsi => [ "PVE::API2::Storage::Scan", 'iscsiscan', ['portal'], { node => $nodename }, sub {
my $res = shift;
my $maxlen = 0;
printf "%-${maxlen}s %s\n", $rec->{target}, $rec->{portal};
}
}],
- lvm => [ __PACKAGE__, 'lvmscan', [], { node => $nodename }, sub {
+ lvm => [ "PVE::API2::Storage::Scan", 'lvmscan', [], { node => $nodename }, sub {
my $res = shift;
foreach my $rec (@$res) {
printf "$rec->{vg}\n";
}
}],
- lvmthin => [ __PACKAGE__, 'lvmthinscan', ['vg'], { node => $nodename }, sub {
+ lvmthin => [ "PVE::API2::Storage::Scan", 'lvmthinscan', ['vg'], { node => $nodename }, sub {
my $res = shift;
foreach my $rec (@$res) {
printf "$rec->{lv}\n";
}
}],
- zfs => [ __PACKAGE__, 'zfsscan', [], { node => $nodename }, sub {
+ pbs => [
+ "PVE::API2::Storage::Scan",
+ 'pbsscan',
+ ['server', 'username'],
+ { node => $nodename },
+ $print_api_result,
+ $PVE::RESTHandler::standard_output_options,
+ ],
+ zfs => [ "PVE::API2::Storage::Scan", 'zfsscan', [], { node => $nodename }, sub {
my $res = shift;
foreach my $rec (@$res) {
print "APIVER $res->{apiver}\n";
print "APIAGE $res->{apiage}\n";
}],
+ 'prune-backups' => [ __PACKAGE__, 'prunebackups', ['storage'], { node => $nodename }, sub {
+ my $res = shift;
+
+ my ($dryrun, $list) = ($res->{dryrun}, $res->{list});
+
+ return if !$dryrun;
+
+ if (!scalar(@{$list})) {
+ print "No backups found\n";
+ return;
+ }
+
+ print "NOTE: this is only a preview and might not be what a subsequent\n" .
+ "prune call does if backups are removed/added in the meantime.\n\n";
+
+ my @sorted = sort {
+ my $vmcmp = PVE::Tools::safe_compare($a->{vmid}, $b->{vmid}, sub { $_[0] <=> $_[1] });
+ return $vmcmp if $vmcmp ne 0;
+ return $a->{ctime} <=> $b->{ctime};
+ } @{$list};
+
+ my $maxlen = 0;
+ foreach my $backup (@sorted) {
+ my $volid = $backup->{volid};
+ $maxlen = length($volid) if length($volid) > $maxlen;
+ }
+ $maxlen+=1;
+
+ printf("%-${maxlen}s %15s %10s\n", 'Backup', 'Backup-ID', 'Prune-Mark');
+ foreach my $backup (@sorted) {
+ my $type = $backup->{type};
+ my $vmid = $backup->{vmid};
+ my $backup_id = defined($vmid) ? "$type/$vmid" : "$type";
+ printf("%-${maxlen}s %15s %10s\n", $backup->{volid}, $backup_id, $backup->{mark});
+ }
+ }],
};
1;
projects / pve-storage.git / blobdiff