]> git.proxmox.com Git - pve-storage.git/blob - PVE/CLI/pvesm.pm
1daed71a1978c528e8c642e81a5608e53947b157
[pve-storage.git] / PVE / CLI / pvesm.pm
1 package PVE::CLI::pvesm;
2
3 use strict;
4 use warnings;
5
6 use POSIX qw(O_RDONLY O_WRONLY O_CREAT O_TRUNC);
7 use Fcntl ':flock';
8 use File::Path;
9 use MIME::Base64 qw(encode_base64);
10
11 use IO::Socket::IP;
12 use IO::Socket::UNIX;
13 use Socket qw(SOCK_STREAM);
14
15 use PVE::SafeSyslog;
16 use PVE::Cluster;
17 use PVE::INotify;
18 use PVE::RPCEnvironment;
19 use PVE::Storage;
20 use PVE::Tools qw(extract_param);
21 use PVE::API2::Storage::Config;
22 use PVE::API2::Storage::Content;
23 use PVE::API2::Storage::PruneBackups;
24 use PVE::API2::Storage::Scan;
25 use PVE::API2::Storage::Status;
26 use PVE::JSONSchema qw(get_standard_option);
27 use PVE::PTY;
28
29 use PVE::CLIHandler;
30
31 use base qw(PVE::CLIHandler);
32
33 my $KNOWN_EXPORT_FORMATS = ['raw+size', 'tar+size', 'qcow2+size', 'vmdk+size', 'zfs', 'btrfs'];
34
35 my $nodename = PVE::INotify::nodename();
36
37 sub param_mapping {
38 my ($name) = @_;
39
40 my $password_map = PVE::CLIHandler::get_standard_mapping('pve-password', {
41 func => sub {
42 my ($value) = @_;
43 return $value if $value;
44 return PVE::PTY::read_password("Enter Password: ");
45 },
46 });
47
48 my $enc_key_map = {
49 name => 'encryption-key',
50 desc => 'a file containing an encryption key, or the special value "autogen"',
51 func => sub {
52 my ($value) = @_;
53 return $value if $value eq 'autogen';
54 return PVE::Tools::file_get_contents($value);
55 }
56 };
57
58 my $master_key_map = {
59 name => 'master-pubkey',
60 desc => 'a file containing a PEM-formatted master public key',
61 func => sub {
62 my ($value) = @_;
63 return encode_base64(PVE::Tools::file_get_contents($value), '');
64 }
65 };
66
67 my $keyring_map = {
68 name => 'keyring',
69 desc => 'file containing the keyring to authenticate in the Ceph cluster',
70 func => sub {
71 my ($value) = @_;
72 return PVE::Tools::file_get_contents($value);
73 },
74 };
75
76 my $mapping = {
77 'cifsscan' => [ $password_map ],
78 'cifs' => [ $password_map ],
79 'pbs' => [ $password_map ],
80 'create' => [ $password_map, $enc_key_map, $master_key_map, $keyring_map ],
81 'update' => [ $password_map, $enc_key_map, $master_key_map, $keyring_map ],
82 };
83 return $mapping->{$name};
84 }
85
86 sub setup_environment {
87 PVE::RPCEnvironment->setup_default_cli_env();
88 }
89
90 __PACKAGE__->register_method ({
91 name => 'apiinfo',
92 path => 'apiinfo',
93 method => 'GET',
94 description => "Returns APIVER and APIAGE.",
95 parameters => {
96 additionalProperties => 0,
97 properties => {},
98 },
99 returns => {
100 type => 'object',
101 properties => {
102 apiver => { type => 'integer' },
103 apiage => { type => 'integer' },
104 },
105 },
106 code => sub {
107 return {
108 apiver => PVE::Storage::APIVER,
109 apiage => PVE::Storage::APIAGE,
110 };
111 }
112 });
113
114 __PACKAGE__->register_method ({
115 name => 'path',
116 path => 'path',
117 method => 'GET',
118 description => "Get filesystem path for specified volume",
119 parameters => {
120 additionalProperties => 0,
121 properties => {
122 volume => {
123 description => "Volume identifier",
124 type => 'string', format => 'pve-volume-id',
125 completion => \&PVE::Storage::complete_volume,
126 },
127 },
128 },
129 returns => { type => 'null' },
130
131 code => sub {
132 my ($param) = @_;
133
134 my $cfg = PVE::Storage::config();
135
136 my $path = PVE::Storage::path ($cfg, $param->{volume});
137
138 print "$path\n";
139
140 return undef;
141
142 }});
143
144 __PACKAGE__->register_method ({
145 name => 'extractconfig',
146 path => 'extractconfig',
147 method => 'GET',
148 description => "Extract configuration from vzdump backup archive.",
149 permissions => {
150 description => "The user needs 'VM.Backup' permissions on the backed up guest ID, and 'Datastore.AllocateSpace' on the backup storage.",
151 user => 'all',
152 },
153 protected => 1,
154 parameters => {
155 additionalProperties => 0,
156 properties => {
157 volume => {
158 description => "Volume identifier",
159 type => 'string',
160 completion => \&PVE::Storage::complete_volume,
161 },
162 },
163 },
164 returns => { type => 'null' },
165 code => sub {
166 my ($param) = @_;
167 my $volume = $param->{volume};
168
169 my $rpcenv = PVE::RPCEnvironment::get();
170 my $authuser = $rpcenv->get_user();
171
172 my $storage_cfg = PVE::Storage::config();
173 PVE::Storage::check_volume_access($rpcenv, $authuser, $storage_cfg, undef, $volume);
174
175 if (PVE::Storage::parse_volume_id($volume, 1)) {
176 my (undef, undef, $ownervm) = PVE::Storage::parse_volname($storage_cfg, $volume);
177 $rpcenv->check($authuser, "/vms/$ownervm", ['VM.Backup']);
178 }
179
180 my $config_raw = PVE::Storage::extract_vzdump_config($storage_cfg, $volume);
181
182 print "$config_raw\n";
183 return;
184 }});
185
186 my $print_content = sub {
187 my ($list) = @_;
188
189 my ($maxlenname, $maxsize) = (0, 0);
190 foreach my $info (@$list) {
191 my $volid = $info->{volid};
192 my $sidlen = length ($volid);
193 $maxlenname = $sidlen if $sidlen > $maxlenname;
194 $maxsize = $info->{size} if ($info->{size} // 0) > $maxsize;
195 }
196 my $sizemaxdigits = length($maxsize);
197
198 my $basefmt = "%-${maxlenname}s %-7s %-9s %${sizemaxdigits}s";
199 printf "$basefmt %s\n", "Volid", "Format", "Type", "Size", "VMID";
200
201 foreach my $info (@$list) {
202 next if !$info->{vmid};
203 my $volid = $info->{volid};
204
205 printf "$basefmt %d\n", $volid, $info->{format}, $info->{content}, $info->{size}, $info->{vmid};
206 }
207
208 foreach my $info (sort { $a->{format} cmp $b->{format} } @$list) {
209 next if $info->{vmid};
210 my $volid = $info->{volid};
211
212 printf "$basefmt\n", $volid, $info->{format}, $info->{content}, $info->{size};
213 }
214 };
215
216 my $print_status = sub {
217 my $res = shift;
218
219 my $maxlen = 0;
220 foreach my $res (@$res) {
221 my $storeid = $res->{storage};
222 $maxlen = length ($storeid) if length ($storeid) > $maxlen;
223 }
224 $maxlen+=1;
225
226 printf "%-${maxlen}s %10s %10s %15s %15s %15s %8s\n", 'Name', 'Type',
227 'Status', 'Total', 'Used', 'Available', '%';
228
229 foreach my $res (sort { $a->{storage} cmp $b->{storage} } @$res) {
230 my $storeid = $res->{storage};
231
232 my $active = $res->{active} ? 'active' : 'inactive';
233 my ($per, $per_fmt) = (0, '% 7.2f%%');
234 $per = ($res->{used}*100)/$res->{total} if $res->{total} > 0;
235
236 if (!$res->{enabled}) {
237 $per = 'N/A';
238 $per_fmt = '% 8s';
239 $active = 'disabled';
240 }
241
242 printf "%-${maxlen}s %10s %10s %15d %15d %15d $per_fmt\n", $storeid,
243 $res->{type}, $active, $res->{total}/1024, $res->{used}/1024,
244 $res->{avail}/1024, $per;
245 }
246 };
247
248 __PACKAGE__->register_method ({
249 name => 'export',
250 path => 'export',
251 method => 'GET',
252 description => "Used internally to export a volume.",
253 protected => 1,
254 parameters => {
255 additionalProperties => 0,
256 properties => {
257 volume => {
258 description => "Volume identifier",
259 type => 'string',
260 completion => \&PVE::Storage::complete_volume,
261 },
262 format => {
263 description => "Export stream format",
264 type => 'string',
265 enum => $KNOWN_EXPORT_FORMATS,
266 },
267 filename => {
268 description => "Destination file name",
269 type => 'string',
270 },
271 base => {
272 description => "Snapshot to start an incremental stream from",
273 type => 'string',
274 pattern => qr/[a-z0-9_\-]{1,40}/i,
275 maxLength => 40,
276 optional => 1,
277 },
278 snapshot => {
279 description => "Snapshot to export",
280 type => 'string',
281 pattern => qr/[a-z0-9_\-]{1,40}/i,
282 maxLength => 40,
283 optional => 1,
284 },
285 'with-snapshots' => {
286 description =>
287 "Whether to include intermediate snapshots in the stream",
288 type => 'boolean',
289 optional => 1,
290 default => 0,
291 },
292 'snapshot-list' => {
293 description => "Ordered list of snapshots to transfer",
294 type => 'string',
295 format => 'string-list',
296 optional => 1,
297 },
298 },
299 },
300 returns => { type => 'null' },
301 code => sub {
302 my ($param) = @_;
303
304 my $with_snapshots = $param->{'with-snapshots'};
305 if (defined(my $list = $param->{'snapshot-list'})) {
306 $with_snapshots = PVE::Tools::split_list($list);
307 }
308
309 my $filename = $param->{filename};
310
311 my $outfh;
312 if ($filename eq '-') {
313 $outfh = \*STDOUT;
314 } else {
315 sysopen($outfh, $filename, O_CREAT|O_WRONLY|O_TRUNC)
316 or die "open($filename): $!\n";
317 }
318
319 eval {
320 my $cfg = PVE::Storage::config();
321 PVE::Storage::volume_export($cfg, $outfh, $param->{volume}, $param->{format},
322 $param->{snapshot}, $param->{base}, $with_snapshots);
323 };
324 my $err = $@;
325 if ($filename ne '-') {
326 close($outfh);
327 unlink($filename) if $err;
328 }
329 die $err if $err;
330 return;
331 }
332 });
333
334 __PACKAGE__->register_method ({
335 name => 'import',
336 path => 'import',
337 method => 'PUT',
338 description => "Used internally to import a volume.",
339 protected => 1,
340 parameters => {
341 additionalProperties => 0,
342 properties => {
343 volume => {
344 description => "Volume identifier",
345 type => 'string',
346 completion => \&PVE::Storage::complete_volume,
347 },
348 format => {
349 description => "Import stream format",
350 type => 'string',
351 enum => $KNOWN_EXPORT_FORMATS,
352 },
353 filename => {
354 description => "Source file name. For '-' stdin is used, the " .
355 "tcp://<IP-or-CIDR> format allows to use a TCP connection, " .
356 "the unix://PATH-TO-SOCKET format a UNIX socket as input." .
357 "Else, the file is treated as common file.",
358 type => 'string',
359 },
360 base => {
361 description => "Base snapshot of an incremental stream",
362 type => 'string',
363 pattern => qr/[a-z0-9_\-]{1,40}/i,
364 maxLength => 40,
365 optional => 1,
366 },
367 'with-snapshots' => {
368 description =>
369 "Whether the stream includes intermediate snapshots",
370 type => 'boolean',
371 optional => 1,
372 default => 0,
373 },
374 'delete-snapshot' => {
375 description => "A snapshot to delete on success",
376 type => 'string',
377 pattern => qr/[a-z0-9_\-]{1,80}/i,
378 maxLength => 80,
379 optional => 1,
380 },
381 'allow-rename' => {
382 description => "Choose a new volume ID if the requested " .
383 "volume ID already exists, instead of throwing an error.",
384 type => 'boolean',
385 optional => 1,
386 default => 0,
387 },
388 snapshot => {
389 description => "The current-state snapshot if the stream contains snapshots",
390 type => 'string',
391 pattern => qr/[a-z0-9_\-]{1,40}/i,
392 maxLength => 40,
393 optional => 1,
394 },
395 },
396 },
397 returns => { type => 'string' },
398 code => sub {
399 my ($param) = @_;
400
401 my $filename = $param->{filename};
402
403 my $infh;
404 if ($filename eq '-') {
405 $infh = \*STDIN;
406 } elsif ($filename =~ m!^tcp://(([^/]+)(/\d+)?)$!) {
407 my ($cidr, $ip, $subnet) = ($1, $2, $3);
408 if ($subnet) { # got real CIDR notation, not just IP
409 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
410 die "Unable to get any local IP address in network '$cidr'\n"
411 if scalar(@$ips) < 1;
412 die "Got multiple local IP address in network '$cidr'\n"
413 if scalar(@$ips) > 1;
414
415 $ip = $ips->[0];
416 }
417 my $family = PVE::Tools::get_host_address_family($ip);
418 my $port = PVE::Tools::next_migrate_port($family, $ip);
419
420 my $sock_params = {
421 Listen => 1,
422 ReuseAddr => 1,
423 Proto => &Socket::IPPROTO_TCP,
424 GetAddrInfoFlags => 0,
425 LocalAddr => $ip,
426 LocalPort => $port,
427 };
428 my $socket = IO::Socket::IP->new(%$sock_params)
429 or die "failed to open socket: $!\n";
430
431 print "$ip\n$port\n"; # tell remote where to connect
432 *STDOUT->flush();
433
434 my $prev_alarm = alarm 0;
435 local $SIG{ALRM} = sub { die "timed out waiting for client\n" };
436 alarm 30;
437 my $client = $socket->accept; # Wait for a client
438 alarm $prev_alarm;
439 close($socket);
440
441 $infh = \*$client;
442 } elsif ($filename =~ m!^unix://(.*)$!) {
443 my $socket_path = $1;
444 my $socket = IO::Socket::UNIX->new(
445 Type => SOCK_STREAM(),
446 Local => $socket_path,
447 Listen => 1,
448 ) or die "failed to open socket: $!\n";
449
450 print "ready\n";
451 *STDOUT->flush();
452
453 my $prev_alarm = alarm 0;
454 local $SIG{ALRM} = sub { die "timed out waiting for client\n" };
455 alarm 30;
456 my $client = $socket->accept; # Wait for a client
457 alarm $prev_alarm;
458 close($socket);
459
460 $infh = \*$client;
461 } else {
462 sysopen($infh, $filename, O_RDONLY)
463 or die "open($filename): $!\n";
464 }
465
466 my $cfg = PVE::Storage::config();
467 my $volume = $param->{volume};
468 my $delete = $param->{'delete-snapshot'};
469 my $imported_volid = PVE::Storage::volume_import($cfg, $infh, $volume, $param->{format},
470 $param->{snapshot}, $param->{base}, $param->{'with-snapshots'},
471 $param->{'allow-rename'});
472 PVE::Storage::volume_snapshot_delete($cfg, $imported_volid, $delete)
473 if defined($delete);
474 return $imported_volid;
475 }
476 });
477
478 __PACKAGE__->register_method ({
479 name => 'prunebackups',
480 path => 'prunebackups',
481 method => 'GET',
482 description => "Prune backups. Only those using the standard naming scheme are considered. " .
483 "If no keep options are specified, those from the storage configuration are used.",
484 protected => 1,
485 proxyto => 'node',
486 parameters => {
487 additionalProperties => 0,
488 properties => {
489 'dry-run' => {
490 description => "Only show what would be pruned, don't delete anything.",
491 type => 'boolean',
492 optional => 1,
493 },
494 node => get_standard_option('pve-node'),
495 storage => get_standard_option('pve-storage-id', {
496 completion => \&PVE::Storage::complete_storage_enabled,
497 }),
498 %{$PVE::Storage::Plugin::prune_backups_format},
499 type => {
500 description => "Either 'qemu' or 'lxc'. Only consider backups for guests of this type.",
501 type => 'string',
502 optional => 1,
503 enum => ['qemu', 'lxc'],
504 },
505 vmid => get_standard_option('pve-vmid', {
506 description => "Only consider backups for this guest.",
507 optional => 1,
508 completion => \&PVE::Cluster::complete_vmid,
509 }),
510 },
511 },
512 returns => {
513 type => 'object',
514 properties => {
515 dryrun => {
516 description => 'If it was a dry run or not. The list will only be defined in that case.',
517 type => 'boolean',
518 },
519 list => {
520 type => 'array',
521 items => {
522 type => 'object',
523 properties => {
524 volid => {
525 description => "Backup volume ID.",
526 type => 'string',
527 },
528 'ctime' => {
529 description => "Creation time of the backup (seconds since the UNIX epoch).",
530 type => 'integer',
531 },
532 'mark' => {
533 description => "Whether the backup would be kept or removed. For backups that don't " .
534 "use the standard naming scheme, it's 'protected'.",
535 type => 'string',
536 },
537 type => {
538 description => "One of 'qemu', 'lxc', 'openvz' or 'unknown'.",
539 type => 'string',
540 },
541 'vmid' => {
542 description => "The VM the backup belongs to.",
543 type => 'integer',
544 optional => 1,
545 },
546 },
547 },
548 },
549 },
550 },
551 code => sub {
552 my ($param) = @_;
553
554 my $dryrun = extract_param($param, 'dry-run') ? 1 : 0;
555
556 my $keep_opts;
557 foreach my $keep (keys %{$PVE::Storage::Plugin::prune_backups_format}) {
558 $keep_opts->{$keep} = extract_param($param, $keep) if defined($param->{$keep});
559 }
560 $param->{'prune-backups'} = PVE::JSONSchema::print_property_string(
561 $keep_opts, $PVE::Storage::Plugin::prune_backups_format) if $keep_opts;
562
563 my $list = [];
564 if ($dryrun) {
565 $list = PVE::API2::Storage::PruneBackups->dryrun($param);
566 } else {
567 PVE::API2::Storage::PruneBackups->delete($param);
568 }
569
570 return {
571 dryrun => $dryrun,
572 list => $list,
573 };
574 }});
575
576 my $print_api_result = sub {
577 my ($data, $schema, $options) = @_;
578 PVE::CLIFormatter::print_api_result($data, $schema, undef, $options);
579 };
580
581 our $cmddef = {
582 add => [ "PVE::API2::Storage::Config", 'create', ['type', 'storage'] ],
583 set => [ "PVE::API2::Storage::Config", 'update', ['storage'] ],
584 remove => [ "PVE::API2::Storage::Config", 'delete', ['storage'] ],
585 status => [ "PVE::API2::Storage::Status", 'index', [],
586 { node => $nodename }, $print_status ],
587 list => [ "PVE::API2::Storage::Content", 'index', ['storage'],
588 { node => $nodename }, $print_content ],
589 alloc => [ "PVE::API2::Storage::Content", 'create', ['storage', 'vmid', 'filename', 'size'],
590 { node => $nodename }, sub {
591 my $volid = shift;
592 print "successfully created '$volid'\n";
593 }],
594 free => [ "PVE::API2::Storage::Content", 'delete', ['volume'],
595 { node => $nodename } ],
596 scan => {
597 nfs => [ "PVE::API2::Storage::Scan", 'nfsscan', ['server'], { node => $nodename }, sub {
598 my $res = shift;
599
600 my $maxlen = 0;
601 foreach my $rec (@$res) {
602 my $len = length ($rec->{path});
603 $maxlen = $len if $len > $maxlen;
604 }
605 foreach my $rec (@$res) {
606 printf "%-${maxlen}s %s\n", $rec->{path}, $rec->{options};
607 }
608 }],
609 cifs => [ "PVE::API2::Storage::Scan", 'cifsscan', ['server'], { node => $nodename }, sub {
610 my $res = shift;
611
612 my $maxlen = 0;
613 foreach my $rec (@$res) {
614 my $len = length ($rec->{share});
615 $maxlen = $len if $len > $maxlen;
616 }
617 foreach my $rec (@$res) {
618 printf "%-${maxlen}s %s\n", $rec->{share}, $rec->{description};
619 }
620 }],
621 glusterfs => [ "PVE::API2::Storage::Scan", 'glusterfsscan', ['server'], { node => $nodename }, sub {
622 my $res = shift;
623
624 foreach my $rec (@$res) {
625 printf "%s\n", $rec->{volname};
626 }
627 }],
628 iscsi => [ "PVE::API2::Storage::Scan", 'iscsiscan', ['portal'], { node => $nodename }, sub {
629 my $res = shift;
630
631 my $maxlen = 0;
632 foreach my $rec (@$res) {
633 my $len = length ($rec->{target});
634 $maxlen = $len if $len > $maxlen;
635 }
636 foreach my $rec (@$res) {
637 printf "%-${maxlen}s %s\n", $rec->{target}, $rec->{portal};
638 }
639 }],
640 lvm => [ "PVE::API2::Storage::Scan", 'lvmscan', [], { node => $nodename }, sub {
641 my $res = shift;
642 foreach my $rec (@$res) {
643 printf "$rec->{vg}\n";
644 }
645 }],
646 lvmthin => [ "PVE::API2::Storage::Scan", 'lvmthinscan', ['vg'], { node => $nodename }, sub {
647 my $res = shift;
648 foreach my $rec (@$res) {
649 printf "$rec->{lv}\n";
650 }
651 }],
652 pbs => [
653 "PVE::API2::Storage::Scan",
654 'pbsscan',
655 ['server', 'username'],
656 { node => $nodename },
657 $print_api_result,
658 $PVE::RESTHandler::standard_output_options,
659 ],
660 zfs => [ "PVE::API2::Storage::Scan", 'zfsscan', [], { node => $nodename }, sub {
661 my $res = shift;
662
663 foreach my $rec (@$res) {
664 printf "$rec->{pool}\n";
665 }
666 }],
667 },
668 nfsscan => { alias => 'scan nfs' },
669 cifsscan => { alias => 'scan cifs' },
670 glusterfsscan => { alias => 'scan glusterfs' },
671 iscsiscan => { alias => 'scan iscsi' },
672 lvmscan => { alias => 'scan lvm' },
673 lvmthinscan => { alias => 'scan lvmthin' },
674 zfsscan => { alias => 'scan zfs' },
675 path => [ __PACKAGE__, 'path', ['volume']],
676 extractconfig => [__PACKAGE__, 'extractconfig', ['volume']],
677 export => [ __PACKAGE__, 'export', ['volume', 'format', 'filename']],
678 import => [ __PACKAGE__, 'import', ['volume', 'format', 'filename'], {}, sub {
679 my $volid = shift;
680 print PVE::Storage::volume_imported_message($volid);
681 }],
682 apiinfo => [ __PACKAGE__, 'apiinfo', [], {}, sub {
683 my $res = shift;
684
685 print "APIVER $res->{apiver}\n";
686 print "APIAGE $res->{apiage}\n";
687 }],
688 'prune-backups' => [ __PACKAGE__, 'prunebackups', ['storage'], { node => $nodename }, sub {
689 my $res = shift;
690
691 my ($dryrun, $list) = ($res->{dryrun}, $res->{list});
692
693 return if !$dryrun;
694
695 if (!scalar(@{$list})) {
696 print "No backups found\n";
697 return;
698 }
699
700 print "NOTE: this is only a preview and might not be what a subsequent\n" .
701 "prune call does if backups are removed/added in the meantime.\n\n";
702
703 my @sorted = sort {
704 my $vmcmp = PVE::Tools::safe_compare($a->{vmid}, $b->{vmid}, sub { $_[0] <=> $_[1] });
705 return $vmcmp if $vmcmp ne 0;
706 return $a->{ctime} <=> $b->{ctime};
707 } @{$list};
708
709 my $maxlen = 0;
710 foreach my $backup (@sorted) {
711 my $volid = $backup->{volid};
712 $maxlen = length($volid) if length($volid) > $maxlen;
713 }
714 $maxlen+=1;
715
716 printf("%-${maxlen}s %15s %10s\n", 'Backup', 'Backup-ID', 'Prune-Mark');
717 foreach my $backup (@sorted) {
718 my $type = $backup->{type};
719 my $vmid = $backup->{vmid};
720 my $backup_id = defined($vmid) ? "$type/$vmid" : "$type";
721 printf("%-${maxlen}s %15s %10s\n", $backup->{volid}, $backup_id, $backup->{mark});
722 }
723 }],
724 };
725
726 1;