1 package PVE
::Storage
::PBSPlugin
;
3 # Plugin to access Proxmox Backup Server
8 use Fcntl
qw(F_GETFD F_SETFD FD_CLOEXEC);
11 use MIME
::Base64
qw(decode_base64);
12 use POSIX
qw(strftime ENOENT);
14 use PVE
::APIClient
::LWP
;
15 use PVE
::JSONSchema
qw(get_standard_option);
18 use PVE
::Storage
::Plugin
;
19 use PVE
::Tools
qw(run_command file_read_firstline trim dir_glob_regex dir_glob_foreach $IPV6RE);
21 use base
qw(PVE::Storage::Plugin);
31 content
=> [ {backup
=> 1, none
=> 1}, { backup
=> 1 }],
38 description
=> "Proxmox Backup Server datastore name.",
41 # openssl s_client -connect <host>:8007 2>&1 |openssl x509 -fingerprint -sha256
42 fingerprint
=> get_standard_option
('fingerprint-sha256'),
44 description
=> "Encryption key. Use 'autogen' to generate one automatically without passphrase.",
48 description
=> "Base64-encoded, PEM-formatted public RSA key. Used to encrypt a copy of the encryption-key which will be added to each encrypted backup.",
52 description
=> "For non default port.",
63 server
=> { fixed
=> 1 },
64 datastore
=> { fixed
=> 1 },
65 port
=> { optional
=> 1 },
66 nodes
=> { optional
=> 1},
67 disable
=> { optional
=> 1},
68 content
=> { optional
=> 1},
69 username
=> { optional
=> 1 },
70 password
=> { optional
=> 1 },
71 'encryption-key' => { optional
=> 1 },
72 'master-pubkey' => { optional
=> 1 },
73 maxfiles
=> { optional
=> 1 },
74 'prune-backups' => { optional
=> 1 },
75 fingerprint
=> { optional
=> 1 },
81 sub pbs_password_file_name
{
82 my ($scfg, $storeid) = @_;
84 return "/etc/pve/priv/storage/${storeid}.pw";
87 sub pbs_set_password
{
88 my ($scfg, $storeid, $password) = @_;
90 my $pwfile = pbs_password_file_name
($scfg, $storeid);
91 mkdir "/etc/pve/priv/storage";
93 PVE
::Tools
::file_set_contents
($pwfile, "$password\n");
96 sub pbs_delete_password
{
97 my ($scfg, $storeid) = @_;
99 my $pwfile = pbs_password_file_name
($scfg, $storeid);
104 sub pbs_get_password
{
105 my ($scfg, $storeid) = @_;
107 my $pwfile = pbs_password_file_name
($scfg, $storeid);
109 return PVE
::Tools
::file_read_firstline
($pwfile);
112 sub pbs_encryption_key_file_name
{
113 my ($scfg, $storeid) = @_;
115 return "/etc/pve/priv/storage/${storeid}.enc";
118 sub pbs_set_encryption_key
{
119 my ($scfg, $storeid, $key) = @_;
121 my $pwfile = pbs_encryption_key_file_name
($scfg, $storeid);
122 mkdir "/etc/pve/priv/storage";
124 PVE
::Tools
::file_set_contents
($pwfile, "$key\n");
127 sub pbs_delete_encryption_key
{
128 my ($scfg, $storeid) = @_;
130 my $pwfile = pbs_encryption_key_file_name
($scfg, $storeid);
132 if (!unlink $pwfile) {
133 return if $! == ENOENT
;
134 die "failed to delete encryption key! $!\n";
136 delete $scfg->{'encryption-key'};
139 sub pbs_get_encryption_key
{
140 my ($scfg, $storeid) = @_;
142 my $pwfile = pbs_encryption_key_file_name
($scfg, $storeid);
144 return PVE
::Tools
::file_get_contents
($pwfile);
147 # Returns a file handle if there is an encryption key, or `undef` if there is not. Dies on error.
148 sub pbs_open_encryption_key
{
149 my ($scfg, $storeid) = @_;
151 my $encryption_key_file = pbs_encryption_key_file_name
($scfg, $storeid);
154 if (!open($keyfd, '<', $encryption_key_file)) {
155 return undef if $! == ENOENT
;
156 die "failed to open encryption key: $encryption_key_file: $!\n";
162 sub pbs_master_pubkey_file_name
{
163 my ($scfg, $storeid) = @_;
165 return "/etc/pve/priv/storage/${storeid}.master.pem";
168 sub pbs_set_master_pubkey
{
169 my ($scfg, $storeid, $key) = @_;
171 my $pwfile = pbs_master_pubkey_file_name
($scfg, $storeid);
172 mkdir "/etc/pve/priv/storage";
174 PVE
::Tools
::file_set_contents
($pwfile, "$key\n");
177 sub pbs_delete_master_pubkey
{
178 my ($scfg, $storeid) = @_;
180 my $pwfile = pbs_master_pubkey_file_name
($scfg, $storeid);
182 if (!unlink $pwfile) {
183 return if $! == ENOENT
;
184 die "failed to delete master public key! $!\n";
186 delete $scfg->{'master-pubkey'};
189 sub pbs_get_master_pubkey
{
190 my ($scfg, $storeid) = @_;
192 my $pwfile = pbs_master_pubkey_file_name
($scfg, $storeid);
194 return PVE
::Tools
::file_get_contents
($pwfile);
197 # Returns a file handle if there is a master key, or `undef` if there is not. Dies on error.
198 sub pbs_open_master_pubkey
{
199 my ($scfg, $storeid) = @_;
201 my $master_pubkey_file = pbs_master_pubkey_file_name
($scfg, $storeid);
204 if (!open($keyfd, '<', $master_pubkey_file)) {
205 return undef if $! == ENOENT
;
206 die "failed to open master public key: $master_pubkey_file: $!\n";
213 my ($storeid, $btype, $bid, $btime) = @_;
215 my $time_str = strftime
("%FT%TZ", gmtime($btime));
216 my $volname = "backup/${btype}/${bid}/${time_str}";
218 return "${storeid}:${volname}";
221 my $USE_CRYPT_PARAMS = {
227 my $USE_MASTER_KEY = {
231 my sub do_raw_client_cmd
{
232 my ($scfg, $storeid, $client_cmd, $param, %opts) = @_;
234 my $use_crypto = $USE_CRYPT_PARAMS->{$client_cmd};
235 my $use_master = $USE_MASTER_KEY->{$client_cmd};
237 my $client_exe = '/usr/bin/proxmox-backup-client';
238 die "executable not found '$client_exe'! Proxmox backup client not installed?\n"
241 my $repo = PVE
::PBSClient
::get_repository
($scfg);
243 my $userns_cmd = delete $opts{userns_cmd
};
247 push @$cmd, @$userns_cmd if defined($userns_cmd);
249 push @$cmd, $client_exe, $client_cmd;
251 # This must live in the top scope to not get closed before the `run_command`
252 my ($keyfd, $master_fd);
254 if (defined($keyfd = pbs_open_encryption_key
($scfg, $storeid))) {
255 my $flags = fcntl($keyfd, F_GETFD
, 0)
256 // die "failed to get file descriptor flags: $!\n";
257 fcntl($keyfd, F_SETFD
, $flags & ~FD_CLOEXEC
)
258 or die "failed to remove FD_CLOEXEC from encryption key file descriptor\n";
259 push @$cmd, '--crypt-mode=encrypt', '--keyfd='.fileno($keyfd);
260 if ($use_master && defined($master_fd = pbs_open_master_pubkey
($scfg, $storeid))) {
261 my $flags = fcntl($master_fd, F_GETFD
, 0)
262 // die "failed to get file descriptor flags: $!\n";
263 fcntl($master_fd, F_SETFD
, $flags & ~FD_CLOEXEC
)
264 or die "failed to remove FD_CLOEXEC from master public key file descriptor\n";
265 push @$cmd, '--master-pubkey-fd='.fileno($master_fd);
268 push @$cmd, '--crypt-mode=none';
272 push @$cmd, @$param if defined($param);
274 push @$cmd, "--repository", $repo;
276 local $ENV{PBS_PASSWORD
} = pbs_get_password
($scfg, $storeid);
278 local $ENV{PBS_FINGERPRINT
} = $scfg->{fingerprint
};
280 # no ascii-art on task logs
281 local $ENV{PROXMOX_OUTPUT_NO_BORDER
} = 1;
282 local $ENV{PROXMOX_OUTPUT_NO_HEADER
} = 1;
284 if (my $logfunc = $opts{logfunc
}) {
285 $logfunc->("run: " . join(' ', @$cmd));
288 run_command
($cmd, %opts);
291 # FIXME: External perl code should NOT have access to this.
293 # There should be separate functions to
298 sub run_raw_client_cmd
{
299 my ($scfg, $storeid, $client_cmd, $param, %opts) = @_;
300 return do_raw_client_cmd
($scfg, $storeid, $client_cmd, $param, %opts);
304 my ($scfg, $storeid, $client_cmd, $param, $no_output) = @_;
307 my $outfunc = sub { $json_str .= "$_[0]\n" };
309 $param = [] if !defined($param);
310 $param = [ $param ] if !ref($param);
312 $param = [@$param, '--output-format=json'] if !$no_output;
314 do_raw_client_cmd
($scfg, $storeid, $client_cmd, $param,
315 outfunc
=> $outfunc, errmsg
=> 'proxmox-backup-client failed');
317 return undef if $no_output;
319 my $res = decode_json
($json_str);
324 # Storage implementation
326 sub extract_vzdump_config
{
327 my ($class, $scfg, $volname, $storeid) = @_;
329 my ($vtype, $name, $vmid, undef, undef, undef, $format) = $class->parse_volname($volname);
332 my $outfunc = sub { $config .= "$_[0]\n" };
335 if ($format eq 'pbs-vm') {
336 $config_name = 'qemu-server.conf';
337 } elsif ($format eq 'pbs-ct') {
338 $config_name = 'pct.conf';
340 die "unable to extract configuration for backup format '$format'\n";
343 do_raw_client_cmd
($scfg, $storeid, 'restore', [ $name, $config_name, '-' ],
344 outfunc
=> $outfunc, errmsg
=> 'proxmox-backup-client failed');
350 my ($class, $scfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc) = @_;
352 $logfunc //= sub { print "$_[1]\n" };
354 my $backups = $class->list_volumes($storeid, $scfg, $vmid, ['backup']);
356 $type = 'vm' if defined($type) && $type eq 'qemu';
357 $type = 'ct' if defined($type) && $type eq 'lxc';
359 my $backup_groups = {};
360 foreach my $backup (@{$backups}) {
361 (my $backup_type = $backup->{format
}) =~ s/^pbs-//;
363 next if defined($type) && $backup_type ne $type;
365 my $backup_group = "$backup_type/$backup->{vmid}";
366 $backup_groups->{$backup_group} = 1;
371 my $keep_all = delete $keep->{'keep-all'};
374 foreach my $opt (keys %{$keep}) {
375 next if $keep->{$opt} == 0;
376 push @param, "--$opt";
377 push @param, "$keep->{$opt}";
379 } else { # no need to pass anything to PBS
380 $keep = { 'keep-all' => 1 };
383 push @param, '--dry-run' if $dryrun;
388 foreach my $backup_group (keys %{$backup_groups}) {
389 $logfunc->('info', "running 'proxmox-backup-client prune' for '$backup_group'")
392 my $res = run_client_cmd
($scfg, $storeid, 'prune', [ $backup_group, @param ]);
394 foreach my $backup (@{$res}) {
395 die "result from proxmox-backup-client is not as expected\n"
396 if !defined($backup->{'backup-time'})
397 || !defined($backup->{'backup-type'})
398 || !defined($backup->{'backup-id'})
399 || !defined($backup->{'keep'});
401 my $ctime = $backup->{'backup-time'};
402 my $type = $backup->{'backup-type'};
403 my $vmid = $backup->{'backup-id'};
404 my $volid = print_volid
($storeid, $type, $vmid, $ctime);
406 push @{$prune_list}, {
408 mark
=> $backup->{keep
} ?
'keep' : 'remove',
409 type
=> $type eq 'vm' ?
'qemu' : 'lxc',
416 $logfunc->('err', "prune '$backup_group': $err\n");
420 die "error pruning backups - check log\n" if $failed;
425 my $autogen_encryption_key = sub {
426 my ($scfg, $storeid) = @_;
427 my $encfile = pbs_encryption_key_file_name
($scfg, $storeid);
429 rename $encfile, "$encfile.old";
431 my $cmd = ['proxmox-backup-client', 'key', 'create', '--kdf', 'none', $encfile];
432 run_command
($cmd, errmsg
=> 'failed to create encryption key');
433 return PVE
::Tools
::file_get_contents
($encfile);
437 my ($class, $storeid, $scfg, %param) = @_;
441 if (defined(my $password = $param{password
})) {
442 pbs_set_password
($scfg, $storeid, $password);
444 pbs_delete_password
($scfg, $storeid);
447 if (defined(my $encryption_key = $param{'encryption-key'})) {
449 if ($encryption_key eq 'autogen') {
450 $res->{'encryption-key'} = $autogen_encryption_key->($scfg, $storeid);
451 $decoded_key = decode_json
($res->{'encryption-key'});
453 $decoded_key = eval { decode_json
($encryption_key) };
454 if ($@ || !exists($decoded_key->{data
})) {
455 die "Value does not seems like a valid, JSON formatted encryption key!\n";
457 pbs_set_encryption_key
($scfg, $storeid, $encryption_key);
458 $res->{'encryption-key'} = $encryption_key;
460 $scfg->{'encryption-key'} = $decoded_key->{fingerprint
} || 1;
462 pbs_delete_encryption_key
($scfg, $storeid);
465 if (defined(my $master_key = delete $param{'master-pubkey'})) {
466 die "'master-pubkey' can only be used together with 'encryption-key'\n"
467 if !defined($scfg->{'encryption-key'});
469 my $decoded = decode_base64
($master_key);
470 pbs_set_master_pubkey
($scfg, $storeid, $decoded);
471 $scfg->{'master-pubkey'} = 1;
473 pbs_delete_master_pubkey
($scfg, $storeid);
480 my ($class, $storeid, $scfg, %param) = @_;
484 if (exists($param{password
})) {
485 if (defined($param{password
})) {
486 pbs_set_password
($scfg, $storeid, $param{password
});
488 pbs_delete_password
($scfg, $storeid);
492 if (exists($param{'encryption-key'})) {
493 if (defined(my $encryption_key = delete($param{'encryption-key'}))) {
495 if ($encryption_key eq 'autogen') {
496 $res->{'encryption-key'} = $autogen_encryption_key->($scfg, $storeid);
497 $decoded_key = decode_json
($res->{'encryption-key'});
499 $decoded_key = eval { decode_json
($encryption_key) };
500 if ($@ || !exists($decoded_key->{data
})) {
501 die "Value does not seems like a valid, JSON formatted encryption key!\n";
503 pbs_set_encryption_key
($scfg, $storeid, $encryption_key);
504 $res->{'encryption-key'} = $encryption_key;
506 $scfg->{'encryption-key'} = $decoded_key->{fingerprint
} || 1;
508 pbs_delete_encryption_key
($scfg, $storeid);
509 delete $scfg->{'encryption-key'};
513 if (exists($param{'master-pubkey'})) {
514 if (defined(my $master_key = delete($param{'master-pubkey'}))) {
515 my $decoded = decode_base64
($master_key);
517 pbs_set_master_pubkey
($scfg, $storeid, $decoded);
518 $scfg->{'master-pubkey'} = 1;
520 pbs_delete_master_pubkey
($scfg, $storeid);
528 my ($class, $storeid, $scfg) = @_;
530 pbs_delete_password
($scfg, $storeid);
531 pbs_delete_encryption_key
($scfg, $storeid);
532 pbs_delete_master_pubkey
($scfg, $storeid);
538 my ($class, $volname) = @_;
540 if ($volname =~ m!^backup/([^\s_]+)/([^\s_]+)/([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)$!) {
544 my $format = "pbs-$btype";
546 my $name = "$btype/$bid/$btime";
548 if ($bid =~ m/^\d+$/) {
549 return ('backup', $name, $bid, undef, undef, undef, $format);
551 return ('backup', $name, undef, undef, undef, undef, $format);
555 die "unable to parse PBS volume name '$volname'\n";
559 my ($class, $scfg, $volname, $storeid, $snapname) = @_;
561 die "volume snapshot is not possible on pbs storage"
562 if defined($snapname);
564 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
566 my $repo = PVE
::PBSClient
::get_repository
($scfg);
568 # artificial url - we currently do not use that anywhere
569 my $path = "pbs://$repo/$name";
571 return ($path, $vmid, $vtype);
575 my ($class, $storeid, $scfg, $volname) = @_;
577 die "can't create base images in pbs storage\n";
581 my ($class, $scfg, $storeid, $volname, $vmid, $snap) = @_;
583 die "can't clone images in pbs storage\n";
587 my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
589 die "can't allocate space in pbs storage\n";
593 my ($class, $storeid, $scfg, $volname, $isBase) = @_;
595 my ($vtype, $name, $vmid) = $class->parse_volname($volname);
597 run_client_cmd
($scfg, $storeid, "forget", [ $name ], 1);
604 my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
611 my sub snapshot_files_encrypted
{
617 for my $file (@$files) {
618 my $fn = $file->{filename
};
619 next if $fn eq 'client.log.blob' || $fn eq 'index.json.blob';
621 my $crypt = $file->{'crypt-mode'};
623 $all = 0 if !$crypt || $crypt ne 'encrypt';
624 $any ||= defined($crypt) && $crypt eq 'encrypt';
630 my ($class, $storeid, $scfg, $vmid, $content_types) = @_;
634 return $res if !grep { $_ eq 'backup' } @$content_types;
636 my $data = run_client_cmd
($scfg, $storeid, "snapshots");
638 foreach my $item (@$data) {
639 my $btype = $item->{"backup-type"};
640 my $bid = $item->{"backup-id"};
641 my $epoch = $item->{"backup-time"};
642 my $size = $item->{size
} // 1;
644 next if !($btype eq 'vm' || $btype eq 'ct');
645 next if $bid !~ m/^\d+$/;
646 next if defined($vmid) && $bid ne $vmid;
648 my $volid = print_volid
($storeid, $btype, $bid, $epoch);
652 format
=> "pbs-$btype",
659 $info->{verification
} = $item->{verification
} if defined($item->{verification
});
660 $info->{notes
} = $item->{comment
} if defined($item->{comment
});
661 if (defined($item->{fingerprint
})) {
662 $info->{encrypted
} = $item->{fingerprint
};
663 } elsif (snapshot_files_encrypted
($item->{files
})) {
664 $info->{encrypted
} = '1';
674 my ($class, $storeid, $scfg, $cache) = @_;
682 my $res = run_client_cmd
($scfg, $storeid, "status");
685 $total = $res->{total
};
686 $used = $res->{used
};
687 $free = $res->{avail
};
693 return ($total, $free, $used, $active);
696 # TODO: use a client with native rust/proxmox-backup bindings to profit from
697 # API schema checks and types
698 my sub pbs_api_connect
{
699 my ($scfg, $password) = @_;
703 my $user = $scfg->{username
} // 'root@pam';
705 if (my $tokenid = PVE
::AccessControl
::pve_verify_tokenid
($user, 1)) {
706 $params->{apitoken
} = "PBSAPIToken=${tokenid}:${password}";
708 $params->{password
} = $password;
709 $params->{username
} = $user;
712 if (my $fp = $scfg->{fingerprint
}) {
713 $params->{cached_fingerprints
}->{uc($fp)} = 1;
716 my $conn = PVE
::APIClient
::LWP-
>new(
718 host
=> $scfg->{server
},
719 port
=> $scfg->{port
} // 8007,
720 timeout
=> 7, # cope with a 401 (3s api delay) and high latency
721 cookie_name
=> 'PBSAuthCookie',
727 # can also be used for not (yet) added storages, pass $scfg with
731 # port (optional default to 8007)
732 # fingerprint (optional for trusted certs)
734 sub scan_datastores
{
735 my ($scfg, $password) = @_;
737 my $conn = pbs_api_connect
($scfg, $password);
739 my $response = eval { $conn->get('/api2/json/admin/datastore', {}) };
740 die "error fetching datastores - $@" if $@;
745 sub activate_storage
{
746 my ($class, $storeid, $scfg, $cache) = @_;
748 my $password = pbs_get_password
($scfg, $storeid);
750 my $datastores = eval { scan_datastores
($scfg, $password) };
751 die "$storeid: $@" if $@;
753 my $datastore = $scfg->{datastore
};
755 for my $ds (@$datastores) {
756 if ($ds->{store
} eq $datastore) {
761 die "$storeid: Cannot find datastore '$datastore', check permissions and existence!\n";
764 sub deactivate_storage
{
765 my ($class, $storeid, $scfg, $cache) = @_;
769 sub activate_volume
{
770 my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
772 die "volume snapshot is not possible on pbs device" if $snapname;
777 sub deactivate_volume
{
778 my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
780 die "volume snapshot is not possible on pbs device" if $snapname;
785 sub get_volume_notes
{
786 my ($class, $scfg, $storeid, $volname, $timeout) = @_;
788 my (undef, $name, undef, undef, undef, undef, $format) = $class->parse_volname($volname);
790 my $data = run_client_cmd
($scfg, $storeid, "snapshot", [ "notes", "show", $name ]);
792 return $data->{notes
};
795 sub update_volume_notes
{
796 my ($class, $scfg, $storeid, $volname, $notes, $timeout) = @_;
798 my (undef, $name, undef, undef, undef, undef, $format) = $class->parse_volname($volname);
800 run_client_cmd
($scfg, $storeid, "snapshot", [ "notes", "update", $name, $notes ], 1);
805 sub volume_size_info
{
806 my ($class, $scfg, $storeid, $volname, $timeout) = @_;
808 my ($vtype, $name, undef, undef, undef, undef, $format) = $class->parse_volname($volname);
810 my $data = run_client_cmd
($scfg, $storeid, "files", [ $name ]);
813 foreach my $info (@$data) {
814 if ($info->{size
} && $info->{size
} =~ /^(\d+)$/) { # untaints
821 return wantarray ?
($size, $format, $used, undef) : $size;
825 my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
826 die "volume resize is not possible on pbs device";
829 sub volume_snapshot
{
830 my ($class, $scfg, $storeid, $volname, $snap) = @_;
831 die "volume snapshot is not possible on pbs device";
834 sub volume_snapshot_rollback
{
835 my ($class, $scfg, $storeid, $volname, $snap) = @_;
836 die "volume snapshot rollback is not possible on pbs device";
839 sub volume_snapshot_delete
{
840 my ($class, $scfg, $storeid, $volname, $snap) = @_;
841 die "volume snapshot delete is not possible on pbs device";
844 sub volume_has_feature
{
845 my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;