use PVE::Storage::Plugin;
use PVE::JSONSchema qw(get_standard_option);
use PVE::RADOS;
+use PVE::CephConfig;
+use JSON;
use base qw(PVE::Storage::Plugin);
-my $rbd_unittobytes = {
- "k" => 1024,
- "M" => 1024*1024,
- "G" => 1024*1024*1024,
- "T" => 1024*1024*1024*1024,
+my $get_parent_image_name = sub {
+ my ($parent) = @_;
+ return undef if !$parent;
+ return $parent->{image} . "@" . $parent->{snapshot};
};
my $add_pool_to_disk = sub {
return "$pool/$disk";
};
-my $hostlist = sub {
- my ($list_text, $separator) = @_;
-
- my @monhostlist = PVE::Tools::split_list($list_text);
- return join($separator, map {
- my ($host, $port) = PVE::Tools::parse_host_and_port($_);
- $port = defined($port) ? ":$port" : '';
- $host = "[$host]" if Net::IP::ip_is_ipv6($host);
- "${host}${port}"
- } @monhostlist);
-};
-
-my $ceph_connect_option = sub {
- my ($scfg, $storeid, %options) = @_;
-
- my $cmd_option = {};
- my $ceph_storeid_conf = "/etc/pve/priv/ceph/${storeid}.conf";
- my $pveceph_config = '/etc/pve/ceph.conf';
- my $keyring = "/etc/pve/priv/ceph/${storeid}.keyring";
- my $pveceph_managed = !defined($scfg->{monhost});
-
- $cmd_option->{ceph_conf} = $pveceph_config if $pveceph_managed;
-
- if (-e $ceph_storeid_conf) {
- if ($pveceph_managed) {
- warn "ignoring custom ceph config for storage '$storeid', 'monhost' is not set (assuming pveceph managed cluster)!\n";
- } else {
- $cmd_option->{ceph_conf} = $ceph_storeid_conf;
- }
- }
-
- $cmd_option->{keyring} = $keyring if (-e $keyring);
- $cmd_option->{auth_supported} = (defined $cmd_option->{keyring}) ? 'cephx' : 'none';
- $cmd_option->{userid} = $scfg->{username} ? $scfg->{username} : 'admin';
- $cmd_option->{mon_host} = $hostlist->($scfg->{monhost}, ',') if (defined($scfg->{monhost}));
-
- if (%options) {
- foreach my $k (keys %options) {
- $cmd_option->{$k} = $options{$k};
- }
- }
-
- return $cmd_option;
-
-};
-
my $build_cmd = sub {
my ($binary, $scfg, $storeid, $op, @options) = @_;
- my $cmd_option = $ceph_connect_option->($scfg, $storeid);
+ my $cmd_option = PVE::CephConfig::ceph_connect_option($scfg, $storeid);
my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
my $cmd = [$binary, '-p', $pool];
my $librados_connect = sub {
my ($scfg, $storeid, $options) = @_;
- my $librados_config = $ceph_connect_option->($scfg, $storeid);
+ my $librados_config = PVE::CephConfig::ceph_connect_option($scfg, $storeid);
my $rados = PVE::RADOS->new(%$librados_config);
my $krbd_feature_disable = sub {
my ($scfg, $storeid, $name) = @_;
- return 1 if !$scfg->{krbd};
-
my ($major, undef, undef, undef) = ceph_version();
return 1 if $major < 10;
my $krbd_feature_blacklist = ['deep-flatten', 'fast-diff', 'object-map', 'exclusive-lock'];
my (undef, undef, undef, undef, $features) = rbd_volume_info($scfg, $storeid, $name);
- my $active_features = { map { $_ => 1 } PVE::Tools::split_list($features)};
+ my $active_features = { map { $_ => 1 } @$features };
my $incompatible_features = join(',', grep { %$active_features{$_} } @$krbd_feature_blacklist);
if ($incompatible_features) {
sub rbd_ls {
my ($scfg, $storeid) = @_;
- my $cmd = &$rbd_cmd($scfg, $storeid, 'ls', '-l');
+ my $cmd = &$rbd_cmd($scfg, $storeid, 'ls', '-l', '--format', 'json');
my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
- my $list = {};
-
- my $parser = sub {
- my $line = shift;
-
- if ($line =~ m/^((vm|base)-(\d+)-\S+)\s+(\d+)(k|M|G|T)\s((\S+)\/((vm|base)-\d+-\S+@\S+))?/) {
- my ($image, $owner, $size, $unit, $parent) = ($1, $3, $4, $5, $8);
- return if $image =~ /@/; #skip snapshots
-
- $list->{$pool}->{$image} = {
- name => $image,
- size => $size*$rbd_unittobytes->{$unit},
- parent => $parent,
- vmid => $owner
- };
- }
- };
+ my $raw = '';
+ my $parser = sub { $raw .= shift };
eval {
run_rbd_command($cmd, errmsg => "rbd error", errfunc => sub {}, outfunc => $parser);
my $err = $@;
die $err if $err && $err !~ m/doesn't contain rbd images/ ;
-
+
+ my $result;
+ if ($raw eq '') {
+ $result = [];
+ } elsif ($raw =~ m/^(\[.*\])$/s) { # untaint
+ $result = JSON::decode_json($1);
+ } else {
+ die "got unexpected data from rbd ls: '$raw'\n";
+ }
+
+ my $list = {};
+
+ foreach my $el (@$result) {
+ next if defined($el->{snapshot});
+
+ my $image = $el->{image};
+
+ my ($owner) = $image =~ m/^(?:vm|base)-(\d+)-/;
+ next if !defined($owner);
+
+ $list->{$pool}->{$image} = {
+ name => $image,
+ size => $el->{size},
+ parent => $get_parent_image_name->($el->{parent}),
+ vmid => $owner
+ };
+ }
+
return $list;
}
my $cmd = undef;
- if($snap){
- $cmd = &$rbd_cmd($scfg, $storeid, 'info', $volname, '--snap', $snap);
- }else{
- $cmd = &$rbd_cmd($scfg, $storeid, 'info', $volname);
+ my @options = ('info', $volname, '--format', 'json');
+ if ($snap) {
+ push @options, '--snap', $snap;
}
- my $size = undef;
- my $parent = undef;
- my $format = undef;
- my $protected = undef;
- my $features = undef;
+ $cmd = &$rbd_cmd($scfg, $storeid, @options);
- my $parser = sub {
- my $line = shift;
+ my $raw = '';
+ my $parser = sub { $raw .= shift };
- if ($line =~ m/size (\d+) (k|M|G|T)B in (\d+) objects/) {
- $size = $1 * $rbd_unittobytes->{$2} if ($1);
- } elsif ($line =~ m/parent:\s(\S+)\/(\S+)/) {
- $parent = $2;
- } elsif ($line =~ m/format:\s(\d+)/) {
- $format = $1;
- } elsif ($line =~ m/protected:\s(\S+)/) {
- $protected = 1 if $1 eq "True";
- } elsif ($line =~ m/features:\s(.+)/) {
- $features = $1;
- }
+ run_rbd_command($cmd, errmsg => "rbd error", errfunc => sub {}, outfunc => $parser);
- };
+ my $volume;
+ if ($raw eq '') {
+ $volume = {};
+ } elsif ($raw =~ m/^(\{.*\})$/s) { # untaint
+ $volume = JSON::decode_json($1);
+ } else {
+ die "got unexpected data from rbd info: '$raw'\n";
+ }
- run_rbd_command($cmd, errmsg => "rbd error", errfunc => sub {}, outfunc => $parser);
+ $volume->{parent} = $get_parent_image_name->($volume->{parent});
+ $volume->{protected} = defined($volume->{protected}) && $volume->{protected} eq "true" ? 1 : undef;
- return ($size, $parent, $format, $protected, $features);
+ return $volume->@{qw(size parent format protected features)};
}
# Configuration
type => 'string',
},
krbd => {
- description => "Access rbd through krbd kernel module.",
+ description => "Always access rbd through krbd kernel module.",
type => 'boolean',
},
};
return if defined($scfg->{monhost}); # nothing to do if not pve managed ceph
- my $ceph_admin_keyring = '/etc/pve/priv/ceph.client.admin.keyring';
- my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.keyring";
-
- die "ceph authx keyring file for storage '$storeid' already exists!\n"
- if -e $ceph_storage_keyring;
-
- eval {
- mkdir '/etc/pve/priv/ceph';
- PVE::Tools::file_copy($ceph_admin_keyring, $ceph_storage_keyring);
- };
- if (my $err = $@) {
- unlink $ceph_storage_keyring;
- die "failed to copy ceph authx keyring for storage '$storeid': $err\n";
- }
-
+ PVE::CephConfig::ceph_create_keyfile($scfg->{type}, $storeid);
}
sub on_delete_hook {
return if defined($scfg->{monhost}); # nothing to do if not pve managed ceph
- my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.keyring";
- if (-f $ceph_storage_keyring) {
- unlink($ceph_storage_keyring) or warn "removing keyring of storage failed: $!\n";
- }
-
+ PVE::CephConfig::ceph_remove_keyfile($scfg->{type}, $storeid);
}
sub parse_volname {
sub path {
my ($class, $scfg, $volname, $storeid, $snapname) = @_;
- my $cmd_option = $ceph_connect_option->($scfg, $storeid);
+ my $cmd_option = PVE::CephConfig::ceph_connect_option($scfg, $storeid);
my ($vtype, $name, $vmid) = $class->parse_volname($volname);
$name .= '@'.$snapname if $snapname;
$path .= ":conf=$cmd_option->{ceph_conf}" if $cmd_option->{ceph_conf};
if (defined($scfg->{monhost})) {
- my $monhost = $hostlist->($scfg->{monhost}, ';');
+ my $monhost = PVE::CephConfig::hostlist($scfg->{monhost}, ';');
$monhost =~ s/:/\\:/g;
$path .= ":mon_host=$monhost";
$path .= ":auth_supported=$cmd_option->{auth_supported}";
my ($storeid, $scfg, $vmid) = @_;
my $cmd = &$rbd_cmd($scfg, $storeid, 'ls');
- my $disk_ids = {};
+ my $disk_list = [];
my $parser = sub {
my $line = shift;
-
- if ($line =~ m/^(vm|base)-\Q$vmid\E+-disk-(\d+)$/) {
- $disk_ids->{$2} = 1;
+ if ($line =~ m/^(.*)$/) { # untaint
+ push @$disk_list, $1;
}
};
die $err if $err && $err !~ m/doesn't contain rbd images/;
- #fix: can we search in $rbd hash key with a regex to find (vm|base) ?
- for (my $i = 1; $i < 100; $i++) {
- if (!$disk_ids->{$i}) {
- return "vm-$vmid-disk-$i";
- }
- }
-
- die "unable to allocate an image name for VM $vmid in storage '$storeid'\n";
+ return PVE::Storage::Plugin::get_next_vm_diskname($disk_list, $storeid, $vmid, undef, $scfg);
};
sub create_base {
die "$volname is not a base image and snapname is not provided\n"
if !$isBase && !length($snapname);
- my $name = &$find_free_diskname($storeid, $scfg, $vmid);
+ my $name = $find_free_diskname->($storeid, $scfg, $vmid);
warn "clone $volname: $basename snapname $snap to $name\n";
run_rbd_command($cmd, errmsg => "rbd clone '$basename' error");
- &$krbd_feature_disable($scfg, $storeid, $name);
-
return $newvol;
}
die "illegal name '$name' - should be 'vm-$vmid-*'\n"
if $name && $name !~ m/^vm-$vmid-/;
- $name = &$find_free_diskname($storeid, $scfg, $vmid) if !$name;
+ $name = $find_free_diskname->($storeid, $scfg, $vmid) if !$name;
my $cmd = &$rbd_cmd($scfg, $storeid, 'create', '--image-format' , 2, '--size', int(($size+1023)/1024), $name);
run_rbd_command($cmd, errmsg => "rbd create $name' error");
- &$krbd_feature_disable($scfg, $storeid, $name);
-
return $name;
}
# max_avail -> max available space for data w/o replication in the pool
# bytes_used -> data w/o replication in the pool
my $free = $d->{stats}->{max_avail};
- my $used = $d->{stats}->{bytes_used};
+ my $used = $d->{stats}->{stored} // $d->{stats}->{bytes_used};
my $total = $used + $free;
my $active = 1;
return 1;
}
-sub activate_volume {
- my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
+my $get_kernel_device_name = sub {
+ my ($pool, $name) = @_;
- return 1 if !$scfg->{krbd};
+ return "/dev/rbd/$pool/$name";
+};
+
+sub map_volume {
+ my ($class, $storeid, $scfg, $volname, $snapname) = @_;
my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+ $name .= '@'.$snapname if $snapname;
+
my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
- my $path = "/dev/rbd/$pool/$name";
- $path .= '@'.$snapname if $snapname;
- return if -b $path;
+ my $kerneldev = $get_kernel_device_name->($pool, $name);
+
+ return $kerneldev if -b $kerneldev; # already mapped
+
+ &$krbd_feature_disable($scfg, $storeid, $name);
- $name .= '@'.$snapname if $snapname;
my $cmd = &$rbd_cmd($scfg, $storeid, 'map', $name);
- run_rbd_command($cmd, errmsg => "can't mount rbd volume $name");
+ run_rbd_command($cmd, errmsg => "can't map rbd volume $name");
+
+ return $kerneldev;
+}
+
+sub unmap_volume {
+ my ($class, $storeid, $scfg, $volname, $snapname) = @_;
+
+ my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+ $name .= '@'.$snapname if $snapname;
+
+ my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
+
+ my $kerneldev = $get_kernel_device_name->($pool, $name);
+
+ if (-b $kerneldev) {
+ my $cmd = &$rbd_cmd($scfg, $storeid, 'unmap', $kerneldev);
+ run_rbd_command($cmd, errmsg => "can't unmap rbd device $kerneldev");
+ }
return 1;
}
-sub deactivate_volume {
+sub activate_volume {
my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
- return 1 if !$scfg->{krbd};
+ $class->map_volume($storeid, $scfg, $volname, $snapname) if $scfg->{krbd};
- my ($vtype, $name, $vmid) = $class->parse_volname($volname);
- my $pool = $scfg->{pool} ? $scfg->{pool} : 'rbd';
+ return 1;
+}
- my $path = "/dev/rbd/$pool/$name";
- $path .= '@'.$snapname if $snapname;
- return if ! -b $path;
+sub deactivate_volume {
+ my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
- my $cmd = &$rbd_cmd($scfg, $storeid, 'unmap', $path);
- run_rbd_command($cmd, errmsg => "can't unmap rbd volume $name");
+ $class->unmap_volume($storeid, $scfg, $volname, $snapname);
return 1;
}
sub volume_resize {
my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
- return 1 if $running && !$scfg->{krbd};
+ return 1 if $running && !$scfg->{krbd}; # FIXME???
my ($vtype, $name, $vmid) = $class->parse_volname($volname);
sub volume_snapshot_delete {
my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
- return 1 if $running && !$scfg->{krbd};
+ return 1 if $running && !$scfg->{krbd}; # FIXME: ????
$class->deactivate_volume($storeid, $scfg, $volname, $snap, {});
projects / pve-storage.git / blobdiff