use IO::File;
use JSON;
-use PVE::Tools qw(run_command dir_glob_foreach);
+use PVE::Tools qw(run_command dir_glob_foreach extract_param);
use PVE::Cluster qw(cfs_read_file);
use PVE::RADOS;
+use PVE::Ceph::Services;
+use PVE::CephConfig;
my $ccname = 'ceph'; # ceph cluster name
my $ceph_cfgdir = "/etc/ceph";
ceph_bootstrap_mds_keyring => $ceph_bootstrap_mds_keyring,
ceph_mds_data_dir => $ceph_mds_data_dir,
long_rados_timeout => 60,
+ ceph_cfgpath => $ceph_cfgpath,
};
sub get_local_version {
if (check_ceph_installed('ceph_bin', $noerr)) {
my $ceph_version;
- run_command([$ceph_service->{ceph_bin}, '--version'],
- noerr => $noerr,
- outfunc => sub { $ceph_version = shift; });
- if ($ceph_version && $ceph_version =~ /^ceph.*\s((\d+)\.(\d+)\.(\d+))/) {
- # return (version, major, minor, patch) : major;
- return wantarray ? ($1, $2, $3, $4) : $2;
+ run_command(
+ [ $ceph_service->{ceph_bin}, '--version' ],
+ noerr => $noerr,
+ outfunc => sub { $ceph_version = shift if !defined $ceph_version },
+ );
+ return undef if !defined $ceph_version;
+
+ if ($ceph_version =~ /^ceph.*\sv?(\d+(?:\.\d+)+(?:-pve\d+)?)\s+(?:\(([a-zA-Z0-9]+)\))?/) {
+ my ($version, $buildcommit) = ($1, $2);
+ my $subversions = [ split(/\.|-/, $version) ];
+
+ # return (version, buildid, major, minor, ...) : major;
+ return wantarray
+ ? ($version, $buildcommit, $subversions)
+ : $subversions->[0];
}
}
}
sub purge_all_ceph_files {
- # fixme: this is very dangerous - should we really support this function?
+ my ($services) = @_;
+ my $is_local_mon;
+ my $monlist = [ split(',', PVE::CephConfig::get_monaddr_list($pve_ceph_cfgpath)) ];
+
+ foreach my $service (keys %$services) {
+ my $type = $services->{$service};
+ next if (!%$type);
+
+ foreach my $name (keys %$type) {
+ my $dir_exists = $type->{$name}->{direxists};
+
+ $is_local_mon = grep($type->{$name}->{addr}, @$monlist)
+ if $service eq 'mon';
+
+ my $path = "/var/lib/ceph/$service";
+ $path = '/var/log/ceph' if $service eq 'logs';
+ if ($dir_exists) {
+ my $err;
+ File::Path::remove_tree($path, {
+ keep_root => 1,
+ error => \$err,
+ });
+ warn "Error removing path, '$path'\n" if @$err;
+ }
+ }
+ }
+
+ if (scalar @$monlist > 0 && !$is_local_mon) {
+ warn "Foreign MON address in ceph.conf. Keeping config & keyrings\n"
+ } else {
+ print "Removing config & keyring files\n";
+ foreach my $file (%$config_hash) {
+ unlink $file if (-e $file);
+ }
+ }
+}
- unlink $ceph_cfgpath;
+sub purge_all_ceph_services {
+ my ($services) = @_;
- unlink $pve_ceph_cfgpath;
- unlink $pve_ckeyring_path;
- unlink $pve_mon_key_path;
+ foreach my $service (keys %$services) {
+ my $type = $services->{$service};
+ next if (!%$type);
- unlink $ceph_bootstrap_osd_keyring;
- unlink $ceph_bootstrap_mds_keyring;
+ foreach my $name (keys %$type) {
+ my $service_exists = $type->{$name}->{service};
- system("rm -rf /var/lib/ceph/mon/ceph-*");
+ if ($service_exists) {
+ eval { PVE::Ceph::Services::ceph_service_cmd('disable', "$service.$name") };
+ warn "Could not disable ceph-$service\@$name, error: $@\n" if $@;
- # remove osd?
+ eval { PVE::Ceph::Services::ceph_service_cmd('stop', "$service.$name") };
+ warn "Could not stop ceph-$service\@$name, error: $@\n" if $@;
+ }
+ }
+ }
}
+sub ceph_install_flag_file { return '/run/pve-ceph-install-flag' };
+
sub check_ceph_installed {
my ($service, $noerr) = @_;
$service = 'ceph_bin' if !defined($service);
- if (! -x $ceph_service->{$service}) {
+ # NOTE: the flag file is checked as on a new installation, the binary gets
+ # extracted by dpkg before the installation is finished
+ if (! -x $ceph_service->{$service} || -f ceph_install_flag_file()) {
die "binary not installed: $ceph_service->{$service}\n" if !$noerr;
return undef;
}
return 1;
}
-sub create_pool {
- my ($pool, $param, $rados) = @_;
+my $set_pool_setting = sub {
+ my ($pool, $setting, $value) = @_;
- if (!defined($rados)) {
- $rados = PVE::RADOS->new();
+ my $command;
+ if ($setting eq 'application') {
+ $command = {
+ prefix => "osd pool application enable",
+ pool => "$pool",
+ app => "$value",
+ };
+ } else {
+ $command = {
+ prefix => "osd pool set",
+ pool => "$pool",
+ var => "$setting",
+ val => "$value",
+ format => 'plain',
+ };
+ }
+
+ my $rados = PVE::RADOS->new();
+ eval { $rados->mon_command($command); };
+ return $@ ? $@ : undef;
+};
+
+sub set_pool {
+ my ($pool, $param) = @_;
+
+ # by default, pool size always resets min_size, so set it as first item
+ # https://tracker.ceph.com/issues/44862
+ my $keys = [ grep { $_ ne 'size' } sort keys %$param ];
+ unshift @$keys, 'size' if exists $param->{size};
+
+ for my $setting (@$keys) {
+ my $value = $param->{$setting};
+
+ print "pool $pool: applying $setting = $value\n";
+ if (my $err = $set_pool_setting->($pool, $setting, $value)) {
+ print "$err";
+ } else {
+ delete $param->{$setting};
+ }
+ }
+
+ if (scalar(keys %$param) > 0) {
+ my $missing = join(', ', sort keys %$param );
+ die "Could not set: $missing\n";
}
+}
+
+sub get_pool_properties {
+ my ($pool) = @_;
+ my $command = {
+ prefix => "osd pool get",
+ pool => "$pool",
+ var => "all",
+ format => 'json',
+ };
+
+ my $rados = PVE::RADOS->new();
+ return $rados->mon_command($command);
+}
+
+sub create_pool {
+ my ($pool, $param, $rados) = @_;
+ $rados = PVE::RADOS->new() if !defined($rados);
+
my $pg_num = $param->{pg_num} || 128;
- my $size = $param->{size} || 3;
- my $min_size = $param->{min_size} || 2;
- my $application = $param->{application} // 'rbd';
- $rados->mon_command({
+ my $mon_params = {
prefix => "osd pool create",
pool => $pool,
pg_num => int($pg_num),
format => 'plain',
- });
-
- $rados->mon_command({
- prefix => "osd pool set",
- pool => $pool,
- var => 'min_size',
- val => "$min_size",
- format => 'plain',
- });
-
- $rados->mon_command({
- prefix => "osd pool set",
- pool => $pool,
- var => 'size',
- val => "$size",
- format => 'plain',
- });
+ };
+ $mon_params->{pool_type} = extract_param($param, 'pool_type') if $param->{pool_type};
+ $mon_params->{erasure_code_profile} = extract_param($param, 'erasure_code_profile')
+ if $param->{erasure_code_profile};
- if (defined($param->{crush_rule})) {
- $rados->mon_command({
- prefix => "osd pool set",
- pool => $pool,
- var => 'crush_rule',
- val => $param->{crush_rule},
- format => 'plain',
- });
- }
+ $rados->mon_command($mon_params);
- $rados->mon_command({
- prefix => "osd pool application enable",
- pool => $pool,
- app => $application,
- });
+ set_pool($pool, $param);
}
sub ls_pools {
my ($pool, $rados) = @_;
-
- if (!defined($rados)) {
- $rados = PVE::RADOS->new();
- }
+ $rados = PVE::RADOS->new() if !defined($rados);
my $res = $rados->mon_command({ prefix => "osd lspools" });
sub destroy_pool {
my ($pool, $rados) = @_;
-
- if (!defined($rados)) {
- $rados = PVE::RADOS->new();
- }
+ $rados = PVE::RADOS->new() if !defined($rados);
# fixme: '--yes-i-really-really-mean-it'
$rados->mon_command({
});
}
+# we get something like:
+#[{
+# 'metadata_pool_id' => 2,
+# 'data_pool_ids' => [ 1 ],
+# 'metadata_pool' => 'cephfs_metadata',
+# 'data_pools' => [ 'cephfs_data' ],
+# 'name' => 'cephfs',
+#}]
+sub ls_fs {
+ my ($rados) = @_;
+ $rados = PVE::RADOS->new() if !defined($rados);
+
+ my $res = $rados->mon_command({ prefix => "fs ls" });
+
+ return $res;
+}
+
+sub create_fs {
+ my ($fs, $param, $rados) = @_;
+
+ if (!defined($rados)) {
+ $rados = PVE::RADOS->new();
+ }
+
+ $rados->mon_command({
+ prefix => "fs new",
+ fs_name => $fs,
+ metadata => $param->{pool_metadata},
+ data => $param->{pool_data},
+ format => 'plain',
+ });
+}
+
+sub destroy_fs {
+ my ($fs, $rados) = @_;
+ $rados = PVE::RADOS->new() if !defined($rados);
+
+ $rados->mon_command({
+ prefix => "fs rm",
+ fs_name => $fs,
+ 'yes_i_really_mean_it' => JSON::true,
+ format => 'plain',
+ });
+}
+
sub setup_pve_symlinks {
# fail if we find a real file instead of a link
if (-f $ceph_cfgpath) {
my $lnk = readlink($ceph_cfgpath);
- die "file '$ceph_cfgpath' already exists\n"
+ die "file '$ceph_cfgpath' already exists and is not a symlink to $pve_ceph_cfgpath\n"
if !$lnk || $lnk ne $pve_ceph_cfgpath;
} else {
mkdir $ceph_cfgdir;
return $pve_ckeyring_path;
}
-# wipe the first 200 MB to clear off leftovers from previous use, otherwise a
-# create OSD fails.
-sub wipe_disks {
- my (@devs) = @_;
-
- my @wipe_cmd = qw(/bin/dd if=/dev/zero bs=1M conv=fdatasync);
-
- foreach my $devpath (@devs) {
- my $devname = basename($devpath);
- my $dev_size = PVE::Tools::file_get_contents("/sys/class/block/$devname/size");
-
- ($dev_size) = $dev_size =~ m|(\d+)|; # untaint $dev_size
- die "Coulnd't get the size of the device $devname\n" if (!defined($dev_size));
-
- my $size = ($dev_size * 512 / 1024 / 1024);
- my $count = ($size < 200) ? $size : 200;
-
- print "wipe disk/partition: $devpath\n";
- eval { run_command([@wipe_cmd, "count=$count", "of=${devpath}"]) };
- warn $@ if $@;
- }
-};
-
# get ceph-volume managed osds
sub ceph_volume_list {
my $result = {};
return $flagmap->{$flag} // $flag;
}
+sub ceph_cluster_status {
+ my ($rados) = @_;
+ $rados = PVE::RADOS->new() if !$rados;
+
+ my $status = $rados->mon_command({ prefix => 'status' });
+ $status->{health} = $rados->mon_command({ prefix => 'health', detail => 'detail' });
+
+ if (!exists $status->{monmap}->{mons}) { # octopus moved most info out of status, re-add
+ $status->{monmap} = $rados->mon_command({ prefix => 'mon dump' });
+ $status->{mgrmap} = $rados->mon_command({ prefix => 'mgr dump' });
+ }
+
+ return $status;
+}
+
+sub ecprofile_exists {
+ my ($name) = @_;
+
+ my $rados = PVE::RADOS->new();
+ my $res = $rados->mon_command({ prefix => 'osd erasure-code-profile ls' });
+
+ my $profiles = { map { $_ => 1 } @$res };
+ return $profiles->{$name};
+}
+
+sub create_ecprofile {
+ my ($name, $k, $m, $failure_domain, $device_class) = @_;
+
+ $failure_domain = 'host' if !$failure_domain;
+
+ my $profile = [
+ "crush-failure-domain=${failure_domain}",
+ "k=${k}",
+ "m=${m}",
+ ];
+
+ push(@$profile, "crush-device-class=${device_class}") if $device_class;
+
+ my $rados = PVE::RADOS->new();
+ $rados->mon_command({
+ prefix => 'osd erasure-code-profile set',
+ name => $name,
+ profile => $profile,
+ });
+}
+
+sub destroy_ecprofile {
+ my ($profile) = @_;
+
+ my $rados = PVE::RADOS->new();
+ my $command = {
+ prefix => 'osd erasure-code-profile rm',
+ name => $profile,
+ format => 'plain',
+ };
+ return $rados->mon_command($command);
+}
+
+sub get_ecprofile_name {
+ my ($name) = @_;
+ return "pve_ec_${name}";
+}
+
+sub destroy_crush_rule {
+ my ($rule) = @_;
+ my $rados = PVE::RADOS->new();
+ my $command = {
+ prefix => 'osd crush rule rm',
+ name => $rule,
+ format => 'plain',
+ };
+ return $rados->mon_command($command);
+}
+
1;
projects / pve-manager.git / blobdiff