PVE/Ceph/Tools.pm

   1 package PVE::Ceph::Tools;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use File::Basename;
   8 use IO::File;
   9
  10 use PVE::Tools qw(run_command dir_glob_foreach);
  11 use PVE::RADOS;
  12
  13 my $ccname = 'ceph'; # ceph cluster name
  14 my $ceph_cfgdir = "/etc/ceph";
  15 my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf";
  16 my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf";
  17
  18 my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
  19 my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring";
  20 my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
  21 my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";
  22 my $ceph_mds_data_dir = '/var/lib/ceph/mds';
  23
  24 my $ceph_service = {
  25     ceph_bin => "/usr/bin/ceph",
  26     ceph_mon => "/usr/bin/ceph-mon",
  27     ceph_mgr => "/usr/bin/ceph-mgr",
  28     ceph_osd => "/usr/bin/ceph-osd",
  29     ceph_mds => "/usr/bin/ceph-mds",
  30 };
  31
  32 my $config_hash = {
  33     ccname => $ccname,
  34     pve_ceph_cfgpath => $pve_ceph_cfgpath,
  35     pve_mon_key_path => $pve_mon_key_path,
  36     pve_ckeyring_path => $pve_ckeyring_path,
  37     ceph_bootstrap_osd_keyring => $ceph_bootstrap_osd_keyring,
  38     ceph_bootstrap_mds_keyring => $ceph_bootstrap_mds_keyring,
  39     ceph_mds_data_dir => $ceph_mds_data_dir,
  40     long_rados_timeout => 60,
  41 };
  42
  43 sub get_local_version {
  44     my ($noerr) = @_;
  45
  46     if (check_ceph_installed('ceph_bin', $noerr)) {
  47         my $ceph_version;
  48         run_command([$ceph_service->{ceph_bin}, '--version'],
  49                     noerr => $noerr,
  50                     outfunc => sub { $ceph_version = shift; });
  51         if ($ceph_version && $ceph_version =~ /^ceph.*\s((\d+)\.(\d+)\.(\d+))/) {
  52             # return (version, major, minor, patch) : major;
  53             return wantarray ? ($1, $2, $3, $4) : $2;
  54         }
  55     }
  56
  57     return undef;
  58 }
  59
  60 sub get_config {
  61     my $key = shift;
  62
  63     my $value = $config_hash->{$key};
  64
  65     die "no such ceph config '$key'" if !$value;
  66
  67     return $value;
  68 }
  69
  70 sub purge_all_ceph_files {
  71     # fixme: this is very dangerous - should we really support this function?
  72
  73     unlink $ceph_cfgpath;
  74
  75     unlink $pve_ceph_cfgpath;
  76     unlink $pve_ckeyring_path;
  77     unlink $pve_mon_key_path;
  78
  79     unlink $ceph_bootstrap_osd_keyring;
  80     unlink $ceph_bootstrap_mds_keyring;
  81
  82     system("rm -rf /var/lib/ceph/mon/ceph-*");
  83
  84     # remove osd?
  85 }
  86
  87 sub check_ceph_installed {
  88     my ($service, $noerr) = @_;
  89
  90     $service = 'ceph_bin' if !defined($service);
  91
  92     if (! -x $ceph_service->{$service}) {
  93         die "binary not installed: $ceph_service->{$service}\n" if !$noerr;
  94         return undef;
  95     }
  96
  97     return 1;
  98 }
  99
 100 sub check_ceph_inited {
 101     my ($noerr) = @_;
 102
 103     return undef if !check_ceph_installed('ceph_mon', $noerr);
 104
 105     if (! -f $pve_ceph_cfgpath) {
 106         die "pveceph configuration not initialized\n" if !$noerr;
 107         return undef;
 108     }
 109
 110     return 1;
 111 }
 112
 113 sub check_ceph_enabled {
 114     my ($noerr) = @_;
 115
 116     return undef if !check_ceph_inited($noerr);
 117
 118     if (! -f $ceph_cfgpath) {
 119         die "pveceph configuration not enabled\n" if !$noerr;
 120         return undef;
 121     }
 122
 123     return 1;
 124 }
 125
 126 sub create_pool {
 127     my ($pool, $param, $rados) = @_;
 128
 129     if (!defined($rados)) {
 130         $rados = PVE::RADOS->new();
 131     }
 132
 133     my $pg_num = $param->{pg_num} || 128;
 134     my $size = $param->{size} || 3;
 135     my $min_size = $param->{min_size} || 2;
 136     my $application = $param->{application} // 'rbd';
 137
 138     $rados->mon_command({
 139         prefix => "osd pool create",
 140         pool => $pool,
 141         pg_num => int($pg_num),
 142         format => 'plain',
 143     });
 144
 145     $rados->mon_command({
 146         prefix => "osd pool set",
 147         pool => $pool,
 148         var => 'min_size',
 149         val => $min_size,
 150         format => 'plain',
 151     });
 152
 153     $rados->mon_command({
 154         prefix => "osd pool set",
 155         pool => $pool,
 156         var => 'size',
 157         val => $size,
 158         format => 'plain',
 159     });
 160
 161     if (defined($param->{crush_rule})) {
 162         $rados->mon_command({
 163             prefix => "osd pool set",
 164             pool => $pool,
 165             var => 'crush_rule',
 166             val => $param->{crush_rule},
 167             format => 'plain',
 168         });
 169     }
 170
 171     $rados->mon_command({
 172         prefix => "osd pool application enable",
 173         pool => $pool,
 174         app => $application,
 175     });
 176
 177 }
 178
 179 sub ls_pools {
 180     my ($pool, $rados) = @_;
 181
 182     if (!defined($rados)) {
 183         $rados = PVE::RADOS->new();
 184     }
 185
 186     my $res = $rados->mon_command({ prefix => "osd lspools" });
 187
 188     return $res;
 189 }
 190
 191 sub destroy_pool {
 192     my ($pool, $rados) = @_;
 193
 194     if (!defined($rados)) {
 195         $rados = PVE::RADOS->new();
 196     }
 197
 198     # fixme: '--yes-i-really-really-mean-it'
 199     $rados->mon_command({
 200         prefix => "osd pool delete",
 201         pool => $pool,
 202         pool2 => $pool,
 203         sure => '--yes-i-really-really-mean-it',
 204         format => 'plain',
 205     });
 206 }
 207
 208 sub setup_pve_symlinks {
 209     # fail if we find a real file instead of a link
 210     if (-f $ceph_cfgpath) {
 211         my $lnk = readlink($ceph_cfgpath);
 212         die "file '$ceph_cfgpath' already exists\n"
 213             if !$lnk || $lnk ne $pve_ceph_cfgpath;
 214     } else {
 215         symlink($pve_ceph_cfgpath, $ceph_cfgpath) ||
 216             die "unable to create symlink '$ceph_cfgpath' - $!\n";
 217     }
 218 }
 219
 220 # Ceph versions greater Hammer use 'ceph' as user and group instead
 221 # of 'root', and use systemd.
 222 sub systemd_managed {
 223
 224     if (-f "/lib/systemd/system/ceph-osd\@.service") {
 225         return 1;
 226     } else {
 227         return 0;
 228     }
 229 }
 230
 231 # wipe the first 200 MB to clear off leftovers from previous use, otherwise a
 232 # create OSD fails.
 233 sub wipe_disks {
 234     my (@devs) = @_;
 235
 236     my @wipe_cmd = qw(/bin/dd if=/dev/zero bs=1M conv=fdatasync);
 237
 238     foreach my $devpath (@devs) {
 239         my $devname = basename($devpath);
 240         my $dev_size = PVE::Tools::file_get_contents("/sys/class/block/$devname/size");
 241
 242         ($dev_size) = $dev_size =~ m|(\d+)|; # untaint $dev_size
 243         die "Coulnd't get the size of the device $devname\n" if (!defined($dev_size));
 244
 245         my $size = ($dev_size * 512 / 1024 / 1024);
 246         my $count = ($size < 200) ? $size : 200;
 247
 248         print "wipe disk/partition: $devpath\n";
 249         eval { run_command([@wipe_cmd, "count=$count", "of=${devpath}"]) };
 250         warn $@ if $@;
 251     }
 252 };
 253
 254 1;