PVE/Ceph/Tools.pm

   1 package PVE::Ceph::Tools;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use File::Basename;
   8 use IO::File;
   9 use JSON;
  10
  11 use PVE::Tools qw(run_command dir_glob_foreach);
  12 use PVE::Cluster qw(cfs_read_file);
  13 use PVE::RADOS;
  14
  15 my $ccname = 'ceph'; # ceph cluster name
  16 my $ceph_cfgdir = "/etc/ceph";
  17 my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf";
  18 my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf";
  19
  20 my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
  21 my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring";
  22 my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
  23 my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";
  24 my $ceph_mds_data_dir = '/var/lib/ceph/mds';
  25
  26 my $ceph_service = {
  27     ceph_bin => "/usr/bin/ceph",
  28     ceph_mon => "/usr/bin/ceph-mon",
  29     ceph_mgr => "/usr/bin/ceph-mgr",
  30     ceph_osd => "/usr/bin/ceph-osd",
  31     ceph_mds => "/usr/bin/ceph-mds",
  32     ceph_volume => '/usr/sbin/ceph-volume',
  33 };
  34
  35 my $config_hash = {
  36     ccname => $ccname,
  37     pve_ceph_cfgpath => $pve_ceph_cfgpath,
  38     pve_mon_key_path => $pve_mon_key_path,
  39     pve_ckeyring_path => $pve_ckeyring_path,
  40     ceph_bootstrap_osd_keyring => $ceph_bootstrap_osd_keyring,
  41     ceph_bootstrap_mds_keyring => $ceph_bootstrap_mds_keyring,
  42     ceph_mds_data_dir => $ceph_mds_data_dir,
  43     long_rados_timeout => 60,
  44 };
  45
  46 sub get_local_version {
  47     my ($noerr) = @_;
  48
  49     if (check_ceph_installed('ceph_bin', $noerr)) {
  50         my $ceph_version;
  51         run_command([$ceph_service->{ceph_bin}, '--version'],
  52                     noerr => $noerr,
  53                     outfunc => sub { $ceph_version = shift; });
  54         if ($ceph_version && $ceph_version =~ /^ceph.*\s((\d+)\.(\d+)\.(\d+))/) {
  55             # return (version, major, minor, patch) : major;
  56             return wantarray ? ($1, $2, $3, $4) : $2;
  57         }
  58     }
  59
  60     return undef;
  61 }
  62
  63 sub get_config {
  64     my $key = shift;
  65
  66     my $value = $config_hash->{$key};
  67
  68     die "no such ceph config '$key'" if !$value;
  69
  70     return $value;
  71 }
  72
  73 sub purge_all_ceph_files {
  74     # fixme: this is very dangerous - should we really support this function?
  75
  76     unlink $ceph_cfgpath;
  77
  78     unlink $pve_ceph_cfgpath;
  79     unlink $pve_ckeyring_path;
  80     unlink $pve_mon_key_path;
  81
  82     unlink $ceph_bootstrap_osd_keyring;
  83     unlink $ceph_bootstrap_mds_keyring;
  84
  85     system("rm -rf /var/lib/ceph/mon/ceph-*");
  86
  87     # remove osd?
  88 }
  89
  90 sub check_ceph_installed {
  91     my ($service, $noerr) = @_;
  92
  93     $service = 'ceph_bin' if !defined($service);
  94
  95     if (! -x $ceph_service->{$service}) {
  96         die "binary not installed: $ceph_service->{$service}\n" if !$noerr;
  97         return undef;
  98     }
  99
 100     return 1;
 101 }
 102
 103 sub check_ceph_inited {
 104     my ($noerr) = @_;
 105
 106     return undef if !check_ceph_installed('ceph_mon', $noerr);
 107
 108     if (! -f $pve_ceph_cfgpath) {
 109         die "pveceph configuration not initialized\n" if !$noerr;
 110         return undef;
 111     }
 112
 113     return 1;
 114 }
 115
 116 sub check_ceph_enabled {
 117     my ($noerr) = @_;
 118
 119     return undef if !check_ceph_inited($noerr);
 120
 121     if (! -f $ceph_cfgpath) {
 122         die "pveceph configuration not enabled\n" if !$noerr;
 123         return undef;
 124     }
 125
 126     return 1;
 127 }
 128
 129 sub create_pool {
 130     my ($pool, $param, $rados) = @_;
 131
 132     if (!defined($rados)) {
 133         $rados = PVE::RADOS->new();
 134     }
 135
 136     my $pg_num = $param->{pg_num} || 128;
 137     my $size = $param->{size} || 3;
 138     my $min_size = $param->{min_size} || 2;
 139     my $application = $param->{application} // 'rbd';
 140
 141     $rados->mon_command({
 142         prefix => "osd pool create",
 143         pool => $pool,
 144         pg_num => int($pg_num),
 145         format => 'plain',
 146     });
 147
 148     $rados->mon_command({
 149         prefix => "osd pool set",
 150         pool => $pool,
 151         var => 'min_size',
 152         val => "$min_size",
 153         format => 'plain',
 154     });
 155
 156     $rados->mon_command({
 157         prefix => "osd pool set",
 158         pool => $pool,
 159         var => 'size',
 160         val => "$size",
 161         format => 'plain',
 162     });
 163
 164     if (defined($param->{crush_rule})) {
 165         $rados->mon_command({
 166             prefix => "osd pool set",
 167             pool => $pool,
 168             var => 'crush_rule',
 169             val => $param->{crush_rule},
 170             format => 'plain',
 171         });
 172     }
 173
 174     $rados->mon_command({
 175         prefix => "osd pool application enable",
 176         pool => $pool,
 177         app => $application,
 178     });
 179
 180 }
 181
 182 sub ls_pools {
 183     my ($pool, $rados) = @_;
 184
 185     if (!defined($rados)) {
 186         $rados = PVE::RADOS->new();
 187     }
 188
 189     my $res = $rados->mon_command({ prefix => "osd lspools" });
 190
 191     return $res;
 192 }
 193
 194 sub destroy_pool {
 195     my ($pool, $rados) = @_;
 196
 197     if (!defined($rados)) {
 198         $rados = PVE::RADOS->new();
 199     }
 200
 201     # fixme: '--yes-i-really-really-mean-it'
 202     $rados->mon_command({
 203         prefix => "osd pool delete",
 204         pool => $pool,
 205         pool2 => $pool,
 206         sure => '--yes-i-really-really-mean-it',
 207         format => 'plain',
 208     });
 209 }
 210
 211 sub setup_pve_symlinks {
 212     # fail if we find a real file instead of a link
 213     if (-f $ceph_cfgpath) {
 214         my $lnk = readlink($ceph_cfgpath);
 215         die "file '$ceph_cfgpath' already exists\n"
 216             if !$lnk || $lnk ne $pve_ceph_cfgpath;
 217     } else {
 218         symlink($pve_ceph_cfgpath, $ceph_cfgpath) ||
 219             die "unable to create symlink '$ceph_cfgpath' - $!\n";
 220     }
 221 }
 222
 223 # wipe the first 200 MB to clear off leftovers from previous use, otherwise a
 224 # create OSD fails.
 225 sub wipe_disks {
 226     my (@devs) = @_;
 227
 228     my @wipe_cmd = qw(/bin/dd if=/dev/zero bs=1M conv=fdatasync);
 229
 230     foreach my $devpath (@devs) {
 231         my $devname = basename($devpath);
 232         my $dev_size = PVE::Tools::file_get_contents("/sys/class/block/$devname/size");
 233
 234         ($dev_size) = $dev_size =~ m|(\d+)|; # untaint $dev_size
 235         die "Coulnd't get the size of the device $devname\n" if (!defined($dev_size));
 236
 237         my $size = ($dev_size * 512 / 1024 / 1024);
 238         my $count = ($size < 200) ? $size : 200;
 239
 240         print "wipe disk/partition: $devpath\n";
 241         eval { run_command([@wipe_cmd, "count=$count", "of=${devpath}"]) };
 242         warn $@ if $@;
 243     }
 244 };
 245
 246 # get ceph-volume managed osds
 247 sub ceph_volume_list {
 248     my $result = {};
 249     my $output = '';
 250
 251     if (!check_ceph_installed('ceph_volume', 1)) {
 252         return $result;
 253     }
 254
 255     my $cmd = [$ceph_service->{ceph_volume}, 'lvm', 'list', '--format', 'json'];
 256     run_command($cmd, outfunc => sub {
 257         $output .= shift;
 258     });
 259
 260     $result = eval { decode_json($output) };
 261     warn $@ if $@;
 262     return $result;
 263 }
 264
 265 sub ceph_volume_zap {
 266     my ($osdid, $destroy) = @_;
 267
 268     die "no osdid given\n" if !defined($osdid);
 269
 270     my $cmd = [$ceph_service->{ceph_volume}, 'lvm', 'zap', '--osd-id', $osdid];
 271     push @$cmd, '--destroy' if $destroy;
 272
 273     run_command($cmd);
 274 }
 275
 276 1;