PVE/CephTools.pm

   1 package PVE::CephTools;
   2
   3 use strict;
   4 use warnings;
   5
   6 use File::Path;
   7 use IO::File;
   8
   9 use PVE::Tools qw(run_command dir_glob_foreach);
  10 use PVE::RADOS;
  11
  12 my $ccname = 'ceph'; # ceph cluster name
  13 my $ceph_cfgdir = "/etc/ceph";
  14 my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf";
  15 my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf";
  16
  17 my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
  18 my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring";
  19 my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
  20 my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";
  21
  22 my $ceph_service = {
  23     ceph_bin => "/usr/bin/ceph",
  24     ceph_mon => "/usr/bin/ceph-mon",
  25     ceph_mgr => "/usr/bin/ceph-mgr",
  26     ceph_osd => "/usr/bin/ceph-osd"
  27 };
  28
  29 my $config_hash = {
  30     ccname => $ccname,
  31     pve_ceph_cfgpath => $pve_ceph_cfgpath,
  32     pve_mon_key_path => $pve_mon_key_path,
  33     pve_ckeyring_path => $pve_ckeyring_path,
  34     ceph_bootstrap_osd_keyring => $ceph_bootstrap_osd_keyring,
  35     ceph_bootstrap_mds_keyring => $ceph_bootstrap_mds_keyring,
  36     long_rados_timeout => 60,
  37 };
  38
  39 sub get_local_version {
  40     my ($noerr) = @_;
  41
  42     if (PVE::CephTools::check_ceph_installed('ceph_bin', $noerr)) {
  43         my $ceph_version;
  44         run_command([$ceph_service->{ceph_bin}, '--version'],
  45                     noerr => $noerr,
  46                     outfunc => sub { $ceph_version = shift; });
  47         if ($ceph_version && $ceph_version =~ /^ceph.*\s((\d+)\.(\d+)\.(\d+))/) {
  48             # return (version, major, minor, patch) : major;
  49             return wantarray ? ($1, $2, $3, $4) : $2;
  50         }
  51     }
  52
  53     return undef;
  54 }
  55
  56 sub get_config {
  57     my $key = shift;
  58
  59     my $value = $config_hash->{$key};
  60
  61     die "no such ceph config '$key'" if !$value;
  62
  63     return $value;
  64 }
  65
  66 sub purge_all_ceph_files {
  67     # fixme: this is very dangerous - should we really support this function?
  68
  69     unlink $ceph_cfgpath;
  70
  71     unlink $pve_ceph_cfgpath;
  72     unlink $pve_ckeyring_path;
  73     unlink $pve_mon_key_path;
  74
  75     unlink $ceph_bootstrap_osd_keyring;
  76     unlink $ceph_bootstrap_mds_keyring;
  77
  78     system("rm -rf /var/lib/ceph/mon/ceph-*");
  79
  80     # remove osd?
  81 }
  82
  83 sub check_ceph_installed {
  84     my ($service, $noerr) = @_;
  85
  86     $service = 'ceph_bin' if !defined($service);
  87
  88     if (! -x $ceph_service->{$service}) {
  89         die "binary not installed: $ceph_service->{$service}\n" if !$noerr;
  90         return undef;
  91     }
  92
  93     return 1;
  94 }
  95
  96 sub check_ceph_inited {
  97     my ($noerr) = @_;
  98
  99     return undef if !check_ceph_installed('ceph_bin', $noerr);
 100
 101     if (! -f $pve_ceph_cfgpath) {
 102         die "pveceph configuration not initialized\n" if !$noerr;
 103         return undef;
 104     }
 105
 106     return 1;
 107 }
 108
 109 sub check_ceph_enabled {
 110     my ($noerr) = @_;
 111
 112     return undef if !check_ceph_inited($noerr);
 113
 114     if (! -f $ceph_cfgpath) {
 115         die "pveceph configuration not enabled\n" if !$noerr;
 116         return undef;
 117     }
 118
 119     return 1;
 120 }
 121
 122 sub parse_ceph_config {
 123     my ($filename) = @_;
 124
 125     $filename = $pve_ceph_cfgpath if !$filename;
 126
 127     my $cfg = {};
 128
 129     return $cfg if ! -f $filename;
 130
 131     my $fh = IO::File->new($filename, "r") ||
 132         die "unable to open '$filename' - $!\n";
 133
 134     my $section;
 135
 136     while (defined(my $line = <$fh>)) {
 137         $line =~ s/[;#].*$//;
 138         $line =~ s/^\s+//;
 139         $line =~ s/\s+$//;
 140         next if !$line;
 141
 142         $section = $1 if $line =~ m/^\[(\S+)\]$/;
 143         if (!$section) {
 144             warn "no section - skip: $line\n";
 145             next;
 146         }
 147
 148         if ($line =~ m/^(.*?\S)\s*=\s*(\S.*)$/) {
 149             $cfg->{$section}->{$1} = $2;
 150         }
 151
 152     }
 153
 154     return $cfg;
 155 }
 156
 157 sub write_ceph_config {
 158     my ($cfg) = @_;
 159
 160     my $out = '';
 161
 162     my $cond_write_sec = sub {
 163         my $re = shift;
 164
 165         foreach my $section (keys %$cfg) {
 166             next if $section !~ m/^$re$/;
 167             $out .= "[$section]\n";
 168             foreach my $key (sort keys %{$cfg->{$section}}) {
 169                 $out .= "\t $key = $cfg->{$section}->{$key}\n";
 170             }
 171             $out .= "\n";
 172         }
 173     };
 174
 175     &$cond_write_sec('global');
 176     &$cond_write_sec('client');
 177     &$cond_write_sec('mds');
 178     &$cond_write_sec('mds\..*');
 179     &$cond_write_sec('mon');
 180     &$cond_write_sec('osd');
 181     &$cond_write_sec('mon\..*');
 182     &$cond_write_sec('osd\..*');
 183
 184     PVE::Tools::file_set_contents($pve_ceph_cfgpath, $out);
 185 }
 186
 187 sub create_pool {
 188     my ($pool, $param, $rados) = @_;
 189
 190     if (!defined($rados)) {
 191         $rados = PVE::RADOS->new();
 192     }
 193
 194     my $pg_num = $param->{pg_num} || 64;
 195     my $size = $param->{size} || 3;
 196     my $min_size = $param->{min_size} || 2;
 197     my $application = $param->{application} // 'rbd';
 198
 199     $rados->mon_command({
 200         prefix => "osd pool create",
 201         pool => $pool,
 202         pg_num => int($pg_num),
 203         format => 'plain',
 204     });
 205
 206     $rados->mon_command({
 207         prefix => "osd pool set",
 208         pool => $pool,
 209         var => 'min_size',
 210         val => $min_size,
 211         format => 'plain',
 212     });
 213
 214     $rados->mon_command({
 215         prefix => "osd pool set",
 216         pool => $pool,
 217         var => 'size',
 218         val => $size,
 219         format => 'plain',
 220     });
 221
 222     if (defined($param->{crush_rule})) {
 223         $rados->mon_command({
 224             prefix => "osd pool set",
 225             pool => $pool,
 226             var => 'crush_rule',
 227             val => $param->{crush_rule},
 228             format => 'plain',
 229         });
 230     }
 231
 232     $rados->mon_command({
 233         prefix => "osd pool application enable",
 234         pool => $pool,
 235         app => $application,
 236     });
 237
 238 }
 239
 240 sub destroy_pool {
 241     my ($pool, $rados) = @_;
 242
 243     if (!defined($rados)) {
 244         $rados = PVE::RADOS->new();
 245     }
 246
 247     # fixme: '--yes-i-really-really-mean-it'
 248     $rados->mon_command({
 249         prefix => "osd pool delete",
 250         pool => $pool,
 251         pool2 => $pool,
 252         sure => '--yes-i-really-really-mean-it',
 253         format => 'plain',
 254     });
 255 }
 256
 257 sub setup_pve_symlinks {
 258     # fail if we find a real file instead of a link
 259     if (-f $ceph_cfgpath) {
 260         my $lnk = readlink($ceph_cfgpath);
 261         die "file '$ceph_cfgpath' already exists\n"
 262             if !$lnk || $lnk ne $pve_ceph_cfgpath;
 263     } else {
 264         symlink($pve_ceph_cfgpath, $ceph_cfgpath) ||
 265             die "unable to create symlink '$ceph_cfgpath' - $!\n";
 266     }
 267 }
 268
 269 sub ceph_service_cmd {
 270     my ($action, $service) = @_;
 271
 272     if (systemd_managed()) {
 273
 274         if ($service && $service =~ m/^(mon|osd|mds|mgr|radosgw)(\.([A-Za-z0-9\-]{1,32}))?$/) {
 275             $service = defined($3) ? "ceph-$1\@$3" : "ceph-$1.target";
 276         } else {
 277             $service = "ceph.target";
 278         }
 279
 280         PVE::Tools::run_command(['/bin/systemctl', $action, $service]);
 281
 282     } else {
 283         # ceph daemons does not call 'setsid', so we do that ourself
 284         # (fork_worker send KILL to whole process group)
 285         PVE::Tools::run_command(['setsid', 'service', 'ceph', '-c', $pve_ceph_cfgpath, $action, $service]);
 286     }
 287 }
 288
 289 # Ceph versions greater Hammer use 'ceph' as user and group instead
 290 # of 'root', and use systemd.
 291 sub systemd_managed {
 292
 293     if (-f "/lib/systemd/system/ceph-osd\@.service") {
 294         return 1;
 295     } else {
 296         return 0;
 297     }
 298 }
 299
 300 1;