PVE/CephTools.pm

   1 package PVE::CephTools;
   2
   3 use strict;
   4 use warnings;
   5 use File::Basename;
   6 use File::Path;
   7 use POSIX qw (LONG_MAX);
   8 use Cwd qw(abs_path);
   9 use IO::Dir;
  10
  11 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
  12
  13 my $ccname = 'ceph'; # ceph cluster name
  14 my $ceph_cfgdir = "/etc/ceph";
  15 my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf";
  16 my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf";
  17
  18 my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
  19 my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring";
  20 my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
  21 my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";
  22
  23 my $ceph_bin = "/usr/bin/ceph";
  24
  25 my $config_hash = {
  26     ccname => $ccname,
  27     pve_ceph_cfgpath => $pve_ceph_cfgpath,
  28     pve_mon_key_path => $pve_mon_key_path,
  29     pve_ckeyring_path => $pve_ckeyring_path,
  30     ceph_bootstrap_osd_keyring => $ceph_bootstrap_osd_keyring,
  31     ceph_bootstrap_mds_keyring => $ceph_bootstrap_mds_keyring,
  32     long_rados_timeout => 60,
  33 };
  34
  35 sub get_config {
  36     my $key = shift;
  37
  38     my $value = $config_hash->{$key};
  39
  40     die "no such ceph config '$key'" if !$value;
  41
  42     return $value;
  43 }
  44
  45 sub verify_blockdev_path {
  46     my ($rel_path) = @_;
  47
  48     die "missing path" if !$rel_path;
  49     my $path = abs_path($rel_path);
  50     die "failed to get absolute path to $rel_path" if !$path;
  51
  52     die "got unusual device path '$path'\n" if $path !~  m|^/dev/(.*)$|;
  53
  54     $path = "/dev/$1"; # untaint
  55
  56     die "no such block device '$path'\n"
  57         if ! -b $path;
  58
  59     return $path;
  60 };
  61
  62 sub purge_all_ceph_files {
  63     # fixme: this is very dangerous - should we really support this function?
  64
  65     unlink $ceph_cfgpath;
  66
  67     unlink $pve_ceph_cfgpath;
  68     unlink $pve_ckeyring_path;
  69     unlink $pve_mon_key_path;
  70
  71     unlink $ceph_bootstrap_osd_keyring;
  72     unlink $ceph_bootstrap_mds_keyring;
  73
  74     system("rm -rf /var/lib/ceph/mon/ceph-*");
  75
  76     # remove osd?
  77 }
  78
  79 sub check_ceph_installed {
  80     my ($noerr) = @_;
  81
  82     if (! -x $ceph_bin) {
  83         die "ceph binaries not installed\n" if !$noerr;
  84         return undef;
  85     }
  86
  87     return 1;
  88 }
  89
  90 sub check_ceph_inited {
  91     my ($noerr) = @_;
  92
  93     return undef if !check_ceph_installed($noerr);
  94
  95     if (! -f $pve_ceph_cfgpath) {
  96         die "pveceph configuration not initialized\n" if !$noerr;
  97         return undef;
  98     }
  99
 100     return 1;
 101 }
 102
 103 sub check_ceph_enabled {
 104     my ($noerr) = @_;
 105
 106     return undef if !check_ceph_inited($noerr);
 107
 108     if (! -f $ceph_cfgpath) {
 109         die "pveceph configuration not enabled\n" if !$noerr;
 110         return undef;
 111     }
 112
 113     return 1;
 114 }
 115
 116 sub parse_ceph_config {
 117     my ($filename) = @_;
 118
 119     $filename = $pve_ceph_cfgpath if !$filename;
 120
 121     my $cfg = {};
 122
 123     return $cfg if ! -f $filename;
 124
 125     my $fh = IO::File->new($filename, "r") ||
 126         die "unable to open '$filename' - $!\n";
 127
 128     my $section;
 129
 130     while (defined(my $line = <$fh>)) {
 131         $line =~ s/[;#].*$//;
 132         $line =~ s/^\s+//;
 133         $line =~ s/\s+$//;
 134         next if !$line;
 135
 136         $section = $1 if $line =~ m/^\[(\S+)\]$/;
 137         if (!$section) {
 138             warn "no section - skip: $line\n";
 139             next;
 140         }
 141
 142         if ($line =~ m/^(.*?\S)\s*=\s*(\S.*)$/) {
 143             $cfg->{$section}->{$1} = $2;
 144         }
 145
 146     }
 147
 148     return $cfg;
 149 }
 150
 151 sub write_ceph_config {
 152     my ($cfg) = @_;
 153
 154     my $out = '';
 155
 156     my $cond_write_sec = sub {
 157         my $re = shift;
 158
 159         foreach my $section (keys %$cfg) {
 160             next if $section !~ m/^$re$/;
 161             $out .= "[$section]\n";
 162             foreach my $key (sort keys %{$cfg->{$section}}) {
 163                 $out .= "\t $key = $cfg->{$section}->{$key}\n";
 164             }
 165             $out .= "\n";
 166         }
 167     };
 168
 169     &$cond_write_sec('global');
 170     &$cond_write_sec('client');
 171     &$cond_write_sec('mds');
 172     &$cond_write_sec('mds\..*');
 173     &$cond_write_sec('mon');
 174     &$cond_write_sec('osd');
 175     &$cond_write_sec('mon\..*');
 176     &$cond_write_sec('osd\..*');
 177
 178     PVE::Tools::file_set_contents($pve_ceph_cfgpath, $out);
 179 }
 180
 181 sub setup_pve_symlinks {
 182     # fail if we find a real file instead of a link
 183     if (-f $ceph_cfgpath) {
 184         my $lnk = readlink($ceph_cfgpath);
 185         die "file '$ceph_cfgpath' already exists\n"
 186             if !$lnk || $lnk ne $pve_ceph_cfgpath;
 187     } else {
 188         symlink($pve_ceph_cfgpath, $ceph_cfgpath) ||
 189             die "unable to create symlink '$ceph_cfgpath' - $!\n";
 190     }
 191 }
 192
 193 sub ceph_service_cmd {
 194     my ($action, $service) = @_;
 195
 196     if (systemd_managed()) {
 197
 198         if ($service && $service =~ m/^(mon|osd|mds|radosgw)(\.([A-Za-z0-9]{1,32}))?$/) {
 199             $service = defined($3) ? "ceph-$1\@$3" : "ceph-$1.target";
 200         } else {
 201             $service = "ceph.target";
 202         }
 203
 204         PVE::Tools::run_command(['/bin/systemctl', $action, $service]);
 205
 206     } else {
 207         # ceph daemons does not call 'setsid', so we do that ourself
 208         # (fork_worker send KILL to whole process group)
 209         PVE::Tools::run_command(['setsid', 'service', 'ceph', '-c', $pve_ceph_cfgpath, $action, $service]);
 210     }
 211 }
 212
 213 sub list_disks {
 214     my $disklist = {};
 215
 216     my $fd = IO::File->new("/proc/mounts", "r") ||
 217         die "unable to open /proc/mounts - $!\n";
 218
 219     my $mounted = {};
 220
 221     while (defined(my $line = <$fd>)) {
 222         my ($dev, $path, $fstype) = split(/\s+/, $line);
 223         next if !($dev && $path && $fstype);
 224         next if $dev !~ m|^/dev/|;
 225         my $real_dev = abs_path($dev);
 226         $mounted->{$real_dev} = $path;
 227     }
 228     close($fd);
 229
 230     my $dev_is_mounted = sub {
 231         my ($dev) = @_;
 232         return $mounted->{$dev};
 233     };
 234
 235     my $dir_is_empty = sub {
 236         my ($dir) = @_;
 237
 238         my $dh = IO::Dir->new ($dir);
 239         return 1 if !$dh;
 240
 241         while (defined(my $tmp = $dh->read)) {
 242             next if $tmp eq '.' || $tmp eq '..';
 243             $dh->close;
 244             return 0;
 245         }
 246         $dh->close;
 247         return 1;
 248     };
 249
 250     my $journal_uuid = '45b0969e-9b03-4f30-b4c6-b4b80ceff106';
 251
 252     my $journalhash = {};
 253     dir_glob_foreach('/dev/disk/by-parttypeuuid', "$journal_uuid\..+", sub {
 254         my ($entry) = @_;
 255         my $real_dev = abs_path("/dev/disk/by-parttypeuuid/$entry");
 256         $journalhash->{$real_dev} = 1;
 257     });
 258
 259     dir_glob_foreach('/sys/block', '.*', sub {
 260         my ($dev) = @_;
 261
 262         return if $dev eq '.';
 263         return if $dev eq '..';
 264
 265         return if $dev =~ m|^ram\d+$|; # skip ram devices
 266         return if $dev =~ m|^loop\d+$|; # skip loop devices
 267         return if $dev =~ m|^md\d+$|; # skip md devices
 268         return if $dev =~ m|^dm-.*$|; # skip dm related things
 269         return if $dev =~ m|^fd\d+$|; # skip Floppy
 270         return if $dev =~ m|^sr\d+$|; # skip CDs
 271
 272         my $devdir = "/sys/block/$dev/device";
 273         return if ! -d $devdir;
 274
 275         my $size = file_read_firstline("/sys/block/$dev/size");
 276         return if !$size;
 277
 278         $size = $size * 512;
 279
 280         my $info = `udevadm info --path /sys/block/$dev --query all`;
 281         return if !$info;
 282
 283         return if $info !~ m/^E: DEVTYPE=disk$/m;
 284         return if $info =~ m/^E: ID_CDROM/m;
 285
 286         my $serial = 'unknown';
 287         if ($info =~ m/^E: ID_SERIAL_SHORT=(\S+)$/m) {
 288             $serial = $1;
 289         }
 290
 291         my $gpt = 0;
 292         if ($info =~ m/^E: ID_PART_TABLE_TYPE=gpt$/m) {
 293             $gpt = 1;
 294         }
 295
 296         # detect SSD (fixme - currently only works for ATA disks)
 297         my $rpm = 7200; # default guess
 298         if ($info =~ m/^E: ID_ATA_ROTATION_RATE_RPM=(\d+)$/m) {
 299             $rpm = $1;
 300         }
 301
 302         my $vendor = file_read_firstline("$devdir/vendor") || 'unknown';
 303         my $model = file_read_firstline("$devdir/model") || 'unknown';
 304
 305         my $used;
 306
 307         $used = 'LVM' if !&$dir_is_empty("/sys/block/$dev/holders");
 308
 309         $used = 'mounted' if &$dev_is_mounted("/dev/$dev");
 310
 311         $disklist->{$dev} = {
 312             vendor => $vendor,
 313             model => $model,
 314             size => $size,
 315             serial => $serial,
 316             gpt => $gpt,
 317             rmp => $rpm,
 318         };
 319
 320         my $osdid = -1;
 321
 322         my $journal_count = 0;
 323
 324         my $found_partitions;
 325         my $found_lvm;
 326         my $found_mountpoints;
 327         dir_glob_foreach("/sys/block/$dev", "$dev.+", sub {
 328             my ($part) = @_;
 329
 330             $found_partitions = 1;
 331
 332             if (my $mp = &$dev_is_mounted("/dev/$part")) {
 333                 $found_mountpoints = 1;
 334                 if ($mp =~ m|^/var/lib/ceph/osd/ceph-(\d+)$|) {
 335                     $osdid = $1;
 336                 }
 337             }
 338             if (!&$dir_is_empty("/sys/block/$dev/$part/holders"))  {
 339                 $found_lvm = 1;
 340             }
 341             $journal_count++ if $journalhash->{"/dev/$part"};
 342         });
 343
 344         $used = 'mounted' if $found_mountpoints && !$used;
 345         $used = 'LVM' if $found_lvm && !$used;
 346         $used = 'partitions' if $found_partitions && !$used;
 347
 348         $disklist->{$dev}->{used} = $used if $used;
 349         $disklist->{$dev}->{osdid} = $osdid;
 350         $disklist->{$dev}->{journals} = $journal_count;
 351     });
 352
 353     return $disklist;
 354 }
 355
 356 # Ceph versions greater Hammer use 'ceph' as user and group instead
 357 # of 'root', and use systemd.
 358 sub systemd_managed {
 359
 360     if (-f "/lib/systemd/system/ceph-osd\@.service") {
 361         return 1;
 362     } else {
 363         return 0;
 364     }
 365 }
 366
 367 1;