PVE/CephTools.pm

   1 package PVE::CephTools;
   2
   3 use strict;
   4 use warnings;
   5 use File::Basename;
   6 use File::Path;
   7 use POSIX qw (LONG_MAX);
   8 use Cwd qw(abs_path);
   9 use IO::Dir;
  10
  11 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach);
  12
  13 my $ccname = 'ceph'; # ceph cluster name
  14 my $ceph_cfgdir = "/etc/ceph";
  15 my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf";
  16 my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf";
  17
  18 my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
  19 my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring";
  20 my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
  21 my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";
  22
  23 my $ceph_bin = "/usr/bin/ceph";
  24
  25 my $config_hash = {
  26     ccname => $ccname,
  27     pve_ceph_cfgpath => $pve_ceph_cfgpath,
  28     pve_mon_key_path => $pve_mon_key_path,
  29     pve_ckeyring_path => $pve_ckeyring_path,
  30     ceph_bootstrap_osd_keyring => $ceph_bootstrap_osd_keyring,
  31     ceph_bootstrap_mds_keyring => $ceph_bootstrap_mds_keyring,
  32     long_rados_timeout => 60,
  33 };
  34
  35 sub get_config {
  36     my $key = shift;
  37
  38     my $value = $config_hash->{$key};
  39
  40     die "no such ceph config '$key'" if !$value;
  41
  42     return $value;
  43 }
  44
  45 sub verify_blockdev_path {
  46     my ($path) = @_;
  47
  48     $path = abs_path($path);
  49
  50     die "got unusual device path '$path'\n" if $path !~  m|^/dev/(.*)$|;
  51
  52     $path = "/dev/$1"; # untaint
  53
  54     die "no such block device '$path'\n"
  55         if ! -b $path;
  56
  57     return $path;
  58 };
  59
  60 sub purge_all_ceph_files {
  61     # fixme: this is very dangerous - should we really support this function?
  62
  63     unlink $ceph_cfgpath;
  64
  65     unlink $pve_ceph_cfgpath;
  66     unlink $pve_ckeyring_path;
  67     unlink $pve_mon_key_path;
  68
  69     unlink $ceph_bootstrap_osd_keyring;
  70     unlink $ceph_bootstrap_mds_keyring;
  71
  72     system("rm -rf /var/lib/ceph/mon/ceph-*");
  73
  74     # remove osd?
  75 }
  76
  77 sub check_ceph_installed {
  78     my ($noerr) = @_;
  79
  80     if (! -x $ceph_bin) {
  81         die "ceph binaries not installed\n" if !$noerr;
  82         return undef;
  83     }
  84
  85     return 1;
  86 }
  87
  88 sub check_ceph_inited {
  89     my ($noerr) = @_;
  90
  91     return undef if !check_ceph_installed($noerr);
  92
  93     if (! -f $pve_ceph_cfgpath) {
  94         die "pveceph configuration not initialized\n" if !$noerr;
  95         return undef;
  96     }
  97
  98     return 1;
  99 }
 100
 101 sub check_ceph_enabled {
 102     my ($noerr) = @_;
 103
 104     return undef if !check_ceph_inited($noerr);
 105
 106     if (! -f $ceph_cfgpath) {
 107         die "pveceph configuration not enabled\n" if !$noerr;
 108         return undef;
 109     }
 110
 111     return 1;
 112 }
 113
 114 sub parse_ceph_config {
 115     my ($filename) = @_;
 116
 117     $filename = $pve_ceph_cfgpath if !$filename;
 118
 119     my $cfg = {};
 120
 121     return $cfg if ! -f $filename;
 122
 123     my $fh = IO::File->new($filename, "r") ||
 124         die "unable to open '$filename' - $!\n";
 125
 126     my $section;
 127
 128     while (defined(my $line = <$fh>)) {
 129         $line =~ s/[;#].*$//;
 130         $line =~ s/^\s+//;
 131         $line =~ s/\s+$//;
 132         next if !$line;
 133
 134         $section = $1 if $line =~ m/^\[(\S+)\]$/;
 135         if (!$section) {
 136             warn "no section - skip: $line\n";
 137             next;
 138         }
 139
 140         if ($line =~ m/^(.*\S)\s*=\s*(\S.*)$/) {
 141             $cfg->{$section}->{$1} = $2;
 142         }
 143
 144     }
 145
 146     return $cfg;
 147 }
 148
 149 sub write_ceph_config {
 150     my ($cfg) = @_;
 151
 152     my $out = '';
 153
 154     my $cond_write_sec = sub {
 155         my $re = shift;
 156
 157         foreach my $section (keys %$cfg) {
 158             next if $section !~ m/^$re$/;
 159             $out .= "[$section]\n";
 160             foreach my $key (sort keys %{$cfg->{$section}}) {
 161                 $out .= "\t $key = $cfg->{$section}->{$key}\n";
 162             }
 163             $out .= "\n";
 164         }
 165     };
 166
 167     &$cond_write_sec('global');
 168     &$cond_write_sec('mon');
 169     &$cond_write_sec('osd');
 170     &$cond_write_sec('mon\..*');
 171     &$cond_write_sec('osd\..*');
 172
 173     PVE::Tools::file_set_contents($pve_ceph_cfgpath, $out);
 174 }
 175
 176 sub setup_pve_symlinks {
 177     # fail if we find a real file instead of a link
 178     if (-f $ceph_cfgpath) {
 179         my $lnk = readlink($ceph_cfgpath);
 180         die "file '$ceph_cfgpath' already exists\n"
 181             if !$lnk || $lnk ne $pve_ceph_cfgpath;
 182     } else {
 183         symlink($pve_ceph_cfgpath, $ceph_cfgpath) ||
 184             die "unable to create symlink '$ceph_cfgpath' - $!\n";
 185     }
 186 }
 187
 188 sub ceph_service_cmd {
 189     # ceph daemons does not call 'setsid', so we do that ourself
 190     # (fork_worker send KILL to whole process group)
 191     PVE::Tools::run_command(['setsid', 'service', 'ceph', '-c', $pve_ceph_cfgpath, @_]);
 192 }
 193
 194 sub list_disks {
 195     my $disklist = {};
 196
 197     my $fd = IO::File->new("/proc/mounts", "r") ||
 198         die "unable to open /proc/mounts - $!\n";
 199
 200     my $mounted = {};
 201
 202     while (defined(my $line = <$fd>)) {
 203         my ($dev, $path, $fstype) = split(/\s+/, $line);
 204         next if !($dev && $path && $fstype);
 205         next if $dev !~ m|^/dev/|;
 206         my $real_dev = abs_path($dev);
 207         $mounted->{$real_dev} = $path;
 208     }
 209     close($fd);
 210
 211     my $dev_is_mounted = sub {
 212         my ($dev) = @_;
 213         return $mounted->{$dev};
 214     };
 215
 216     my $dir_is_epmty = sub {
 217         my ($dir) = @_;
 218
 219         my $dh = IO::Dir->new ($dir);
 220         return 1 if !$dh;
 221
 222         while (defined(my $tmp = $dh->read)) {
 223             next if $tmp eq '.' || $tmp eq '..';
 224             $dh->close;
 225             return 0;
 226         }
 227         $dh->close;
 228         return 1;
 229     };
 230
 231     my $journal_uuid = '45b0969e-9b03-4f30-b4c6-b4b80ceff106';
 232
 233     my $journalhash = {};
 234     dir_glob_foreach('/dev/disk/by-parttypeuuid', "$journal_uuid\..+", sub {
 235         my ($entry) = @_;
 236         my $real_dev = abs_path("/dev/disk/by-parttypeuuid/$entry");
 237         $journalhash->{$real_dev} = 1;
 238     });
 239
 240     dir_glob_foreach('/sys/block', '.*', sub {
 241         my ($dev) = @_;
 242
 243         return if $dev eq '.';
 244         return if $dev eq '..';
 245
 246         return if $dev =~ m|^ram\d+$|; # skip ram devices
 247         return if $dev =~ m|^loop\d+$|; # skip loop devices
 248         return if $dev =~ m|^md\d+$|; # skip md devices
 249         return if $dev =~ m|^dm-.*$|; # skip dm related things
 250         return if $dev =~ m|^fd\d+$|; # skip Floppy
 251         return if $dev =~ m|^sr\d+$|; # skip CDs
 252
 253         my $devdir = "/sys/block/$dev/device";
 254         return if ! -d $devdir;
 255
 256         my $size = file_read_firstline("/sys/block/$dev/size");
 257         return if !$size;
 258
 259         $size = $size * 512;
 260
 261         my $info = `udevadm info --path /sys/block/$dev --query all`;
 262         return if !$info;
 263
 264         return if $info !~ m/^E: DEVTYPE=disk$/m;
 265         return if $info =~ m/^E: ID_CDROM/m;
 266
 267         my $serial = 'unknown';
 268         if ($info =~ m/^E: ID_SERIAL_SHORT=(\S+)$/m) {
 269             $serial = $1;
 270         }
 271
 272         my $gpt = 0;
 273         if ($info =~ m/^E: ID_PART_TABLE_TYPE=gpt$/m) {
 274             $gpt = 1;
 275         }
 276
 277         # detect SSD (fixme - currently only works for ATA disks)
 278         my $rpm = 7200; # default guess
 279         if ($info =~ m/^E: ID_ATA_ROTATION_RATE_RPM=(\d+)$/m) {
 280             $rpm = $1;
 281         }
 282
 283         my $vendor = file_read_firstline("$devdir/vendor") || 'unknown';
 284         my $model = file_read_firstline("$devdir/model") || 'unknown';
 285
 286         my $used;
 287
 288         $used = 'LVM' if !&$dir_is_epmty("/sys/block/$dev/holders");
 289
 290         $used = 'mounted' if &$dev_is_mounted("/dev/$dev");
 291
 292         $disklist->{$dev} = {
 293             vendor => $vendor,
 294             model => $model,
 295             size => $size,
 296             serial => $serial,
 297             gpt => $gpt,
 298             rmp => $rpm,
 299         };
 300
 301         my $osdid = -1;
 302
 303         my $journal_count = 0;
 304
 305         my $found_partitions;
 306         my $found_lvm;
 307         my $found_mountpoints;
 308         dir_glob_foreach("/sys/block/$dev", "$dev.+", sub {
 309             my ($part) = @_;
 310
 311             $found_partitions = 1;
 312
 313             if (my $mp = &$dev_is_mounted("/dev/$part")) {
 314                 $found_mountpoints = 1;
 315                 if ($mp =~ m|^/var/lib/ceph/osd/ceph-(\d+)$|) {
 316                     $osdid = $1;
 317                 }
 318             }
 319             if (!&$dir_is_epmty("/sys/block/$dev/$part/holders"))  {
 320                 $found_lvm = 1;
 321             }
 322             $journal_count++ if $journalhash->{"/dev/$part"};
 323         });
 324
 325         $used = 'mounted' if $found_mountpoints && !$used;
 326         $used = 'LVM' if $found_lvm && !$used;
 327         $used = 'partitions' if $found_partitions && !$used;
 328
 329         $disklist->{$dev}->{used} = $used if $used;
 330         $disklist->{$dev}->{osdid} = $osdid;
 331         $disklist->{$dev}->{journals} = $journal_count;
 332     });
 333
 334     return $disklist;
 335 }
 336
 337 1;