Diskmanage: detect osds/journals/etc. created with ceph-volume

[pve-storage.git] / PVE / Diskmanage.pm

diff --git a/PVE/Diskmanage.pm b/PVE/Diskmanage.pm
index 5a5fc87799fc767406fadac086391352d39c2b4a..72c1432f438575e86a0e82c2805c141acdc9d08d 100644 (file)
--- a/PVE/Diskmanage.pm
+++ b/PVE/Diskmanage.pm
@@ -13,6 +13,7 @@ my $SMARTCTL = "/usr/sbin/smartctl";
 my $ZPOOL = "/sbin/zpool";
 my $SGDISK = "/sbin/sgdisk";
 my $PVS = "/sbin/pvs";
+my $LVS = "/sbin/lvs";
 my $UDEVADM = "/bin/udevadm";
 
 sub verify_blockdev_path {
@@ -79,8 +80,11 @@ sub get_smart_data {
 
     my $returncode = 0;
 
-    $disk =~ s/n\d+$//
-        if $disk =~ m!^/dev/nvme\d+n\d+$!;
+    if ($disk =~ m!^/dev/(nvme\d+n\d+)$!) {
+       my $info = get_sysdir_info("/sys/block/$1");
+       $disk = "/dev/".($info->{device}
+           or die "failed to get nvme controller device for $disk\n");
+    }
 
     my $cmd = [$SMARTCTL, '-H'];
     push @$cmd, '-A', '-f', 'brief' if !$healthonly;
@@ -149,6 +153,8 @@ sub get_smart_data {
 sub get_zfs_devices {
     my $list = {};
 
+    return {} if ! -x $ZPOOL;
+
     # use zpool and parttype uuid,
     # because log and cache do not have
     # zfs type uuid
@@ -230,6 +236,39 @@ sub get_ceph_journals {
     return $journalhash;
 }
 
+# reads the lv_tags and matches them with the devices
+sub get_ceph_volume_infos {
+    my $result = {};
+
+    my $cmd = [$LVS, '-S', 'lv_name=~^osd-','-o','devices,lv_name,lv_tags',
+              '--noheadings', '--readonly', '--separator', ';'];
+
+    run_command($cmd, outfunc => sub {
+       my $line = shift;
+       $line =~ s/(?:^\s+)|(?:\s+$)//g; # trim
+       my $fields = [split(';', $line)];
+
+       # lvs syntax is /dev/sdX(Y) where Y is the start (which we do not need)
+       my ($dev) = $fields->[0] =~ m|^(/dev/[a-z]+)|;
+       if ($fields->[1] =~ m|^osd-([^-]+)-|) {
+           my $type = $1;
+           # we use autovivification here to not concern us with
+           # creation of empty hashes
+           if (($type eq 'block' || $type eq 'data') &&
+               $fields->[2] =~ m/ceph.osd_id=([^,])/)
+           {
+               $result->{$dev}->{osdid} = $1;
+               $result->{$dev}->{bluestore} = ($type eq 'block');
+           } else {
+               # if $foo is undef $foo++ results in '1' (and is well defined)
+               $result->{$dev}->{$type}++;
+           }
+       }
+    });
+
+    return $result;
+}
+
 sub get_udev_info {
     my ($dev) = @_;
 
@@ -306,6 +345,11 @@ sub get_sysdir_info {
     $data->{vendor} = file_read_firstline("$sysdir/device/vendor") || 'unknown';
     $data->{model} = file_read_firstline("$sysdir/device/model") || 'unknown';
 
+    if (defined(my $device = readlink("$sysdir/device"))) {
+       # strip directory and untaint:
+       ($data->{device}) = $device =~ m!([^/]+)$!;
+    }
+
     return $data;
 }
 
@@ -363,6 +407,16 @@ sub dir_is_empty {
     return 1;
 }
 
+sub is_iscsi {
+    my ($sysdir) = @_;
+
+    if (-l $sysdir && readlink($sysdir) =~ m|host[^/]*/session[^/]*|) {
+       return 1;
+    }
+
+    return 0;
+}
+
 sub get_disks {
     my ($disk, $nosmart) = @_;
     my $disklist = {};
@@ -382,6 +436,7 @@ sub get_disks {
     };
 
     my $journalhash = get_ceph_journals();
+    my $ceph_volume_infos = get_ceph_volume_infos();
 
     my $zfslist = get_zfs_devices();
 
@@ -413,7 +468,7 @@ sub get_disks {
        my $sysdir = "/sys/block/$dev";
 
        # we do not want iscsi devices
-       return if -l $sysdir && readlink($sysdir) =~ m|host[^/]*/session[^/]*|;
+       return if is_iscsi($sysdir);
 
        my $sysdata = get_sysdir_info($sysdir);
        return if !defined($sysdata);
@@ -529,6 +584,16 @@ sub get_disks {
            }
        });
 
+       if ($ceph_volume_infos->{$devpath}) {
+           $journal_count += $ceph_volume_infos->{$devpath}->{journal} // 0;
+           $db_count += $ceph_volume_infos->{$devpath}->{db} // 0;
+           $wal_count += $ceph_volume_infos->{$devpath}->{wal} // 0;
+           if ($ceph_volume_infos->{$devpath}->{osdid}) {
+               $osdid = $ceph_volume_infos->{$devpath}->{osdid};
+               $bluestore = 1 if $ceph_volume_infos->{$devpath}->{bluestore};
+           }
+       }
+
        $used = 'mounted' if $found_mountpoints && !$used;
        $used = 'LVM' if $found_lvm && !$used;
        $used = 'ZFS' if $found_zfs && !$used;
@@ -558,8 +623,8 @@ sub get_partnum {
     my ($mode, $rdev) = (stat($part_path))[2,6];
 
     next if !$mode || !S_ISBLK($mode) || !$rdev;
-    my $major = int($rdev / 0x100);
-    my $minor = $rdev % 0x100;
+    my $major = PVE::Tools::dev_t_major($rdev);
+    my $minor = PVE::Tools::dev_t_minor($rdev);
     my $partnum_path = "/sys/dev/block/$major:$minor/";
 
     my $partnum;
@@ -602,10 +667,10 @@ sub locked_disk_action {
     return $res;
 }
 
-sub check_unused {
+sub assert_disk_unused {
     my ($dev) = @_;
 
-    die "device $dev is already in use\n" if disk_is_used($dev);
+    die "device '$dev' is already in use\n" if disk_is_used($dev);
 
     return undef;
 }