PVE/Diskmanage.pm

   1 package PVE::Diskmanage;
   2
   3 use strict;
   4 use warnings;
   5
   6 use PVE::ProcFSTools;
   7 use Data::Dumper;
   8 use Cwd qw(abs_path);
   9 use Fcntl ':mode';
  10 use File::Basename;
  11 use File::stat;
  12 use JSON;
  13
  14 use PVE::Tools qw(extract_param run_command file_get_contents file_read_firstline dir_glob_regex dir_glob_foreach trim);
  15
  16 my $SMARTCTL = "/usr/sbin/smartctl";
  17 my $ZPOOL = "/sbin/zpool";
  18 my $SGDISK = "/sbin/sgdisk";
  19 my $PVS = "/sbin/pvs";
  20 my $LVS = "/sbin/lvs";
  21 my $LSBLK = "/bin/lsblk";
  22
  23 my sub strip_dev :prototype($) {
  24     my ($devpath) = @_;
  25     $devpath =~ s|^/dev/||;
  26     return $devpath;
  27 }
  28
  29 sub check_bin {
  30     my ($path) = @_;
  31     return -x $path;
  32 }
  33
  34 sub verify_blockdev_path {
  35     my ($rel_path) = @_;
  36
  37     die "missing path" if !$rel_path;
  38     my $path = abs_path($rel_path);
  39     die "failed to get absolute path to $rel_path\n" if !$path;
  40
  41     die "got unusual device path '$path'\n" if $path !~  m|^/dev/(.*)$|;
  42
  43     $path = "/dev/$1"; # untaint
  44
  45     assert_blockdev($path);
  46
  47     return $path;
  48 }
  49
  50 sub assert_blockdev {
  51     my ($dev, $noerr) = @_;
  52
  53     if ($dev !~ m|^/dev/| || !(-b $dev)) {
  54         return if $noerr;
  55         die "not a valid block device\n";
  56     }
  57
  58     return 1;
  59 }
  60
  61 sub init_disk {
  62     my ($disk, $uuid) = @_;
  63
  64     assert_blockdev($disk);
  65
  66     # we should already have checked these in the api call, but we check again for safety
  67     die "$disk is a partition\n" if is_partition($disk);
  68     die "disk $disk is already in use\n" if disk_is_used($disk);
  69
  70     my $id = $uuid || 'R';
  71     run_command([$SGDISK, $disk, '-U', $id]);
  72     return 1;
  73 }
  74
  75 sub disk_is_used {
  76     my ($disk) = @_;
  77
  78     my $dev = $disk;
  79     $dev =~ s|^/dev/||;
  80
  81     my $disklist = get_disks($dev, 1, 1);
  82
  83     die "'$disk' is not a valid local disk\n" if !defined($disklist->{$dev});
  84     return 1 if $disklist->{$dev}->{used};
  85
  86     return 0;
  87 }
  88
  89 sub get_smart_data {
  90     my ($disk, $healthonly) = @_;
  91
  92     assert_blockdev($disk);
  93     my $smartdata = {};
  94     my $type;
  95
  96     my $cmd = [$SMARTCTL, '-H'];
  97     push @$cmd, '-A', '-f', 'brief' if !$healthonly;
  98     push @$cmd, $disk;
  99
 100     my $returncode = eval {
 101         run_command($cmd, noerr => 1, outfunc => sub {
 102             my ($line) = @_;
 103
 104 # ATA SMART attributes, e.g.:
 105 # ID# ATTRIBUTE_NAME          FLAGS    VALUE WORST THRESH FAIL RAW_VALUE
 106 #   1 Raw_Read_Error_Rate     POSR-K   100   100   000    -    0
 107 #
 108 # SAS and NVME disks, e.g.:
 109 # Data Units Written:                 5,584,952 [2.85 TB]
 110 # Accumulated start-stop cycles:  34
 111
 112             if (defined($type) && $type eq 'ata' && $line =~ m/^([ \d]{2}\d)\s+(\S+)\s+(\S{6})\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(.*)$/) {
 113                 my $entry = {};
 114
 115                 $entry->{name} = $2 if defined $2;
 116                 $entry->{flags} = $3 if defined $3;
 117                 # the +0 makes a number out of the strings
 118                 # FIXME: 'value' is depreacated by 'normalized'; remove with PVE 7.0
 119                 $entry->{value} = $4+0 if defined $4;
 120                 $entry->{normalized} = $4+0 if defined $4;
 121                 $entry->{worst} = $5+0 if defined $5;
 122                 # some disks report the default threshold as --- instead of 000
 123                 if (defined($6) && $6 eq '---') {
 124                     $entry->{threshold} = 0;
 125                 } else {
 126                     $entry->{threshold} = $6+0 if defined $6;
 127                 }
 128                 $entry->{fail} = $7 if defined $7;
 129                 $entry->{raw} = $8 if defined $8;
 130                 $entry->{id} = $1 if defined $1;
 131                 push @{$smartdata->{attributes}}, $entry;
 132             } elsif ($line =~ m/(?:Health Status|self\-assessment test result): (.*)$/ ) {
 133                 $smartdata->{health} = $1;
 134             } elsif ($line =~ m/Vendor Specific SMART Attributes with Thresholds:/) {
 135                 $type = 'ata';
 136                 delete $smartdata->{text};
 137             } elsif ($line =~ m/=== START OF (READ )?SMART DATA SECTION ===/) {
 138                 $type = 'text';
 139             } elsif (defined($type) && $type eq 'text') {
 140                 $smartdata->{text} = '' if !defined $smartdata->{text};
 141                 $smartdata->{text} .= "$line\n";
 142                 # extract wearout from nvme/sas text, allow for decimal values
 143                 if ($line =~ m/Percentage Used(?: endurance indicator)?:\s*(\d+(?:\.\d+)?)\%/i) {
 144                     $smartdata->{wearout} = 100 - $1;
 145                 }
 146             } elsif ($line =~ m/SMART Disabled/) {
 147                 $smartdata->{health} = "SMART Disabled";
 148             }
 149         })
 150     };
 151     my $err = $@;
 152
 153     # bit 0 and 1 mark a fatal error, other bits are for disk status -> ignore (see man 8 smartctl)
 154     if ((defined($returncode) && ($returncode & 0b00000011)) || $err) {
 155         die "Error getting S.M.A.R.T. data: Exit code: $returncode\n";
 156     }
 157
 158     $smartdata->{type} = $type;
 159
 160     return $smartdata;
 161 }
 162
 163 sub get_lsblk_info {
 164     my $cmd = [$LSBLK, '--json', '-o', 'path,parttype,fstype'];
 165     my $output = "";
 166     eval { run_command($cmd, outfunc => sub { $output .= "$_[0]\n"; }) };
 167     warn "$@\n" if $@;
 168     return {} if $output eq '';
 169
 170     my $parsed = eval { decode_json($output) } // {};
 171     warn "$@\n" if $@;
 172     my $list = $parsed->{blockdevices} // [];
 173
 174     return {
 175         map {
 176             $_->{path} => {
 177                 parttype => $_->{parttype},
 178                 fstype => $_->{fstype}
 179             }
 180         } @{$list}
 181     };
 182 }
 183
 184 my sub get_devices_by_partuuid {
 185     my ($lsblk_info, $uuids, $res) = @_;
 186
 187     $res = {} if !defined($res);
 188
 189     foreach my $dev (sort keys %{$lsblk_info}) {
 190         my $uuid = $lsblk_info->{$dev}->{parttype};
 191         next if !defined($uuid) || !defined($uuids->{$uuid});
 192         $res->{$dev} = $uuids->{$uuid};
 193     }
 194
 195     return $res;
 196 }
 197
 198 sub get_zfs_devices {
 199     my ($lsblk_info) = @_;
 200     my $res = {};
 201
 202     return {} if !check_bin($ZPOOL);
 203
 204     # use zpool and parttype uuid, because log and cache do not have zfs type uuid
 205     eval {
 206         run_command([$ZPOOL, 'list', '-HPLv'], outfunc => sub {
 207              my ($line) = @_;
 208              if ($line =~ m|^\t([^\t]+)\t|) {
 209                 $res->{$1} = 1;
 210              }
 211         });
 212     };
 213
 214     # only warn here, because maybe zfs tools are not installed
 215     warn "$@\n" if $@;
 216
 217     my $uuids = {
 218         "6a898cc3-1dd2-11b2-99a6-080020736631" => 1, # apple
 219         "516e7cba-6ecf-11d6-8ff8-00022d09712b" => 1, # bsd
 220     };
 221
 222
 223     $res = get_devices_by_partuuid($lsblk_info, $uuids, $res);
 224
 225     return $res;
 226 }
 227
 228 sub get_lvm_devices {
 229     my ($lsblk_info) = @_;
 230     my $res = {};
 231     eval {
 232         run_command([$PVS, '--noheadings', '--readonly', '-o', 'pv_name'], outfunc => sub{
 233             my ($line) = @_;
 234             $line = trim($line);
 235             if ($line =~ m|^/dev/|) {
 236                 $res->{$line} = 1;
 237             }
 238         });
 239     };
 240
 241     # if something goes wrong, we do not want to give up, but indicate an error has occurred
 242     warn "$@\n" if $@;
 243
 244     my $uuids = {
 245         "e6d6d379-f507-44c2-a23c-238f2a3df928" => 1,
 246     };
 247
 248     $res = get_devices_by_partuuid($lsblk_info, $uuids, $res);
 249
 250     return $res;
 251 }
 252
 253 sub get_ceph_journals {
 254     my ($lsblk_info) = @_;
 255     my $res = {};
 256
 257     my $uuids = {
 258         '45b0969e-9b03-4f30-b4c6-b4b80ceff106' => 1, # journal
 259         '30cd0809-c2b2-499c-8879-2d6b78529876' => 2, # db
 260         '5ce17fce-4087-4169-b7ff-056cc58473f9' => 3, # wal
 261         'cafecafe-9b03-4f30-b4c6-b4b80ceff106' => 4, # block
 262     };
 263
 264     $res = get_devices_by_partuuid($lsblk_info, $uuids, $res);
 265
 266     return $res;
 267 }
 268
 269 # reads the lv_tags and matches them with the devices
 270 sub get_ceph_volume_infos {
 271     my $result = {};
 272
 273     my $cmd = [ $LVS, '-S', 'lv_name=~^osd-', '-o', 'devices,lv_name,lv_tags',
 274                '--noheadings', '--readonly', '--separator', ';' ];
 275
 276     run_command($cmd, outfunc => sub {
 277         my $line = shift;
 278         $line =~ s/(?:^\s+)|(?:\s+$)//g; # trim whitespaces
 279
 280         my $fields = [ split(';', $line) ];
 281
 282         # lvs syntax is /dev/sdX(Y) where Y is the start (which we do not need)
 283         my ($dev) = $fields->[0] =~ m|^(/dev/[a-z]+[^(]*)|;
 284         if ($fields->[1] =~ m|^osd-([^-]+)-|) {
 285             my $type = $1;
 286             # $result autovivification is wanted, to not creating empty hashes
 287             if (($type eq 'block' || $type eq 'data') && $fields->[2] =~ m/ceph.osd_id=([^,]+)/) {
 288                 $result->{$dev}->{osdid} = $1;
 289                 $result->{$dev}->{bluestore} = ($type eq 'block');
 290                 if ($fields->[2] =~ m/ceph\.encrypted=1/) {
 291                     $result->{$dev}->{encrypted} = 1;
 292                 }
 293             } else {
 294                 # undef++ becomes '1' (see `perldoc perlop`: Auto-increment)
 295                 $result->{$dev}->{$type}++;
 296             }
 297         }
 298     });
 299
 300     return $result;
 301 }
 302
 303 sub get_udev_info {
 304     my ($dev) = @_;
 305
 306     my $info = "";
 307     my $data = {};
 308     eval {
 309         run_command(['udevadm', 'info', '-p', $dev, '--query', 'all'], outfunc => sub {
 310             my ($line) = @_;
 311             $info .= "$line\n";
 312         });
 313     };
 314     warn $@ if $@;
 315     return if !$info;
 316
 317     return if $info !~ m/^E: DEVTYPE=(disk|partition)$/m;
 318     return if $info =~ m/^E: ID_CDROM/m;
 319
 320     # we use this, because some disks are not simply in /dev e.g. /dev/cciss/c0d0
 321     if ($info =~ m/^E: DEVNAME=(\S+)$/m) {
 322         $data->{devpath} = $1;
 323     }
 324     return if !defined($data->{devpath});
 325
 326     $data->{serial} = 'unknown';
 327     $data->{serial} = $1 if $info =~ m/^E: ID_SERIAL_SHORT=(\S+)$/m;
 328
 329     $data->{gpt} = $info =~ m/^E: ID_PART_TABLE_TYPE=gpt$/m ? 1 : 0;
 330
 331     $data->{rpm} = -1;
 332     $data->{rpm} = $1 if $info =~ m/^E: ID_ATA_ROTATION_RATE_RPM=(\d+)$/m; # detects SSD implicit
 333
 334     $data->{usb} = 1 if $info =~ m/^E: ID_BUS=usb$/m;
 335
 336     $data->{model} = $1 if $info =~ m/^E: ID_MODEL=(.+)$/m;
 337
 338     $data->{wwn} = 'unknown';
 339     $data->{wwn} = $1 if $info =~ m/^E: ID_WWN=(.*)$/m;
 340
 341     if ($info =~ m/^E: DEVLINKS=(.+)$/m) {
 342         my @devlinks = grep(m#^/dev/disk/by-id/(ata|scsi|nvme(?!-eui))#, split (/ /, $1));
 343         $data->{by_id_link} = $devlinks[0] if defined($devlinks[0]);
 344     }
 345
 346     return $data;
 347 }
 348
 349 sub get_sysdir_size {
 350     my ($sysdir) = @_;
 351
 352     my $size = file_read_firstline("$sysdir/size");
 353     return if !$size;
 354
 355     # linux always considers sectors to be 512 bytes, independently of real block size
 356     return $size * 512;
 357 }
 358
 359 sub get_sysdir_info {
 360     my ($sysdir) = @_;
 361
 362     return if ! -d "$sysdir/device";
 363
 364     my $data = {};
 365
 366     $data->{size} = get_sysdir_size($sysdir) or return;
 367
 368     # dir/queue/rotational should be 1 for hdd, 0 for ssd
 369     $data->{rotational} = file_read_firstline("$sysdir/queue/rotational") // -1;
 370
 371     $data->{vendor} = file_read_firstline("$sysdir/device/vendor") || 'unknown';
 372     $data->{model} = file_read_firstline("$sysdir/device/model") || 'unknown';
 373
 374     return $data;
 375 }
 376
 377 sub get_wear_leveling_info {
 378     my ($smartdata) = @_;
 379     my $attributes = $smartdata->{attributes};
 380
 381     if (defined($smartdata->{wearout})) {
 382         return $smartdata->{wearout};
 383     }
 384
 385     my $wearout;
 386
 387     # Common register names that represent percentage values of potential failure indicators used
 388     # in drivedb.h of smartmontool's. Order matters, as some drives may have multiple definitions
 389     my @wearoutregisters = (
 390         "Media_Wearout_Indicator",
 391         "SSD_Life_Left",
 392         "Wear_Leveling_Count",
 393         "Perc_Write\/Erase_Ct_BC",
 394         "Perc_Rated_Life_Remain",
 395         "Remaining_Lifetime_Perc",
 396         "Percent_Lifetime_Remain",
 397         "Lifetime_Left",
 398         "PCT_Life_Remaining",
 399         "Lifetime_Remaining",
 400         "Percent_Life_Remaining",
 401         "Percent_Lifetime_Used",
 402         "Perc_Rated_Life_Used"
 403     );
 404
 405     # Search for S.M.A.R.T. attributes for known register
 406     foreach my $register (@wearoutregisters) {
 407         last if defined $wearout;
 408         foreach my $attr (@$attributes) {
 409            next if $attr->{name} !~ m/$register/;
 410            $wearout = $attr->{value};
 411            last;
 412         }
 413     }
 414
 415     return $wearout;
 416 }
 417
 418 sub dir_is_empty {
 419     my ($dir) = @_;
 420
 421     my $dh = IO::Dir->new ($dir);
 422     return 1 if !$dh;
 423
 424     while (defined(my $tmp = $dh->read)) {
 425         next if $tmp eq '.' || $tmp eq '..';
 426         $dh->close;
 427         return 0;
 428     }
 429     $dh->close;
 430     return 1;
 431 }
 432
 433 sub is_iscsi {
 434     my ($sysdir) = @_;
 435
 436     if (-l $sysdir && readlink($sysdir) =~ m|host[^/]*/session[^/]*|) {
 437         return 1;
 438     }
 439
 440     return 0;
 441 }
 442
 443 my sub is_ssdlike {
 444     my ($type) = @_;
 445     return $type eq 'ssd' || $type eq 'nvme';
 446 }
 447
 448 sub mounted_blockdevs {
 449     my $mounted = {};
 450
 451     my $mounts = PVE::ProcFSTools::parse_proc_mounts();
 452
 453     foreach my $mount (@$mounts) {
 454         next if $mount->[0] !~ m|^/dev/|;
 455         $mounted->{abs_path($mount->[0])} = $mount->[1];
 456     };
 457
 458     return $mounted;
 459 }
 460
 461 # returns hashmap of abs mount path -> first part of /proc/mounts (what)
 462 sub mounted_paths {
 463     my $mounted = {};
 464
 465     my $mounts = PVE::ProcFSTools::parse_proc_mounts();
 466
 467     foreach my $mount (@$mounts) {
 468         $mounted->{abs_path($mount->[1])} = $mount->[0];
 469     };
 470
 471     return $mounted;
 472 }
 473
 474 sub get_disks {
 475     my ($disks, $nosmart, $include_partitions) = @_;
 476     my $disklist = {};
 477
 478     my $mounted = mounted_blockdevs();
 479
 480     my $lsblk_info = get_lsblk_info();
 481
 482     my $journalhash = get_ceph_journals($lsblk_info);
 483     my $ceph_volume_infos = get_ceph_volume_infos();
 484
 485     my $zfshash = get_zfs_devices($lsblk_info);
 486
 487     my $lvmhash = get_lvm_devices($lsblk_info);
 488
 489     my $disk_regex = ".*";
 490     if (defined($disks)) {
 491         if (!ref($disks)) {
 492             $disks = [ $disks ];
 493         } elsif (ref($disks) ne 'ARRAY') {
 494             die "disks is not a string or array reference\n";
 495         }
 496         # we get cciss/c0d0 but need cciss!c0d0
 497         $_ =~ s|cciss/|cciss!| for @$disks;
 498
 499         if ($include_partitions) {
 500             # Proper blockdevice is needed for the regex, use parent for partitions.
 501             for my $disk ($disks->@*) {
 502                 next if !is_partition("/dev/$disk");
 503                 $disk = strip_dev(get_blockdev("/dev/$disk"));
 504             }
 505         }
 506
 507         $disk_regex = "(?:" . join('|', @$disks) . ")";
 508     }
 509
 510     dir_glob_foreach('/sys/block', $disk_regex, sub {
 511         my ($dev) = @_;
 512         # whitelisting following devices
 513         # - hdX         ide block device
 514         # - sdX         scsi/sata block device
 515         # - vdX         virtIO block device
 516         # - xvdX:       xen virtual block device
 517         # - nvmeXnY:    nvme devices
 518         # - cciss!cXnY  cciss devices
 519         return if $dev !~ m/^(h|s|x?v)d[a-z]+$/ &&
 520                   $dev !~ m/^nvme\d+n\d+$/ &&
 521                   $dev !~ m/^cciss\!c\d+d\d+$/;
 522
 523         my $data = get_udev_info("/sys/block/$dev") // return;
 524         my $devpath = $data->{devpath};
 525
 526         my $sysdir = "/sys/block/$dev";
 527
 528         # we do not want iscsi devices
 529         return if is_iscsi($sysdir);
 530
 531         my $sysdata = get_sysdir_info($sysdir);
 532         return if !defined($sysdata);
 533
 534         my $type = 'unknown';
 535
 536         if ($sysdata->{rotational} == 0) {
 537             $type = 'ssd';
 538             $type = 'nvme' if $dev =~ m/^nvme\d+n\d+$/;
 539             $data->{rpm} = 0;
 540         } elsif ($sysdata->{rotational} == 1) {
 541             if ($data->{rpm} != -1) {
 542                 $type = 'hdd';
 543             } elsif ($data->{usb}) {
 544                 $type = 'usb';
 545                 $data->{rpm} = 0;
 546             }
 547         }
 548
 549         my ($health, $wearout) = ('UNKNOWN', 'N/A');
 550         if (!$nosmart) {
 551             eval {
 552                 my $smartdata = get_smart_data($devpath, !is_ssdlike($type));
 553                 $health = $smartdata->{health} if $smartdata->{health};
 554
 555                 if (is_ssdlike($type)) { # if we have an ssd we try to get the wearout indicator
 556                     my $wear_level = get_wear_leveling_info($smartdata);
 557                     $wearout = $wear_level if defined($wear_level);
 558                 }
 559             };
 560         }
 561
 562         # we replaced cciss/ with cciss! above, but in the result we need cciss/ again because the
 563         # caller might want to check the result again with the original parameter
 564         if ($dev =~ m|^cciss!|) {
 565             $dev =~ s|^cciss!|cciss/|;
 566         }
 567
 568         $disklist->{$dev} = {
 569             vendor => $sysdata->{vendor},
 570             model => $data->{model} || $sysdata->{model},
 571             size => $sysdata->{size},
 572             serial => $data->{serial},
 573             gpt => $data->{gpt},
 574             rpm => $data->{rpm},
 575             type =>  $type,
 576             wwn => $data->{wwn},
 577             health => $health,
 578             devpath => $devpath,
 579             wearout => $wearout,
 580         };
 581         $disklist->{$dev}->{mounted} = 1 if exists $mounted->{$devpath};
 582
 583         my $by_id_link = $data->{by_id_link};
 584         $disklist->{$dev}->{by_id_link} = $by_id_link if defined($by_id_link);
 585
 586         my ($osdid, $bluestore, $osdencrypted) = (-1, 0, 0);
 587         my ($journal_count, $db_count, $wal_count) = (0, 0, 0);
 588
 589         my $partpath = $devpath;
 590         # remove trailing part to get the partition base path, e.g. /dev/cciss/c0d0 -> /dev/cciss
 591         $partpath =~ s/\/[^\/]+$//;
 592
 593         my $determine_usage = sub {
 594             my ($devpath, $sysdir, $is_partition) = @_;
 595
 596             return 'LVM' if $lvmhash->{$devpath};
 597             return 'ZFS' if $zfshash->{$devpath};
 598
 599             my $info = $lsblk_info->{$devpath} // {};
 600
 601             if (defined(my $parttype = $info->{parttype})) {
 602                 return 'BIOS boot'if $parttype eq '21686148-6449-6e6f-744e-656564454649';
 603                 return 'EFI' if $parttype eq 'c12a7328-f81f-11d2-ba4b-00a0c93ec93b';
 604                 return 'ZFS reserved' if $parttype eq '6a945a3b-1dd2-11b2-99a6-080020736631';
 605             }
 606
 607             return "$info->{fstype}" if defined($info->{fstype});
 608             return 'mounted' if $mounted->{$devpath};
 609
 610             return if !$is_partition;
 611
 612             # for devices, this check is done explicitly later
 613             return 'Device Mapper' if !dir_is_empty("$sysdir/holders");
 614
 615             return; # unused partition
 616         };
 617
 618         my $collect_ceph_info = sub {
 619             my ($devpath) = @_;
 620
 621             my $ceph_volume = $ceph_volume_infos->{$devpath} or return;
 622             $journal_count += $ceph_volume->{journal} // 0;
 623             $db_count += $ceph_volume->{db} // 0;
 624             $wal_count += $ceph_volume->{wal} // 0;
 625             if (defined($ceph_volume->{osdid})) {
 626                 $osdid = $ceph_volume->{osdid};
 627                 $bluestore = 1 if $ceph_volume->{bluestore};
 628                 $osdencrypted = 1 if $ceph_volume->{encrypted};
 629             }
 630
 631             my $result = { %{$ceph_volume} };
 632             $result->{journals} = delete $result->{journal} if $result->{journal};
 633             return $result;
 634         };
 635
 636         my $partitions = {};
 637         dir_glob_foreach("$sysdir", "$dev.+", sub {
 638             my ($part) = @_;
 639
 640             $partitions->{$part} = $collect_ceph_info->("$partpath/$part");
 641             my $lvm_based_osd = defined($partitions->{$part});
 642
 643             $partitions->{$part}->{devpath} = "$partpath/$part";
 644             $partitions->{$part}->{parent} = "$devpath";
 645             $partitions->{$part}->{mounted} = 1 if exists $mounted->{"$partpath/$part"};
 646             $partitions->{$part}->{gpt} = $data->{gpt};
 647             $partitions->{$part}->{type} = 'partition';
 648             $partitions->{$part}->{size} = get_sysdir_size("$sysdir/$part") // 0;
 649             $partitions->{$part}->{used} = $determine_usage->("$partpath/$part", "$sysdir/$part", 1);
 650             $partitions->{$part}->{osdid} //= -1;
 651
 652             # avoid counting twice (e.g. partition with the LVM for the DB OSD is in $journalhash)
 653             return if $lvm_based_osd;
 654
 655             # Legacy handling for non-LVM based OSDs
 656             if (my $mp = $mounted->{"$partpath/$part"}) {
 657                 if ($mp =~ m|^/var/lib/ceph/osd/ceph-(\d+)$|) {
 658                     $osdid = $1;
 659                     $partitions->{$part}->{osdid} = $osdid;
 660                 }
 661             }
 662
 663             if (my $journal_part = $journalhash->{"$partpath/$part"}) {
 664                 $journal_count++ if $journal_part == 1;
 665                 $db_count++ if $journal_part == 2;
 666                 $wal_count++ if $journal_part == 3;
 667                 $bluestore = 1 if $journal_part == 4;
 668
 669                 $partitions->{$part}->{journals} = 1 if $journal_part == 1;
 670                 $partitions->{$part}->{db} = 1 if $journal_part == 2;
 671                 $partitions->{$part}->{wal} = 1 if $journal_part == 3;
 672                 $partitions->{$part}->{bluestore} = 1 if $journal_part == 4;
 673             }
 674         });
 675
 676         my $used = $determine_usage->($devpath, $sysdir, 0);
 677         if (!$include_partitions) {
 678             foreach my $part (sort keys %{$partitions}) {
 679                 $used //= $partitions->{$part}->{used};
 680             }
 681         } else {
 682             # fstype might be set even if there are partitions, but showing that is confusing
 683             $used = 'partitions' if scalar(keys %{$partitions});
 684         }
 685         $used //= 'partitions' if scalar(keys %{$partitions});
 686         # multipath, software raid, etc.
 687         # this check comes in last, to show more specific info
 688         # if we have it
 689         $used //= 'Device Mapper' if !dir_is_empty("$sysdir/holders");
 690
 691         $disklist->{$dev}->{used} = $used if $used;
 692
 693         $collect_ceph_info->($devpath);
 694
 695         $disklist->{$dev}->{osdid} = $osdid;
 696         $disklist->{$dev}->{journals} = $journal_count if $journal_count;
 697         $disklist->{$dev}->{bluestore} = $bluestore if $osdid != -1;
 698         $disklist->{$dev}->{osdencrypted} = $osdencrypted if $osdid != -1;
 699         $disklist->{$dev}->{db} = $db_count if $db_count;
 700         $disklist->{$dev}->{wal} = $wal_count if $wal_count;
 701
 702         if ($include_partitions) {
 703             $disklist->{$_} = $partitions->{$_} for keys %{$partitions};
 704         }
 705     });
 706
 707     return $disklist;
 708 }
 709
 710 sub get_partnum {
 711     my ($part_path) = @_;
 712
 713     my $st = stat($part_path);
 714
 715     die "error detecting block device '$part_path'\n"
 716         if !$st || !$st->mode || !S_ISBLK($st->mode) || !$st->rdev;
 717
 718     my $major = PVE::Tools::dev_t_major($st->rdev);
 719     my $minor = PVE::Tools::dev_t_minor($st->rdev);
 720     my $partnum_path = "/sys/dev/block/$major:$minor/";
 721
 722     my $partnum = file_read_firstline("${partnum_path}partition");
 723     die "Partition does not exist\n" if !defined($partnum);
 724     die "Failed to get partition number\n" if $partnum !~ m/(\d+)/; # untaint
 725     $partnum = $1;
 726     die "Partition number $partnum is invalid\n" if $partnum > 128;
 727
 728     return $partnum;
 729 }
 730
 731 sub get_blockdev {
 732     my ($part_path) = @_;
 733
 734     my ($dev, $block_dev);
 735     if ($part_path =~ m|^/dev/(.*)$|) {
 736         $dev = $1;
 737         my $link = readlink "/sys/class/block/$dev";
 738         $block_dev = $1 if $link =~ m|([^/]*)/$dev$|;
 739     }
 740
 741     die "Can't parse parent device\n" if !defined($block_dev);
 742     die "No valid block device\n" if index($dev, $block_dev) == -1;
 743
 744     $block_dev = "/dev/$block_dev";
 745     die "Block device does not exists\n" if !(-b $block_dev);
 746
 747     return $block_dev;
 748 }
 749
 750 sub is_partition {
 751     my ($dev_path) = @_;
 752
 753     return defined(eval { get_partnum($dev_path) });
 754 }
 755
 756 sub locked_disk_action {
 757     my ($sub) = @_;
 758     my $res = PVE::Tools::lock_file('/run/lock/pve-diskmanage.lck', undef, $sub);
 759     die $@ if $@;
 760     return $res;
 761 }
 762
 763 sub assert_disk_unused {
 764     my ($dev) = @_;
 765     die "device '$dev' is already in use\n" if disk_is_used($dev);
 766     return;
 767 }
 768
 769 sub append_partition {
 770     my ($dev, $size) = @_;
 771
 772     my $devname = $dev;
 773     $devname =~ s|^/dev/||;
 774
 775     my $newpartid = 1;
 776     dir_glob_foreach("/sys/block/$devname", qr/\Q$devname\E.*?(\d+)/, sub {
 777         my ($part, $partid) = @_;
 778
 779         if ($partid >= $newpartid) {
 780             $newpartid = $partid + 1;
 781         }
 782     });
 783
 784     $size = PVE::Tools::convert_size($size, 'b' => 'mb');
 785
 786     run_command([ $SGDISK, '-n', "$newpartid:0:+${size}M", $dev ],
 787                 errmsg => "error creating partition '$newpartid' on '$dev'");
 788
 789     my $partition;
 790
 791     # loop again to detect the real partition device which does not always follow
 792     # a strict $devname$partition scheme like /dev/nvme0n1 -> /dev/nvme0n1p1
 793     dir_glob_foreach("/sys/block/$devname", qr/\Q$devname\E.*$newpartid/, sub {
 794         my ($part) = @_;
 795
 796         $partition = "/dev/$part";
 797     });
 798
 799     return $partition;
 800 }
 801
 802 # Check if a disk or any of its partitions has a holder.
 803 # Can also be called with a partition.
 804 # Expected to be called with a result of verify_blockdev_path().
 805 sub has_holder {
 806     my ($devpath) = @_;
 807
 808     my $dev = strip_dev($devpath);
 809
 810     return $devpath if !dir_is_empty("/sys/class/block/${dev}/holders");
 811
 812     my $found;
 813     dir_glob_foreach("/sys/block/${dev}", "${dev}.+", sub {
 814         my ($part) = @_;
 815         $found = "/dev/${part}" if !dir_is_empty("/sys/class/block/${part}/holders");
 816     });
 817
 818     return $found;
 819 }
 820
 821 # Basic check if a disk or any of its partitions is mounted.
 822 # Can also be called with a partition.
 823 # Expected to be called with a result of verify_blockdev_path().
 824 sub is_mounted {
 825     my ($devpath) = @_;
 826
 827     my $mounted = mounted_blockdevs();
 828
 829     return $devpath if $mounted->{$devpath};
 830
 831     my $dev = strip_dev($devpath);
 832
 833     my $found;
 834     dir_glob_foreach("/sys/block/${dev}", "${dev}.+", sub {
 835         my ($part) = @_;
 836         my $partpath = "/dev/${part}";
 837
 838         $found = $partpath if $mounted->{$partpath};
 839     });
 840
 841     return $found;
 842 }
 843
 844 # Currently only supports GPT-partitioned disks.
 845 sub change_parttype {
 846     my ($partpath, $parttype) = @_;
 847
 848     my $err = "unable to change partition type for $partpath";
 849
 850     my $partnum = get_partnum($partpath);
 851     my $blockdev = get_blockdev($partpath);
 852     my $dev = strip_dev($blockdev);
 853
 854     my $info = get_disks($dev, 1);
 855     die "$err - unable to get disk info for '$blockdev'\n" if !defined($info->{$dev});
 856     die "$err - disk '$blockdev' is not GPT partitioned\n" if !$info->{$dev}->{gpt};
 857
 858     run_command(['sgdisk', "-t${partnum}:${parttype}", $blockdev], errmsg => $err);
 859 }
 860
 861 # Wipes all labels and the first 200 MiB of a disk/partition (or the whole if it is smaller).
 862 # If called with a partition, also sets the partition type to 0x83 'Linux filesystem'.
 863 # Expected to be called with a result of verify_blockdev_path().
 864 sub wipe_blockdev {
 865     my ($devpath) = @_;
 866
 867     my $devname = basename($devpath);
 868     my $dev_size = PVE::Tools::file_get_contents("/sys/class/block/$devname/size");
 869
 870     ($dev_size) = $dev_size =~ m|(\d+)|; # untaint $dev_size
 871     die "Couldn't get the size of the device $devname\n" if !defined($dev_size);
 872
 873     my $size = ($dev_size * 512 / 1024 / 1024);
 874     my $count = ($size < 200) ? $size : 200;
 875
 876     my $to_wipe = [];
 877     dir_glob_foreach("/sys/class/block/${devname}", "${devname}.+", sub {
 878         my ($part) = @_;
 879         push $to_wipe->@*, "/dev/${part}" if -b "/dev/${part}";
 880     });
 881
 882     if (scalar($to_wipe->@*) > 0) {
 883         print "found child partitions to wipe: ". join(', ', $to_wipe->@*) ."\n";
 884     }
 885     push $to_wipe->@*, $devpath; # put actual device last
 886
 887     print "wiping block device ${devpath}\n";
 888
 889     run_command(['wipefs', '--all', $to_wipe->@*], errmsg => "error wiping '${devpath}'");
 890
 891     run_command(
 892         ['dd', 'if=/dev/zero', "of=${devpath}", 'bs=1M', 'conv=fdatasync', "count=${count}"],
 893         errmsg => "error wiping '${devpath}'",
 894     );
 895
 896     if (is_partition($devpath)) {
 897         eval { change_parttype($devpath, '8300'); };
 898         warn $@ if $@;
 899     }
 900 }
 901
 902 # FIXME: Remove once we depend on systemd >= v249.
 903 # Work around udev bug https://github.com/systemd/systemd/issues/18525 ensuring database is updated.
 904 sub udevadm_trigger {
 905     my @devs = @_;
 906
 907     return if scalar(@devs) == 0;
 908
 909     eval { run_command(['udevadm', 'trigger', @devs]); };
 910     warn $@ if $@;
 911 }
 912
 913 1;