ceph/src/ceph-volume/ceph_volume/util/disk.py

   1 import logging
   2 import os
   3 import re
   4 import stat
   5 import time
   6 from ceph_volume import process
   7 from ceph_volume.api import lvm
   8 from ceph_volume.util.system import get_file_contents
   9
  10
  11 logger = logging.getLogger(__name__)
  12
  13
  14 # The blkid CLI tool has some oddities which prevents having one common call
  15 # to extract the information instead of having separate utilities. The `udev`
  16 # type of output is needed in older versions of blkid (v 2.23) that will not
  17 # work correctly with just the ``-p`` flag to bypass the cache for example.
  18 # Xenial doesn't have this problem as it uses a newer blkid version.
  19
  20
  21 def get_partuuid(device):
  22     """
  23     If a device is a partition, it will probably have a PARTUUID on it that
  24     will persist and can be queried against `blkid` later to detect the actual
  25     device
  26     """
  27     out, err, rc = process.call(
  28         ['blkid', '-c', '/dev/null', '-s', 'PARTUUID', '-o', 'value', device]
  29     )
  30     return ' '.join(out).strip()
  31
  32
  33 def _blkid_parser(output):
  34     """
  35     Parses the output from a system ``blkid`` call, requires output to be
  36     produced using the ``-p`` flag which bypasses the cache, mangling the
  37     names. These names are corrected to what it would look like without the
  38     ``-p`` flag.
  39
  40     Normal output::
  41
  42         /dev/sdb1: UUID="62416664-cbaf-40bd-9689-10bd337379c3" TYPE="xfs" [...]
  43     """
  44     # first spaced separated item is garbage, gets tossed:
  45     output = ' '.join(output.split()[1:])
  46     # split again, respecting possible whitespace in quoted values
  47     pairs = output.split('" ')
  48     raw = {}
  49     processed = {}
  50     mapping = {
  51         'UUID': 'UUID',
  52         'TYPE': 'TYPE',
  53         'PART_ENTRY_NAME': 'PARTLABEL',
  54         'PART_ENTRY_UUID': 'PARTUUID',
  55         'PART_ENTRY_TYPE': 'PARTTYPE',
  56         'PTTYPE': 'PTTYPE',
  57     }
  58
  59     for pair in pairs:
  60         try:
  61             column, value = pair.split('=')
  62         except ValueError:
  63             continue
  64         raw[column] = value.strip().strip().strip('"')
  65
  66     for key, value in raw.items():
  67         new_key = mapping.get(key)
  68         if not new_key:
  69             continue
  70         processed[new_key] = value
  71
  72     return processed
  73
  74
  75 def blkid(device):
  76     """
  77     The blkid interface to its CLI, creating an output similar to what is
  78     expected from ``lsblk``. In most cases, ``lsblk()`` should be the preferred
  79     method for extracting information about a device. There are some corner
  80     cases where it might provide information that is otherwise unavailable.
  81
  82     The system call uses the ``-p`` flag which bypasses the cache, the caveat
  83     being that the keys produced are named completely different to expected
  84     names.
  85
  86     For example, instead of ``PARTLABEL`` it provides a ``PART_ENTRY_NAME``.
  87     A bit of translation between these known keys is done, which is why
  88     ``lsblk`` should always be preferred: the output provided here is not as
  89     rich, given that a translation of keys is required for a uniform interface
  90     with the ``-p`` flag.
  91
  92     Label name to expected output chart:
  93
  94     cache bypass name               expected name
  95
  96     UUID                            UUID
  97     TYPE                            TYPE
  98     PART_ENTRY_NAME                 PARTLABEL
  99     PART_ENTRY_UUID                 PARTUUID
 100     """
 101     out, err, rc = process.call(
 102         ['blkid', '-c', '/dev/null', '-p', device]
 103     )
 104     return _blkid_parser(' '.join(out))
 105
 106
 107 def get_part_entry_type(device):
 108     """
 109     Parses the ``ID_PART_ENTRY_TYPE`` from the "low level" (bypasses the cache)
 110     output that uses the ``udev`` type of output. This output is intended to be
 111     used for udev rules, but it is useful in this case as it is the only
 112     consistent way to retrieve the GUID used by ceph-disk to identify devices.
 113     """
 114     out, err, rc = process.call(['blkid', '-c', '/dev/null', '-p', '-o', 'udev', device])
 115     for line in out:
 116         if 'ID_PART_ENTRY_TYPE=' in line:
 117             return line.split('=')[-1].strip()
 118     return ''
 119
 120
 121 def get_device_from_partuuid(partuuid):
 122     """
 123     If a device has a partuuid, query blkid so that it can tell us what that
 124     device is
 125     """
 126     out, err, rc = process.call(
 127         ['blkid', '-c', '/dev/null', '-t', 'PARTUUID="%s"' % partuuid, '-o', 'device']
 128     )
 129     return ' '.join(out).strip()
 130
 131
 132 def remove_partition(device):
 133     """
 134     Removes a partition using parted
 135
 136     :param device: A ``Device()`` object
 137     """
 138     # Sometimes there's a race condition that makes 'ID_PART_ENTRY_NUMBER' be not present
 139     # in the output of `udevadm info --query=property`.
 140     # Probably not ideal and not the best fix but this allows to get around that issue.
 141     # The idea is to make it retry multiple times before actually failing.
 142     for i in range(10):
 143         udev_info = udevadm_property(device.path)
 144         partition_number = udev_info.get('ID_PART_ENTRY_NUMBER')
 145         if partition_number:
 146             break
 147         time.sleep(0.2)
 148     if not partition_number:
 149         raise RuntimeError('Unable to detect the partition number for device: %s' % device.path)
 150
 151     process.run(
 152         ['parted', device.parent_device, '--script', '--', 'rm', partition_number]
 153     )
 154
 155
 156 def _stat_is_device(stat_obj):
 157     """
 158     Helper function that will interpret ``os.stat`` output directly, so that other
 159     functions can call ``os.stat`` once and interpret that result several times
 160     """
 161     return stat.S_ISBLK(stat_obj)
 162
 163
 164 def _lsblk_parser(line):
 165     """
 166     Parses lines in lsblk output. Requires output to be in pair mode (``-P`` flag). Lines
 167     need to be whole strings, the line gets split when processed.
 168
 169     :param line: A string, with the full line from lsblk output
 170     """
 171     # parse the COLUMN="value" output to construct the dictionary
 172     pairs = line.split('" ')
 173     parsed = {}
 174     for pair in pairs:
 175         try:
 176             column, value = pair.split('=')
 177         except ValueError:
 178             continue
 179         parsed[column] = value.strip().strip().strip('"')
 180     return parsed
 181
 182
 183 def device_family(device):
 184     """
 185     Returns a list of associated devices. It assumes that ``device`` is
 186     a parent device. It is up to the caller to ensure that the device being
 187     used is a parent, not a partition.
 188     """
 189     labels = ['NAME', 'PARTLABEL', 'TYPE']
 190     command = ['lsblk', '-P', '-p', '-o', ','.join(labels), device]
 191     out, err, rc = process.call(command)
 192     devices = []
 193     for line in out:
 194         devices.append(_lsblk_parser(line))
 195
 196     return devices
 197
 198
 199 def udevadm_property(device, properties=[]):
 200     """
 201     Query udevadm for information about device properties.
 202     Optionally pass a list of properties to return. A requested property might
 203     not be returned if not present.
 204
 205     Expected output format::
 206         # udevadm info --query=property --name=/dev/sda                                  :(
 207         DEVNAME=/dev/sda
 208         DEVTYPE=disk
 209         ID_ATA=1
 210         ID_BUS=ata
 211         ID_MODEL=SK_hynix_SC311_SATA_512GB
 212         ID_PART_TABLE_TYPE=gpt
 213         ID_PART_TABLE_UUID=c8f91d57-b26c-4de1-8884-0c9541da288c
 214         ID_PATH=pci-0000:00:17.0-ata-3
 215         ID_PATH_TAG=pci-0000_00_17_0-ata-3
 216         ID_REVISION=70000P10
 217         ID_SERIAL=SK_hynix_SC311_SATA_512GB_MS83N71801150416A
 218         TAGS=:systemd:
 219         USEC_INITIALIZED=16117769
 220         ...
 221     """
 222     out = _udevadm_info(device)
 223     ret = {}
 224     for line in out:
 225         p, v = line.split('=', 1)
 226         if not properties or p in properties:
 227             ret[p] = v
 228     return ret
 229
 230
 231 def _udevadm_info(device):
 232     """
 233     Call udevadm and return the output
 234     """
 235     cmd = ['udevadm', 'info', '--query=property', device]
 236     out, _err, _rc = process.call(cmd)
 237     return out
 238
 239
 240 def lsblk(device, columns=None, abspath=False):
 241     result = []
 242     if not os.path.isdir(device):
 243         result = lsblk_all(device=device,
 244                            columns=columns,
 245                            abspath=abspath)
 246     if not result:
 247         logger.debug(f"{device} not found is lsblk report")
 248         return {}
 249
 250     return result[0]
 251
 252 def lsblk_all(device='', columns=None, abspath=False):
 253     """
 254     Create a dictionary of identifying values for a device using ``lsblk``.
 255     Each supported column is a key, in its *raw* format (all uppercase
 256     usually).  ``lsblk`` has support for certain "columns" (in blkid these
 257     would be labels), and these columns vary between distributions and
 258     ``lsblk`` versions. The newer versions support a richer set of columns,
 259     while older ones were a bit limited.
 260
 261     These are a subset of lsblk columns which are known to work on both CentOS 7 and Xenial:
 262
 263          NAME  device name
 264         KNAME  internal kernel device name
 265         PKNAME internal kernel parent device name
 266       MAJ:MIN  major:minor device number
 267        FSTYPE  filesystem type
 268    MOUNTPOINT  where the device is mounted
 269         LABEL  filesystem LABEL
 270          UUID  filesystem UUID
 271            RO  read-only device
 272            RM  removable device
 273         MODEL  device identifier
 274          SIZE  size of the device
 275         STATE  state of the device
 276         OWNER  user name
 277         GROUP  group name
 278          MODE  device node permissions
 279     ALIGNMENT  alignment offset
 280        MIN-IO  minimum I/O size
 281        OPT-IO  optimal I/O size
 282       PHY-SEC  physical sector size
 283       LOG-SEC  logical sector size
 284          ROTA  rotational device
 285         SCHED  I/O scheduler name
 286       RQ-SIZE  request queue size
 287          TYPE  device type
 288       PKNAME   internal parent kernel device name
 289      DISC-ALN  discard alignment offset
 290     DISC-GRAN  discard granularity
 291      DISC-MAX  discard max bytes
 292     DISC-ZERO  discard zeroes data
 293
 294     There is a bug in ``lsblk`` where using all the available (supported)
 295     columns will result in no output (!), in order to workaround this the
 296     following columns have been removed from the default reporting columns:
 297
 298     * RQ-SIZE (request queue size)
 299     * MIN-IO  minimum I/O size
 300     * OPT-IO  optimal I/O size
 301
 302     These should be available however when using `columns`. For example::
 303
 304         >>> lsblk('/dev/sda1', columns=['OPT-IO'])
 305         {'OPT-IO': '0'}
 306
 307     Normal CLI output, as filtered by the flags in this function will look like ::
 308
 309         $ lsblk -P -o NAME,KNAME,PKNAME,MAJ:MIN,FSTYPE,MOUNTPOINT
 310         NAME="sda1" KNAME="sda1" MAJ:MIN="8:1" FSTYPE="ext4" MOUNTPOINT="/"
 311
 312     :param columns: A list of columns to report as keys in its original form.
 313     :param abspath: Set the flag for absolute paths on the report
 314     """
 315     default_columns = [
 316         'NAME', 'KNAME', 'PKNAME', 'MAJ:MIN', 'FSTYPE', 'MOUNTPOINT', 'LABEL',
 317         'UUID', 'RO', 'RM', 'MODEL', 'SIZE', 'STATE', 'OWNER', 'GROUP', 'MODE',
 318         'ALIGNMENT', 'PHY-SEC', 'LOG-SEC', 'ROTA', 'SCHED', 'TYPE', 'DISC-ALN',
 319         'DISC-GRAN', 'DISC-MAX', 'DISC-ZERO', 'PKNAME', 'PARTLABEL'
 320     ]
 321     columns = columns or default_columns
 322     # -P       -> Produce pairs of COLUMN="value"
 323     # -p       -> Return full paths to devices, not just the names, when ``abspath`` is set
 324     # -o       -> Use the columns specified or default ones provided by this function
 325     base_command = ['lsblk', '-P']
 326     if abspath:
 327         base_command.append('-p')
 328     base_command.append('-o')
 329     base_command.append(','.join(columns))
 330     if device:
 331         base_command.append('--nodeps')
 332         base_command.append(device)
 333
 334     out, err, rc = process.call(base_command)
 335
 336     if rc != 0:
 337         raise RuntimeError(f"Error: {err}")
 338
 339     result = []
 340
 341     for line in out:
 342         result.append(_lsblk_parser(line))
 343
 344     return result
 345
 346
 347 def is_device(dev):
 348     """
 349     Boolean to determine if a given device is a block device (**not**
 350     a partition!)
 351
 352     For example: /dev/sda would return True, but not /dev/sdc1
 353     """
 354     if not os.path.exists(dev):
 355         return False
 356     if not dev.startswith('/dev/'):
 357         return False
 358     if dev[len('/dev/'):].startswith('loop'):
 359         if not allow_loop_devices():
 360             return False
 361
 362     # fallback to stat
 363     return _stat_is_device(os.lstat(dev).st_mode)
 364
 365
 366 def is_partition(dev):
 367     """
 368     Boolean to determine if a given device is a partition, like /dev/sda1
 369     """
 370     if not os.path.exists(dev):
 371         return False
 372     # use lsblk first, fall back to using stat
 373     TYPE = lsblk(dev).get('TYPE')
 374     if TYPE:
 375         return TYPE == 'part'
 376
 377     # fallback to stat
 378     stat_obj = os.stat(dev)
 379     if _stat_is_device(stat_obj.st_mode):
 380         return False
 381
 382     major = os.major(stat_obj.st_rdev)
 383     minor = os.minor(stat_obj.st_rdev)
 384     if os.path.exists('/sys/dev/block/%d:%d/partition' % (major, minor)):
 385         return True
 386     return False
 387
 388
 389 def is_ceph_rbd(dev):
 390     """
 391     Boolean to determine if a given device is a ceph RBD device, like /dev/rbd0
 392     """
 393     return dev.startswith(('/dev/rbd'))
 394
 395
 396 class BaseFloatUnit(float):
 397     """
 398     Base class to support float representations of size values. Suffix is
 399     computed on child classes by inspecting the class name
 400     """
 401
 402     def __repr__(self):
 403         return "<%s(%s)>" % (self.__class__.__name__, self.__float__())
 404
 405     def __str__(self):
 406         return "{size:.2f} {suffix}".format(
 407             size=self.__float__(),
 408             suffix=self.__class__.__name__.split('Float')[-1]
 409         )
 410
 411     def as_int(self):
 412         return int(self.real)
 413
 414     def as_float(self):
 415         return self.real
 416
 417
 418 class FloatB(BaseFloatUnit):
 419     pass
 420
 421
 422 class FloatMB(BaseFloatUnit):
 423     pass
 424
 425
 426 class FloatGB(BaseFloatUnit):
 427     pass
 428
 429
 430 class FloatKB(BaseFloatUnit):
 431     pass
 432
 433
 434 class FloatTB(BaseFloatUnit):
 435     pass
 436
 437 class FloatPB(BaseFloatUnit):
 438     pass
 439
 440 class Size(object):
 441     """
 442     Helper to provide an interface for different sizes given a single initial
 443     input. Allows for comparison between different size objects, which avoids
 444     the need to convert sizes before comparison (e.g. comparing megabytes
 445     against gigabytes).
 446
 447     Common comparison operators are supported::
 448
 449         >>> hd1 = Size(gb=400)
 450         >>> hd2 = Size(gb=500)
 451         >>> hd1 > hd2
 452         False
 453         >>> hd1 < hd2
 454         True
 455         >>> hd1 == hd2
 456         False
 457         >>> hd1 == Size(gb=400)
 458         True
 459
 460     The Size object can also be multiplied or divided::
 461
 462         >>> hd1
 463         <Size(400.00 GB)>
 464         >>> hd1 * 2
 465         <Size(800.00 GB)>
 466         >>> hd1
 467         <Size(800.00 GB)>
 468
 469     Additions and subtractions are only supported between Size objects::
 470
 471         >>> Size(gb=224) - Size(gb=100)
 472         <Size(124.00 GB)>
 473         >>> Size(gb=1) + Size(mb=300)
 474         <Size(1.29 GB)>
 475
 476     Can also display a human-readable representation, with automatic detection
 477     on best suited unit, or alternatively, specific unit representation::
 478
 479         >>> s = Size(mb=2211)
 480         >>> s
 481         <Size(2.16 GB)>
 482         >>> s.mb
 483         <FloatMB(2211.0)>
 484         >>> print("Total size: %s" % s.mb)
 485         Total size: 2211.00 MB
 486         >>> print("Total size: %s" % s)
 487         Total size: 2.16 GB
 488     """
 489
 490     @classmethod
 491     def parse(cls, size):
 492         if (len(size) > 2 and
 493             size[-2].lower() in ['k', 'm', 'g', 't', 'p'] and
 494             size[-1].lower() == 'b'):
 495             return cls(**{size[-2:].lower(): float(size[0:-2])})
 496         elif size[-1].lower() in ['b', 'k', 'm', 'g', 't', 'p']:
 497             return cls(**{size[-1].lower(): float(size[0:-1])})
 498         else:
 499             return cls(b=float(size))
 500
 501
 502     def __init__(self, multiplier=1024, **kw):
 503         self._multiplier = multiplier
 504         # create a mapping of units-to-multiplier, skip bytes as that is
 505         # calculated initially always and does not need to convert
 506         aliases = [
 507             [('k', 'kb', 'kilobytes'), self._multiplier],
 508             [('m', 'mb', 'megabytes'), self._multiplier ** 2],
 509             [('g', 'gb', 'gigabytes'), self._multiplier ** 3],
 510             [('t', 'tb', 'terabytes'), self._multiplier ** 4],
 511             [('p', 'pb', 'petabytes'), self._multiplier ** 5]
 512         ]
 513         # and mappings for units-to-formatters, including bytes and aliases for
 514         # each
 515         format_aliases = [
 516             [('b', 'bytes'), FloatB],
 517             [('kb', 'kilobytes'), FloatKB],
 518             [('mb', 'megabytes'), FloatMB],
 519             [('gb', 'gigabytes'), FloatGB],
 520             [('tb', 'terabytes'), FloatTB],
 521             [('pb', 'petabytes'), FloatPB],
 522         ]
 523         self._formatters = {}
 524         for key, value in format_aliases:
 525             for alias in key:
 526                 self._formatters[alias] = value
 527         self._factors = {}
 528         for key, value in aliases:
 529             for alias in key:
 530                 self._factors[alias] = value
 531
 532         for k, v in kw.items():
 533             self._convert(v, k)
 534             # only pursue the first occurrence
 535             break
 536
 537     def _convert(self, size, unit):
 538         """
 539         Convert any size down to bytes so that other methods can rely on bytes
 540         being available always, regardless of what they pass in, avoiding the
 541         need for a mapping of every permutation.
 542         """
 543         if unit in ['b', 'bytes']:
 544             self._b = size
 545             return
 546         factor = self._factors[unit]
 547         self._b = float(size * factor)
 548
 549     def _get_best_format(self):
 550         """
 551         Go through all the supported units, and use the first one that is less
 552         than 1024. This allows to represent size in the most readable format
 553         available
 554         """
 555         for unit in ['b', 'kb', 'mb', 'gb', 'tb', 'pb']:
 556             if getattr(self, unit) > 1024:
 557                 continue
 558             return getattr(self, unit)
 559
 560     def __repr__(self):
 561         return "<Size(%s)>" % self._get_best_format()
 562
 563     def __str__(self):
 564         return "%s" % self._get_best_format()
 565
 566     def __format__(self, spec):
 567         return str(self._get_best_format()).__format__(spec)
 568
 569     def __int__(self):
 570         return int(self._b)
 571
 572     def __float__(self):
 573         return self._b
 574
 575     def __lt__(self, other):
 576         if isinstance(other, Size):
 577             return self._b < other._b
 578         else:
 579             return self.b < other
 580
 581     def __le__(self, other):
 582         if isinstance(other, Size):
 583             return self._b <= other._b
 584         else:
 585             return self.b <= other
 586
 587     def __eq__(self, other):
 588         if isinstance(other, Size):
 589             return self._b == other._b
 590         else:
 591             return self.b == other
 592
 593     def __ne__(self, other):
 594         if isinstance(other, Size):
 595             return self._b != other._b
 596         else:
 597             return self.b != other
 598
 599     def __ge__(self, other):
 600         if isinstance(other, Size):
 601             return self._b >= other._b
 602         else:
 603             return self.b >= other
 604
 605     def __gt__(self, other):
 606         if isinstance(other, Size):
 607             return self._b > other._b
 608         else:
 609             return self.b > other
 610
 611     def __add__(self, other):
 612         if isinstance(other, Size):
 613             _b = self._b + other._b
 614             return Size(b=_b)
 615         raise TypeError('Cannot add "Size" object with int')
 616
 617     def __sub__(self, other):
 618         if isinstance(other, Size):
 619             _b = self._b - other._b
 620             return Size(b=_b)
 621         raise TypeError('Cannot subtract "Size" object from int')
 622
 623     def __mul__(self, other):
 624         if isinstance(other, Size):
 625             raise TypeError('Cannot multiply with "Size" object')
 626         _b = self._b * other
 627         return Size(b=_b)
 628
 629     def __truediv__(self, other):
 630         if isinstance(other, Size):
 631             return self._b / other._b
 632         _b = self._b / other
 633         return Size(b=_b)
 634
 635     def __div__(self, other):
 636         if isinstance(other, Size):
 637             return self._b / other._b
 638         _b = self._b / other
 639         return Size(b=_b)
 640
 641     def __bool__(self):
 642         return self.b != 0
 643
 644     def __nonzero__(self):
 645         return self.__bool__()
 646
 647     def __getattr__(self, unit):
 648         """
 649         Calculate units on the fly, relies on the fact that ``bytes`` has been
 650         converted at instantiation. Units that don't exist will trigger an
 651         ``AttributeError``
 652         """
 653         try:
 654             formatter = self._formatters[unit]
 655         except KeyError:
 656             raise AttributeError('Size object has not attribute "%s"' % unit)
 657         if unit in ['b', 'bytes']:
 658             return formatter(self._b)
 659         try:
 660             factor = self._factors[unit]
 661         except KeyError:
 662             raise AttributeError('Size object has not attribute "%s"' % unit)
 663         return formatter(float(self._b) / factor)
 664
 665
 666 def human_readable_size(size):
 667     """
 668     Take a size in bytes, and transform it into a human readable size with up
 669     to two decimals of precision.
 670     """
 671     suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
 672     for suffix in suffixes:
 673         if size >= 1024:
 674             size = size / 1024
 675         else:
 676             break
 677     return "{size:.2f} {suffix}".format(
 678         size=size,
 679         suffix=suffix)
 680
 681
 682 def size_from_human_readable(s):
 683     """
 684     Takes a human readable string and converts into a Size. If no unit is
 685     passed, bytes is assumed.
 686     """
 687     s = s.replace(' ', '')
 688     if s[-1].isdigit():
 689         return Size(b=float(s))
 690     n = float(s[:-1])
 691     if s[-1].lower() == 'p':
 692         return Size(pb=n)
 693     if s[-1].lower() == 't':
 694         return Size(tb=n)
 695     if s[-1].lower() == 'g':
 696         return Size(gb=n)
 697     if s[-1].lower() == 'm':
 698         return Size(mb=n)
 699     if s[-1].lower() == 'k':
 700         return Size(kb=n)
 701     return None
 702
 703
 704 def get_partitions_facts(sys_block_path):
 705     partition_metadata = {}
 706     for folder in os.listdir(sys_block_path):
 707         folder_path = os.path.join(sys_block_path, folder)
 708         if os.path.exists(os.path.join(folder_path, 'partition')):
 709             contents = get_file_contents(os.path.join(folder_path, 'partition'))
 710             if contents:
 711                 part = {}
 712                 partname = folder
 713                 part_sys_block_path = os.path.join(sys_block_path, partname)
 714
 715                 part['start'] = get_file_contents(part_sys_block_path + "/start", 0)
 716                 part['sectors'] = get_file_contents(part_sys_block_path + "/size", 0)
 717
 718                 part['sectorsize'] = get_file_contents(
 719                     part_sys_block_path + "/queue/logical_block_size")
 720                 if not part['sectorsize']:
 721                     part['sectorsize'] = get_file_contents(
 722                         part_sys_block_path + "/queue/hw_sector_size", 512)
 723                 part['size'] = float(part['sectors']) * 512
 724                 part['human_readable_size'] = human_readable_size(float(part['sectors']) * 512)
 725                 part['holders'] = []
 726                 for holder in os.listdir(part_sys_block_path + '/holders'):
 727                     part['holders'].append(holder)
 728
 729                 partition_metadata[partname] = part
 730     return partition_metadata
 731
 732
 733 def is_mapper_device(device_name):
 734     return device_name.startswith(('/dev/mapper', '/dev/dm-'))
 735
 736
 737 def is_locked_raw_device(disk_path):
 738     """
 739     A device can be locked by a third party software like a database.
 740     To detect that case, the device is opened in Read/Write and exclusive mode
 741     """
 742     open_flags = (os.O_RDWR | os.O_EXCL)
 743     open_mode = 0
 744     fd = None
 745
 746     try:
 747         fd = os.open(disk_path, open_flags, open_mode)
 748     except OSError:
 749         return 1
 750
 751     try:
 752         os.close(fd)
 753     except OSError:
 754         return 1
 755
 756     return 0
 757
 758
 759 class AllowLoopDevices(object):
 760     allow = False
 761     warned = False
 762
 763     @classmethod
 764     def __call__(cls):
 765         val = os.environ.get("CEPH_VOLUME_ALLOW_LOOP_DEVICES", "false").lower()
 766         if val not in ("false", 'no', '0'):
 767             cls.allow = True
 768             if not cls.warned:
 769                 logger.warning(
 770                     "CEPH_VOLUME_ALLOW_LOOP_DEVICES is set in your "
 771                     "environment, so we will allow the use of unattached loop"
 772                     " devices as disks. This feature is intended for "
 773                     "development purposes only and will never be supported in"
 774                     " production. Issues filed based on this behavior will "
 775                     "likely be ignored."
 776                 )
 777                 cls.warned = True
 778         return cls.allow
 779
 780
 781 allow_loop_devices = AllowLoopDevices()
 782
 783
 784 def get_block_devs_sysfs(_sys_block_path='/sys/block', _sys_dev_block_path='/sys/dev/block'):
 785     def holder_inner_loop():
 786         for holder in holders:
 787             # /sys/block/sdy/holders/dm-8/dm/uuid
 788             holder_dm_type = get_file_contents(os.path.join(_sys_block_path, dev, f'holders/{holder}/dm/uuid')).split('-')[0].lower()
 789             if holder_dm_type == 'mpath':
 790                 return True
 791
 792     # First, get devices that are _not_ partitions
 793     result = list()
 794     dev_names = os.listdir(_sys_block_path)
 795     for dev in dev_names:
 796         name = kname = os.path.join("/dev", dev)
 797         if not os.path.exists(name):
 798             continue
 799         type_ = 'disk'
 800         holders = os.listdir(os.path.join(_sys_block_path, dev, 'holders'))
 801         if get_file_contents(os.path.join(_sys_block_path, dev, 'removable')) == "1":
 802             continue
 803         if holder_inner_loop():
 804             continue
 805         dm_dir_path = os.path.join(_sys_block_path, dev, 'dm')
 806         if os.path.isdir(dm_dir_path):
 807             dm_type = get_file_contents(os.path.join(dm_dir_path, 'uuid'))
 808             type_ = dm_type.split('-')[0].lower()
 809             basename = get_file_contents(os.path.join(dm_dir_path, 'name'))
 810             name = os.path.join("/dev/mapper", basename)
 811         if dev.startswith('loop'):
 812             if not allow_loop_devices():
 813                 continue
 814             # Skip loop devices that are not attached
 815             if not os.path.exists(os.path.join(_sys_block_path, dev, 'loop')):
 816                 continue
 817             type_ = 'loop'
 818         result.append([kname, name, type_])
 819     # Next, look for devices that _are_ partitions
 820     for item in os.listdir(_sys_dev_block_path):
 821         is_part = get_file_contents(os.path.join(_sys_dev_block_path, item, 'partition')) == "1"
 822         dev = os.path.basename(os.readlink(os.path.join(_sys_dev_block_path, item)))
 823         if not is_part:
 824             continue
 825         name = kname = os.path.join("/dev", dev)
 826         result.append([name, kname, "part"])
 827     return sorted(result, key=lambda x: x[0])
 828
 829
 830 def get_devices(_sys_block_path='/sys/block', device=''):
 831     """
 832     Captures all available block devices as reported by lsblk.
 833     Additional interesting metadata like sectors, size, vendor,
 834     solid/rotational, etc. is collected from /sys/block/<device>
 835
 836     Returns a dictionary, where keys are the full paths to devices.
 837
 838     ..note:: loop devices, removable media, and logical volumes are never included.
 839     """
 840
 841     device_facts = {}
 842
 843     block_devs = get_block_devs_sysfs(_sys_block_path)
 844
 845     block_types = ['disk', 'mpath']
 846     if allow_loop_devices():
 847         block_types.append('loop')
 848
 849     for block in block_devs:
 850         devname = os.path.basename(block[0])
 851         diskname = block[1]
 852         if block[2] not in block_types:
 853             continue
 854         sysdir = os.path.join(_sys_block_path, devname)
 855         metadata = {}
 856
 857         # If the device is ceph rbd it gets excluded
 858         if is_ceph_rbd(diskname):
 859             continue
 860
 861         # If the mapper device is a logical volume it gets excluded
 862         if is_mapper_device(diskname):
 863             if lvm.get_device_lvs(diskname):
 864                 continue
 865
 866         # all facts that have no defaults
 867         # (<name>, <path relative to _sys_block_path>)
 868         facts = [('removable', 'removable'),
 869                  ('ro', 'ro'),
 870                  ('vendor', 'device/vendor'),
 871                  ('model', 'device/model'),
 872                  ('rev', 'device/rev'),
 873                  ('sas_address', 'device/sas_address'),
 874                  ('sas_device_handle', 'device/sas_device_handle'),
 875                  ('support_discard', 'queue/discard_granularity'),
 876                  ('rotational', 'queue/rotational'),
 877                  ('nr_requests', 'queue/nr_requests'),
 878                 ]
 879         for key, file_ in facts:
 880             metadata[key] = get_file_contents(os.path.join(sysdir, file_))
 881
 882         device_slaves = os.listdir(os.path.join(sysdir, 'slaves'))
 883         if device_slaves:
 884             metadata['device_nodes'] = ','.join(device_slaves)
 885         else:
 886             metadata['device_nodes'] = devname
 887
 888         metadata['scheduler_mode'] = ""
 889         scheduler = get_file_contents(sysdir + "/queue/scheduler")
 890         if scheduler is not None:
 891             m = re.match(r".*?(\[(.*)\])", scheduler)
 892             if m:
 893                 metadata['scheduler_mode'] = m.group(2)
 894
 895         metadata['partitions'] = get_partitions_facts(sysdir)
 896
 897         size = get_file_contents(os.path.join(sysdir, 'size'), 0)
 898
 899         metadata['sectors'] = get_file_contents(os.path.join(sysdir, 'sectors'), 0)
 900         fallback_sectorsize = get_file_contents(sysdir + "/queue/hw_sector_size", 512)
 901         metadata['sectorsize'] = get_file_contents(sysdir +
 902                                                    "/queue/logical_block_size",
 903                                                    fallback_sectorsize)
 904         metadata['size'] = float(size) * 512
 905         metadata['human_readable_size'] = human_readable_size(metadata['size'])
 906         metadata['path'] = diskname
 907         metadata['locked'] = is_locked_raw_device(metadata['path'])
 908         metadata['type'] = block[2]
 909
 910         device_facts[diskname] = metadata
 911     return device_facts
 912
 913 def has_bluestore_label(device_path):
 914     isBluestore = False
 915     bluestoreDiskSignature = 'bluestore block device' # 22 bytes long
 916
 917     # throws OSError on failure
 918     logger.info("opening device {} to check for BlueStore label".format(device_path))
 919     try:
 920         with open(device_path, "rb") as fd:
 921             # read first 22 bytes looking for bluestore disk signature
 922             signature = fd.read(22)
 923             if signature.decode('ascii', 'replace') == bluestoreDiskSignature:
 924                 isBluestore = True
 925     except IsADirectoryError:
 926         logger.info(f'{device_path} is a directory, skipping.')
 927
 928     return isBluestore