]>
git.proxmox.com Git - ceph.git/blob - ceph/src/ceph-volume/ceph_volume/util/disk.py
5 from ceph_volume
import process
6 from ceph_volume
.api
import lvm
7 from ceph_volume
.util
.system
import get_file_contents
10 logger
= logging
.getLogger(__name__
)
13 # The blkid CLI tool has some oddities which prevents having one common call
14 # to extract the information instead of having separate utilities. The `udev`
15 # type of output is needed in older versions of blkid (v 2.23) that will not
16 # work correctly with just the ``-p`` flag to bypass the cache for example.
17 # Xenial doesn't have this problem as it uses a newer blkid version.
20 def get_partuuid(device
):
22 If a device is a partition, it will probably have a PARTUUID on it that
23 will persist and can be queried against `blkid` later to detect the actual
26 out
, err
, rc
= process
.call(
27 ['blkid', '-s', 'PARTUUID', '-o', 'value', device
]
29 return ' '.join(out
).strip()
32 def _blkid_parser(output
):
34 Parses the output from a system ``blkid`` call, requires output to be
35 produced using the ``-p`` flag which bypasses the cache, mangling the
36 names. These names are corrected to what it would look like without the
41 /dev/sdb1: UUID="62416664-cbaf-40bd-9689-10bd337379c3" TYPE="xfs" [...]
43 # first spaced separated item is garbage, gets tossed:
44 output
= ' '.join(output
.split()[1:])
45 # split again, respecting possible whitespace in quoted values
46 pairs
= output
.split('" ')
52 'PART_ENTRY_NAME': 'PARTLABEL',
53 'PART_ENTRY_UUID': 'PARTUUID',
54 'PART_ENTRY_TYPE': 'PARTTYPE',
60 column
, value
= pair
.split('=')
63 raw
[column
] = value
.strip().strip().strip('"')
65 for key
, value
in raw
.items():
66 new_key
= mapping
.get(key
)
69 processed
[new_key
] = value
76 The blkid interface to its CLI, creating an output similar to what is
77 expected from ``lsblk``. In most cases, ``lsblk()`` should be the preferred
78 method for extracting information about a device. There are some corner
79 cases where it might provide information that is otherwise unavailable.
81 The system call uses the ``-p`` flag which bypasses the cache, the caveat
82 being that the keys produced are named completely different to expected
85 For example, instead of ``PARTLABEL`` it provides a ``PART_ENTRY_NAME``.
86 A bit of translation between these known keys is done, which is why
87 ``lsblk`` should always be preferred: the output provided here is not as
88 rich, given that a translation of keys is required for a uniform interface
91 Label name to expected output chart:
93 cache bypass name expected name
97 PART_ENTRY_NAME PARTLABEL
98 PART_ENTRY_UUID PARTUUID
100 out
, err
, rc
= process
.call(
101 ['blkid', '-p', device
]
103 return _blkid_parser(' '.join(out
))
106 def get_part_entry_type(device
):
108 Parses the ``ID_PART_ENTRY_TYPE`` from the "low level" (bypasses the cache)
109 output that uses the ``udev`` type of output. This output is intended to be
110 used for udev rules, but it is useful in this case as it is the only
111 consistent way to retrieve the GUID used by ceph-disk to identify devices.
113 out
, err
, rc
= process
.call(['blkid', '-p', '-o', 'udev', device
])
115 if 'ID_PART_ENTRY_TYPE=' in line
:
116 return line
.split('=')[-1].strip()
120 def get_device_from_partuuid(partuuid
):
122 If a device has a partuuid, query blkid so that it can tell us what that
125 out
, err
, rc
= process
.call(
126 ['blkid', '-t', 'PARTUUID="%s"' % partuuid
, '-o', 'device']
128 return ' '.join(out
).strip()
131 def remove_partition(device
):
133 Removes a partition using parted
135 :param device: A ``Device()`` object
137 parent_device
= '/dev/%s' % device
.disk_api
['PKNAME']
138 udev_info
= udevadm_property(device
.abspath
)
139 partition_number
= udev_info
.get('ID_PART_ENTRY_NUMBER')
140 if not partition_number
:
141 raise RuntimeError('Unable to detect the partition number for device: %s' % device
.abspath
)
144 ['parted', parent_device
, '--script', '--', 'rm', partition_number
]
148 def _stat_is_device(stat_obj
):
150 Helper function that will interpret ``os.stat`` output directly, so that other
151 functions can call ``os.stat`` once and interpret that result several times
153 return stat
.S_ISBLK(stat_obj
)
156 def _lsblk_parser(line
):
158 Parses lines in lsblk output. Requires output to be in pair mode (``-P`` flag). Lines
159 need to be whole strings, the line gets split when processed.
161 :param line: A string, with the full line from lsblk output
163 # parse the COLUMN="value" output to construct the dictionary
164 pairs
= line
.split('" ')
168 column
, value
= pair
.split('=')
171 parsed
[column
] = value
.strip().strip().strip('"')
175 def device_family(device
):
177 Returns a list of associated devices. It assumes that ``device`` is
178 a parent device. It is up to the caller to ensure that the device being
179 used is a parent, not a partition.
181 labels
= ['NAME', 'PARTLABEL', 'TYPE']
182 command
= ['lsblk', '-P', '-p', '-o', ','.join(labels
), device
]
183 out
, err
, rc
= process
.call(command
)
186 devices
.append(_lsblk_parser(line
))
191 def udevadm_property(device
, properties
=[]):
193 Query udevadm for information about device properties.
194 Optionally pass a list of properties to return. A requested property might
195 not be returned if not present.
197 Expected output format::
198 # udevadm info --query=property --name=/dev/sda :(
203 ID_MODEL=SK_hynix_SC311_SATA_512GB
204 ID_PART_TABLE_TYPE=gpt
205 ID_PART_TABLE_UUID=c8f91d57-b26c-4de1-8884-0c9541da288c
206 ID_PATH=pci-0000:00:17.0-ata-3
207 ID_PATH_TAG=pci-0000_00_17_0-ata-3
209 ID_SERIAL=SK_hynix_SC311_SATA_512GB_MS83N71801150416A
211 USEC_INITIALIZED=16117769
214 out
= _udevadm_info(device
)
217 p
, v
= line
.split('=', 1)
218 if not properties
or p
in properties
:
223 def _udevadm_info(device
):
225 Call udevadm and return the output
227 cmd
= ['udevadm', 'info', '--query=property', device
]
228 out
, _err
, _rc
= process
.call(cmd
)
232 def lsblk(device
, columns
=None, abspath
=False):
234 Create a dictionary of identifying values for a device using ``lsblk``.
235 Each supported column is a key, in its *raw* format (all uppercase
236 usually). ``lsblk`` has support for certain "columns" (in blkid these
237 would be labels), and these columns vary between distributions and
238 ``lsblk`` versions. The newer versions support a richer set of columns,
239 while older ones were a bit limited.
241 These are a subset of lsblk columns which are known to work on both CentOS 7 and Xenial:
244 KNAME internal kernel device name
245 MAJ:MIN major:minor device number
246 FSTYPE filesystem type
247 MOUNTPOINT where the device is mounted
248 LABEL filesystem LABEL
252 MODEL device identifier
253 SIZE size of the device
254 STATE state of the device
257 MODE device node permissions
258 ALIGNMENT alignment offset
259 MIN-IO minimum I/O size
260 OPT-IO optimal I/O size
261 PHY-SEC physical sector size
262 LOG-SEC logical sector size
263 ROTA rotational device
264 SCHED I/O scheduler name
265 RQ-SIZE request queue size
267 PKNAME internal parent kernel device name
268 DISC-ALN discard alignment offset
269 DISC-GRAN discard granularity
270 DISC-MAX discard max bytes
271 DISC-ZERO discard zeroes data
273 There is a bug in ``lsblk`` where using all the available (supported)
274 columns will result in no output (!), in order to workaround this the
275 following columns have been removed from the default reporting columns:
277 * RQ-SIZE (request queue size)
278 * MIN-IO minimum I/O size
279 * OPT-IO optimal I/O size
281 These should be available however when using `columns`. For example::
283 >>> lsblk('/dev/sda1', columns=['OPT-IO'])
286 Normal CLI output, as filtered by the flags in this function will look like ::
288 $ lsblk --nodeps -P -o NAME,KNAME,MAJ:MIN,FSTYPE,MOUNTPOINT
289 NAME="sda1" KNAME="sda1" MAJ:MIN="8:1" FSTYPE="ext4" MOUNTPOINT="/"
291 :param columns: A list of columns to report as keys in its original form.
292 :param abspath: Set the flag for absolute paths on the report
295 'NAME', 'KNAME', 'MAJ:MIN', 'FSTYPE', 'MOUNTPOINT', 'LABEL', 'UUID',
296 'RO', 'RM', 'MODEL', 'SIZE', 'STATE', 'OWNER', 'GROUP', 'MODE',
297 'ALIGNMENT', 'PHY-SEC', 'LOG-SEC', 'ROTA', 'SCHED', 'TYPE', 'DISC-ALN',
298 'DISC-GRAN', 'DISC-MAX', 'DISC-ZERO', 'PKNAME', 'PARTLABEL'
300 device
= device
.rstrip('/')
301 columns
= columns
or default_columns
302 # --nodeps -> Avoid adding children/parents to the device, only give information
303 # on the actual device we are querying for
304 # -P -> Produce pairs of COLUMN="value"
305 # -p -> Return full paths to devices, not just the names, when ``abspath`` is set
306 # -o -> Use the columns specified or default ones provided by this function
307 base_command
= ['lsblk', '--nodeps', '-P']
309 base_command
.append('-p')
310 base_command
.append('-o')
311 base_command
.append(','.join(columns
))
312 base_command
.append(device
)
313 out
, err
, rc
= process
.call(base_command
)
318 return _lsblk_parser(' '.join(out
))
323 Boolean to determine if a given device is a block device (**not**
326 For example: /dev/sda would return True, but not /dev/sdc1
328 if not os
.path
.exists(dev
):
330 # use lsblk first, fall back to using stat
331 TYPE
= lsblk(dev
).get('TYPE')
333 return TYPE
in ['disk', 'mpath']
336 return _stat_is_device(os
.lstat(dev
).st_mode
)
337 if stat
.S_ISBLK(os
.lstat(dev
)):
342 def is_partition(dev
):
344 Boolean to determine if a given device is a partition, like /dev/sda1
346 if not os
.path
.exists(dev
):
348 # use lsblk first, fall back to using stat
349 TYPE
= lsblk(dev
).get('TYPE')
351 return TYPE
== 'part'
354 stat_obj
= os
.stat(dev
)
355 if _stat_is_device(stat_obj
.st_mode
):
358 major
= os
.major(stat_obj
.st_rdev
)
359 minor
= os
.minor(stat_obj
.st_rdev
)
360 if os
.path
.exists('/sys/dev/block/%d:%d/partition' % (major
, minor
)):
365 class BaseFloatUnit(float):
367 Base class to support float representations of size values. Suffix is
368 computed on child classes by inspecting the class name
372 return "<%s(%s)>" % (self
.__class
__.__name
__, self
.__float
__())
375 return "{size:.2f} {suffix}".format(
376 size
=self
.__float
__(),
377 suffix
=self
.__class
__.__name
__.split('Float')[-1]
381 return int(self
.real
)
387 class FloatB(BaseFloatUnit
):
391 class FloatMB(BaseFloatUnit
):
395 class FloatGB(BaseFloatUnit
):
399 class FloatKB(BaseFloatUnit
):
403 class FloatTB(BaseFloatUnit
):
409 Helper to provide an interface for different sizes given a single initial
410 input. Allows for comparison between different size objects, which avoids
411 the need to convert sizes before comparison (e.g. comparing megabytes
414 Common comparison operators are supported::
416 >>> hd1 = Size(gb=400)
417 >>> hd2 = Size(gb=500)
424 >>> hd1 == Size(gb=400)
427 The Size object can also be multiplied or divided::
436 Additions and subtractions are only supported between Size objects::
438 >>> Size(gb=224) - Size(gb=100)
440 >>> Size(gb=1) + Size(mb=300)
443 Can also display a human-readable representation, with automatic detection
444 on best suited unit, or alternatively, specific unit representation::
446 >>> s = Size(mb=2211)
451 >>> print("Total size: %s" % s.mb)
452 Total size: 2211.00 MB
453 >>> print("Total size: %s" % s)
458 def parse(cls
, size
):
459 if (len(size
) > 2 and
460 size
[-2].lower() in ['k', 'm', 'g', 't'] and
461 size
[-1].lower() == 'b'):
462 return cls(**{size
[-2:].lower(): float(size
[0:-2])})
463 elif size
[-1].lower() in ['b', 'k', 'm', 'g', 't']:
464 return cls(**{size
[-1].lower(): float(size
[0:-1])})
466 return cls(b
=float(size
))
469 def __init__(self
, multiplier
=1024, **kw
):
470 self
._multiplier
= multiplier
471 # create a mapping of units-to-multiplier, skip bytes as that is
472 # calculated initially always and does not need to convert
474 [('k', 'kb', 'kilobytes'), self
._multiplier
],
475 [('m', 'mb', 'megabytes'), self
._multiplier
** 2],
476 [('g', 'gb', 'gigabytes'), self
._multiplier
** 3],
477 [('t', 'tb', 'terabytes'), self
._multiplier
** 4],
479 # and mappings for units-to-formatters, including bytes and aliases for
482 [('b', 'bytes'), FloatB
],
483 [('kb', 'kilobytes'), FloatKB
],
484 [('mb', 'megabytes'), FloatMB
],
485 [('gb', 'gigabytes'), FloatGB
],
486 [('tb', 'terabytes'), FloatTB
],
488 self
._formatters
= {}
489 for key
, value
in format_aliases
:
491 self
._formatters
[alias
] = value
493 for key
, value
in aliases
:
495 self
._factors
[alias
] = value
497 for k
, v
in kw
.items():
499 # only pursue the first occurrence
502 def _convert(self
, size
, unit
):
504 Convert any size down to bytes so that other methods can rely on bytes
505 being available always, regardless of what they pass in, avoiding the
506 need for a mapping of every permutation.
508 if unit
in ['b', 'bytes']:
511 factor
= self
._factors
[unit
]
512 self
._b
= float(size
* factor
)
514 def _get_best_format(self
):
516 Go through all the supported units, and use the first one that is less
517 than 1024. This allows to represent size in the most readable format
520 for unit
in ['b', 'kb', 'mb', 'gb', 'tb']:
521 if getattr(self
, unit
) > 1024:
523 return getattr(self
, unit
)
526 return "<Size(%s)>" % self
._get
_best
_format
()
529 return "%s" % self
._get
_best
_format
()
531 def __format__(self
, spec
):
532 return str(self
._get
_best
_format
()).__format
__(spec
)
540 def __lt__(self
, other
):
541 if isinstance(other
, Size
):
542 return self
._b
< other
._b
544 return self
.b
< other
546 def __le__(self
, other
):
547 if isinstance(other
, Size
):
548 return self
._b
<= other
._b
550 return self
.b
<= other
552 def __eq__(self
, other
):
553 if isinstance(other
, Size
):
554 return self
._b
== other
._b
556 return self
.b
== other
558 def __ne__(self
, other
):
559 if isinstance(other
, Size
):
560 return self
._b
!= other
._b
562 return self
.b
!= other
564 def __ge__(self
, other
):
565 if isinstance(other
, Size
):
566 return self
._b
>= other
._b
568 return self
.b
>= other
570 def __gt__(self
, other
):
571 if isinstance(other
, Size
):
572 return self
._b
> other
._b
574 return self
.b
> other
576 def __add__(self
, other
):
577 if isinstance(other
, Size
):
578 _b
= self
._b
+ other
._b
580 raise TypeError('Cannot add "Size" object with int')
582 def __sub__(self
, other
):
583 if isinstance(other
, Size
):
584 _b
= self
._b
- other
._b
586 raise TypeError('Cannot subtract "Size" object from int')
588 def __mul__(self
, other
):
589 if isinstance(other
, Size
):
590 raise TypeError('Cannot multiply with "Size" object')
594 def __truediv__(self
, other
):
595 if isinstance(other
, Size
):
596 return self
._b
/ other
._b
600 def __div__(self
, other
):
601 if isinstance(other
, Size
):
602 return self
._b
/ other
._b
609 def __nonzero__(self
):
610 return self
.__bool
__()
612 def __getattr__(self
, unit
):
614 Calculate units on the fly, relies on the fact that ``bytes`` has been
615 converted at instantiation. Units that don't exist will trigger an
619 formatter
= self
._formatters
[unit
]
621 raise AttributeError('Size object has not attribute "%s"' % unit
)
622 if unit
in ['b', 'bytes']:
623 return formatter(self
._b
)
625 factor
= self
._factors
[unit
]
627 raise AttributeError('Size object has not attribute "%s"' % unit
)
628 return formatter(float(self
._b
) / factor
)
631 def human_readable_size(size
):
633 Take a size in bytes, and transform it into a human readable size with up
634 to two decimals of precision.
636 suffixes
= ['B', 'KB', 'MB', 'GB', 'TB']
641 return "{size:.2f} {suffix}".format(
643 suffix
=suffixes
[suffix_index
])
646 def size_from_human_readable(s
):
648 Takes a human readable string and converts into a Size. If no unit is
649 passed, bytes is assumed.
651 s
= s
.replace(' ', '')
653 return Size(b
=float(s
))
655 if s
[-1].lower() == 't':
657 if s
[-1].lower() == 'g':
659 if s
[-1].lower() == 'm':
661 if s
[-1].lower() == 'k':
666 def get_partitions_facts(sys_block_path
):
667 partition_metadata
= {}
668 for folder
in os
.listdir(sys_block_path
):
669 folder_path
= os
.path
.join(sys_block_path
, folder
)
670 if os
.path
.exists(os
.path
.join(folder_path
, 'partition')):
671 contents
= get_file_contents(os
.path
.join(folder_path
, 'partition'))
675 part_sys_block_path
= os
.path
.join(sys_block_path
, partname
)
677 part
['start'] = get_file_contents(part_sys_block_path
+ "/start", 0)
678 part
['sectors'] = get_file_contents(part_sys_block_path
+ "/size", 0)
680 part
['sectorsize'] = get_file_contents(
681 part_sys_block_path
+ "/queue/logical_block_size")
682 if not part
['sectorsize']:
683 part
['sectorsize'] = get_file_contents(
684 part_sys_block_path
+ "/queue/hw_sector_size", 512)
685 part
['size'] = float(part
['sectors']) * 512
686 part
['human_readable_size'] = human_readable_size(float(part
['sectors']) * 512)
688 for holder
in os
.listdir(part_sys_block_path
+ '/holders'):
689 part
['holders'].append(holder
)
691 partition_metadata
[partname
] = part
692 return partition_metadata
695 def is_mapper_device(device_name
):
696 return device_name
.startswith(('/dev/mapper', '/dev/dm-'))
699 def is_locked_raw_device(disk_path
):
701 A device can be locked by a third party software like a database.
702 To detect that case, the device is opened in Read/Write and exclusive mode
704 open_flags
= (os
.O_RDWR | os
.O_EXCL
)
709 fd
= os
.open(disk_path
, open_flags
, open_mode
)
721 def get_block_devs_lsblk():
723 This returns a list of lists with 3 items per inner list.
724 KNAME - reflects the kernel device name , for example /dev/sda or /dev/dm-0
725 NAME - the device name, for example /dev/sda or
726 /dev/mapper/<vg_name>-<lv_name>
727 TYPE - the block device type: disk, partition, lvm and such
730 cmd
= ['lsblk', '-plno', 'KNAME,NAME,TYPE']
731 stdout
, stderr
, rc
= process
.call(cmd
)
732 # lsblk returns 1 on failure
734 raise OSError('lsblk returned failure, stderr: {}'.format(stderr
))
735 return [re
.split(r
'\s+', line
) for line
in stdout
]
738 def get_devices(_sys_block_path
='/sys/block'):
740 Captures all available block devices as reported by lsblk.
741 Additional interesting metadata like sectors, size, vendor,
742 solid/rotational, etc. is collected from /sys/block/<device>
744 Returns a dictionary, where keys are the full paths to devices.
746 ..note:: loop devices, removable media, and logical volumes are never included.
751 block_devs
= get_block_devs_lsblk()
753 for block
in block_devs
:
754 devname
= os
.path
.basename(block
[0])
756 if block
[2] not in ['disk', 'mpath']:
758 sysdir
= os
.path
.join(_sys_block_path
, devname
)
761 # If the mapper device is a logical volume it gets excluded
762 if is_mapper_device(diskname
):
763 if lvm
.get_device_lvs(diskname
):
766 # all facts that have no defaults
767 # (<name>, <path relative to _sys_block_path>)
768 facts
= [('removable', 'removable'),
770 ('vendor', 'device/vendor'),
771 ('model', 'device/model'),
772 ('rev', 'device/rev'),
773 ('sas_address', 'device/sas_address'),
774 ('sas_device_handle', 'device/sas_device_handle'),
775 ('support_discard', 'queue/discard_granularity'),
776 ('rotational', 'queue/rotational'),
777 ('nr_requests', 'queue/nr_requests'),
779 for key
, file_
in facts
:
780 metadata
[key
] = get_file_contents(os
.path
.join(sysdir
, file_
))
782 metadata
['scheduler_mode'] = ""
783 scheduler
= get_file_contents(sysdir
+ "/queue/scheduler")
784 if scheduler
is not None:
785 m
= re
.match(r
".*?(\[(.*)\])", scheduler
)
787 metadata
['scheduler_mode'] = m
.group(2)
789 metadata
['partitions'] = get_partitions_facts(sysdir
)
791 size
= get_file_contents(os
.path
.join(sysdir
, 'size'), 0)
793 metadata
['sectors'] = get_file_contents(os
.path
.join(sysdir
, 'sectors'), 0)
794 fallback_sectorsize
= get_file_contents(sysdir
+ "/queue/hw_sector_size", 512)
795 metadata
['sectorsize'] = get_file_contents(sysdir
+
796 "/queue/logical_block_size",
798 metadata
['size'] = float(size
) * 512
799 metadata
['human_readable_size'] = human_readable_size(metadata
['size'])
800 metadata
['path'] = diskname
801 metadata
['locked'] = is_locked_raw_device(metadata
['path'])
803 device_facts
[diskname
] = metadata