]>
git.proxmox.com Git - ceph.git/blob - ceph/src/ceph-volume/ceph_volume/util/disk.py
5 from ceph_volume
import process
6 from ceph_volume
.api
import lvm
7 from ceph_volume
.util
.system
import get_file_contents
10 logger
= logging
.getLogger(__name__
)
13 # The blkid CLI tool has some oddities which prevents having one common call
14 # to extract the information instead of having separate utilities. The `udev`
15 # type of output is needed in older versions of blkid (v 2.23) that will not
16 # work correctly with just the ``-p`` flag to bypass the cache for example.
17 # Xenial doesn't have this problem as it uses a newer blkid version.
20 def get_partuuid(device
):
22 If a device is a partition, it will probably have a PARTUUID on it that
23 will persist and can be queried against `blkid` later to detect the actual
26 out
, err
, rc
= process
.call(
27 ['blkid', '-s', 'PARTUUID', '-o', 'value', device
]
29 return ' '.join(out
).strip()
32 def _blkid_parser(output
):
34 Parses the output from a system ``blkid`` call, requires output to be
35 produced using the ``-p`` flag which bypasses the cache, mangling the
36 names. These names are corrected to what it would look like without the
41 /dev/sdb1: UUID="62416664-cbaf-40bd-9689-10bd337379c3" TYPE="xfs" [...]
43 # first spaced separated item is garbage, gets tossed:
44 output
= ' '.join(output
.split()[1:])
45 # split again, respecting possible whitespace in quoted values
46 pairs
= output
.split('" ')
52 'PART_ENTRY_NAME': 'PARTLABEL',
53 'PART_ENTRY_UUID': 'PARTUUID',
59 column
, value
= pair
.split('=')
62 raw
[column
] = value
.strip().strip().strip('"')
64 for key
, value
in raw
.items():
65 new_key
= mapping
.get(key
)
68 processed
[new_key
] = value
75 The blkid interface to its CLI, creating an output similar to what is
76 expected from ``lsblk``. In most cases, ``lsblk()`` should be the preferred
77 method for extracting information about a device. There are some corner
78 cases where it might provide information that is otherwise unavailable.
80 The system call uses the ``-p`` flag which bypasses the cache, the caveat
81 being that the keys produced are named completely different to expected
84 For example, instead of ``PARTLABEL`` it provides a ``PART_ENTRY_NAME``.
85 A bit of translation between these known keys is done, which is why
86 ``lsblk`` should always be preferred: the output provided here is not as
87 rich, given that a translation of keys is required for a uniform interface
90 Label name to expected output chart:
92 cache bypass name expected name
96 PART_ENTRY_NAME PARTLABEL
97 PART_ENTRY_UUID PARTUUID
99 out
, err
, rc
= process
.call(
100 ['blkid', '-p', device
]
102 return _blkid_parser(' '.join(out
))
105 def get_part_entry_type(device
):
107 Parses the ``ID_PART_ENTRY_TYPE`` from the "low level" (bypasses the cache)
108 output that uses the ``udev`` type of output. This output is intended to be
109 used for udev rules, but it is useful in this case as it is the only
110 consistent way to retrieve the GUID used by ceph-disk to identify devices.
112 out
, err
, rc
= process
.call(['blkid', '-p', '-o', 'udev', device
])
114 if 'ID_PART_ENTRY_TYPE=' in line
:
115 return line
.split('=')[-1].strip()
119 def get_device_from_partuuid(partuuid
):
121 If a device has a partuuid, query blkid so that it can tell us what that
124 out
, err
, rc
= process
.call(
125 ['blkid', '-t', 'PARTUUID="%s"' % partuuid
, '-o', 'device']
127 return ' '.join(out
).strip()
130 def _stat_is_device(stat_obj
):
132 Helper function that will interpret ``os.stat`` output directly, so that other
133 functions can call ``os.stat`` once and interpret that result several times
135 return stat
.S_ISBLK(stat_obj
)
138 def _lsblk_parser(line
):
140 Parses lines in lsblk output. Requires output to be in pair mode (``-P`` flag). Lines
141 need to be whole strings, the line gets split when processed.
143 :param line: A string, with the full line from lsblk output
145 # parse the COLUMN="value" output to construct the dictionary
146 pairs
= line
.split('" ')
150 column
, value
= pair
.split('=')
153 parsed
[column
] = value
.strip().strip().strip('"')
157 def device_family(device
):
159 Returns a list of associated devices. It assumes that ``device`` is
160 a parent device. It is up to the caller to ensure that the device being
161 used is a parent, not a partition.
163 labels
= ['NAME', 'PARTLABEL', 'TYPE']
164 command
= ['lsblk', '-P', '-p', '-o', ','.join(labels
), device
]
165 out
, err
, rc
= process
.call(command
)
168 devices
.append(_lsblk_parser(line
))
173 def lsblk(device
, columns
=None, abspath
=False):
175 Create a dictionary of identifying values for a device using ``lsblk``.
176 Each supported column is a key, in its *raw* format (all uppercase
177 usually). ``lsblk`` has support for certain "columns" (in blkid these
178 would be labels), and these columns vary between distributions and
179 ``lsblk`` versions. The newer versions support a richer set of columns,
180 while older ones were a bit limited.
182 These are a subset of lsblk columns which are known to work on both CentOS 7 and Xenial:
185 KNAME internal kernel device name
186 MAJ:MIN major:minor device number
187 FSTYPE filesystem type
188 MOUNTPOINT where the device is mounted
189 LABEL filesystem LABEL
193 MODEL device identifier
194 SIZE size of the device
195 STATE state of the device
198 MODE device node permissions
199 ALIGNMENT alignment offset
200 MIN-IO minimum I/O size
201 OPT-IO optimal I/O size
202 PHY-SEC physical sector size
203 LOG-SEC logical sector size
204 ROTA rotational device
205 SCHED I/O scheduler name
206 RQ-SIZE request queue size
208 PKNAME internal parent kernel device name
209 DISC-ALN discard alignment offset
210 DISC-GRAN discard granularity
211 DISC-MAX discard max bytes
212 DISC-ZERO discard zeroes data
214 There is a bug in ``lsblk`` where using all the available (supported)
215 columns will result in no output (!), in order to workaround this the
216 following columns have been removed from the default reporting columns:
218 * RQ-SIZE (request queue size)
219 * MIN-IO minimum I/O size
220 * OPT-IO optimal I/O size
222 These should be available however when using `columns`. For example::
224 >>> lsblk('/dev/sda1', columns=['OPT-IO'])
227 Normal CLI output, as filtered by the flags in this function will look like ::
229 $ lsblk --nodeps -P -o NAME,KNAME,MAJ:MIN,FSTYPE,MOUNTPOINT
230 NAME="sda1" KNAME="sda1" MAJ:MIN="8:1" FSTYPE="ext4" MOUNTPOINT="/"
232 :param columns: A list of columns to report as keys in its original form.
233 :param abspath: Set the flag for absolute paths on the report
236 'NAME', 'KNAME', 'MAJ:MIN', 'FSTYPE', 'MOUNTPOINT', 'LABEL', 'UUID',
237 'RO', 'RM', 'MODEL', 'SIZE', 'STATE', 'OWNER', 'GROUP', 'MODE',
238 'ALIGNMENT', 'PHY-SEC', 'LOG-SEC', 'ROTA', 'SCHED', 'TYPE', 'DISC-ALN',
239 'DISC-GRAN', 'DISC-MAX', 'DISC-ZERO', 'PKNAME', 'PARTLABEL'
241 device
= device
.rstrip('/')
242 columns
= columns
or default_columns
243 # --nodeps -> Avoid adding children/parents to the device, only give information
244 # on the actual device we are querying for
245 # -P -> Produce pairs of COLUMN="value"
246 # -p -> Return full paths to devices, not just the names, when ``abspath`` is set
247 # -o -> Use the columns specified or default ones provided by this function
248 base_command
= ['lsblk', '--nodeps', '-P']
250 base_command
.append('-p')
251 base_command
.append('-o')
252 base_command
.append(','.join(columns
))
253 base_command
.append(device
)
254 out
, err
, rc
= process
.call(base_command
)
259 return _lsblk_parser(' '.join(out
))
264 Boolean to determine if a given device is a block device (**not**
267 For example: /dev/sda would return True, but not /dev/sdc1
269 if not os
.path
.exists(dev
):
271 # use lsblk first, fall back to using stat
272 TYPE
= lsblk(dev
).get('TYPE')
274 return TYPE
== 'disk'
277 return _stat_is_device(os
.lstat(dev
).st_mode
)
278 if stat
.S_ISBLK(os
.lstat(dev
)):
283 def is_partition(dev
):
285 Boolean to determine if a given device is a partition, like /dev/sda1
287 if not os
.path
.exists(dev
):
289 # use lsblk first, fall back to using stat
290 TYPE
= lsblk(dev
).get('TYPE')
292 return TYPE
== 'part'
295 stat_obj
= os
.stat(dev
)
296 if _stat_is_device(stat_obj
.st_mode
):
299 major
= os
.major(stat_obj
.st_rdev
)
300 minor
= os
.minor(stat_obj
.st_rdev
)
301 if os
.path
.exists('/sys/dev/block/%d:%d/partition' % (major
, minor
)):
306 def _map_dev_paths(_path
, include_abspath
=False, include_realpath
=False):
308 Go through all the items in ``_path`` and map them to their absolute path::
312 If ``include_abspath`` is set, then a reverse mapping is set as well::
314 {'sda': '/dev/sda', '/dev/sda': 'sda'}
316 If ``include_realpath`` is set then the same operation is done for any
317 links found when listing, these are *not* reversed to avoid clashing on
318 existing keys, but both abspath and basename can be included. For example::
321 'ceph-data': '/dev/mapper/ceph-data',
322 '/dev/mapper/ceph-data': 'ceph-data',
323 '/dev/dm-0': '/dev/mapper/ceph-data',
324 'dm-0': '/dev/mapper/ceph-data'
328 In case of possible exceptions the mapping is returned empty, and the
333 dev_names
= os
.listdir(_path
)
334 except (OSError, IOError):
335 logger
.exception('unable to list block devices from: %s' % _path
)
338 for dev_name
in dev_names
:
339 mapping
[dev_name
] = os
.path
.join(_path
, dev_name
)
342 for k
, v
in list(mapping
.items()):
346 for abspath
in list(mapping
.values()):
347 if not os
.path
.islink(abspath
):
350 realpath
= os
.path
.realpath(abspath
)
351 basename
= os
.path
.basename(realpath
)
352 mapping
[basename
] = abspath
354 mapping
[realpath
] = abspath
359 def get_block_devs(sys_block_path
="/sys/block", skip_loop
=True):
361 Go through all the items in /sys/block and return them as a list.
363 The ``sys_block_path`` argument is set for easier testing and is not
364 required for proper operation.
366 devices
= _map_dev_paths(sys_block_path
).keys()
368 return [d
for d
in devices
if not d
.startswith('loop')]
372 def get_dev_devs(dev_path
="/dev"):
374 Go through all the items in /dev and return them as a list.
376 The ``dev_path`` argument is set for easier testing and is not
377 required for proper operation.
379 return _map_dev_paths(dev_path
, include_abspath
=True)
382 def get_mapper_devs(mapper_path
="/dev/mapper"):
384 Go through all the items in /dev and return them as a list.
386 The ``dev_path`` argument is set for easier testing and is not
387 required for proper operation.
389 return _map_dev_paths(mapper_path
, include_abspath
=True, include_realpath
=True)
392 class BaseFloatUnit(float):
394 Base class to support float representations of size values. Suffix is
395 computed on child classes by inspecting the class name
399 return "<%s(%s)>" % (self
.__class
__.__name
__, self
.__float
__())
402 return "{size:.2f} {suffix}".format(
403 size
=self
.__float
__(),
404 suffix
=self
.__class
__.__name
__.split('Float')[-1]
408 return int(self
.real
)
414 class FloatB(BaseFloatUnit
):
418 class FloatMB(BaseFloatUnit
):
422 class FloatGB(BaseFloatUnit
):
426 class FloatKB(BaseFloatUnit
):
430 class FloatTB(BaseFloatUnit
):
436 Helper to provide an interface for different sizes given a single initial
437 input. Allows for comparison between different size objects, which avoids
438 the need to convert sizes before comparison (e.g. comparing megabytes
441 Common comparison operators are supported::
443 >>> hd1 = Size(gb=400)
444 >>> hd2 = Size(gb=500)
451 >>> hd1 == Size(gb=400)
454 The Size object can also be multiplied or divided::
463 Additions and subtractions are only supported between Size objects::
465 >>> Size(gb=224) - Size(gb=100)
467 >>> Size(gb=1) + Size(mb=300)
470 Can also display a human-readable representation, with automatic detection
471 on best suited unit, or alternatively, specific unit representation::
473 >>> s = Size(mb=2211)
478 >>> print "Total size: %s" % s.mb
479 Total size: 2211.00 MB
480 >>> print "Total size: %s" % s
484 def __init__(self
, multiplier
=1024, **kw
):
485 self
._multiplier
= multiplier
486 # create a mapping of units-to-multiplier, skip bytes as that is
487 # calculated initially always and does not need to convert
489 [('kb', 'kilobytes'), self
._multiplier
],
490 [('mb', 'megabytes'), self
._multiplier
** 2],
491 [('gb', 'gigabytes'), self
._multiplier
** 3],
492 [('tb', 'terabytes'), self
._multiplier
** 4],
494 # and mappings for units-to-formatters, including bytes and aliases for
497 [('b', 'bytes'), FloatB
],
498 [('kb', 'kilobytes'), FloatKB
],
499 [('mb', 'megabytes'), FloatMB
],
500 [('gb', 'gigabytes'), FloatGB
],
501 [('tb', 'terabytes'), FloatTB
],
503 self
._formatters
= {}
504 for key
, value
in format_aliases
:
506 self
._formatters
[alias
] = value
508 for key
, value
in aliases
:
510 self
._factors
[alias
] = value
512 for k
, v
in kw
.items():
514 # only pursue the first occurence
517 def _convert(self
, size
, unit
):
519 Convert any size down to bytes so that other methods can rely on bytes
520 being available always, regardless of what they pass in, avoiding the
521 need for a mapping of every permutation.
523 if unit
in ['b', 'bytes']:
526 factor
= self
._factors
[unit
]
527 self
._b
= float(size
* factor
)
529 def _get_best_format(self
):
531 Go through all the supported units, and use the first one that is less
532 than 1024. This allows to represent size in the most readable format
535 for unit
in ['b', 'kb', 'mb', 'gb', 'tb']:
536 if getattr(self
, unit
) > 1024:
538 return getattr(self
, unit
)
541 return "<Size(%s)>" % self
._get
_best
_format
()
544 return "%s" % self
._get
_best
_format
()
546 def __lt__(self
, other
):
547 return self
._b
< other
._b
549 def __le__(self
, other
):
550 return self
._b
<= other
._b
552 def __eq__(self
, other
):
553 return self
._b
== other
._b
555 def __ne__(self
, other
):
556 return self
._b
!= other
._b
558 def __ge__(self
, other
):
559 return self
._b
>= other
._b
561 def __gt__(self
, other
):
562 return self
._b
> other
._b
564 def __add__(self
, other
):
565 if isinstance(other
, Size
):
566 _b
= self
._b
+ other
._b
568 raise TypeError('Cannot add "Size" object with int')
570 def __sub__(self
, other
):
571 if isinstance(other
, Size
):
572 _b
= self
._b
- other
._b
574 raise TypeError('Cannot subtract "Size" object from int')
576 def __mul__(self
, other
):
577 if isinstance(other
, Size
):
578 raise TypeError('Cannot multiply with "Size" object')
582 def __truediv__(self
, other
):
583 if isinstance(other
, Size
):
584 return self
._b
/ other
._b
588 def __div__(self
, other
):
589 if isinstance(other
, Size
):
590 return self
._b
/ other
._b
594 def __getattr__(self
, unit
):
596 Calculate units on the fly, relies on the fact that ``bytes`` has been
597 converted at instantiation. Units that don't exist will trigger an
601 formatter
= self
._formatters
[unit
]
603 raise AttributeError('Size object has not attribute "%s"' % unit
)
604 if unit
in ['b', 'bytes']:
605 return formatter(self
._b
)
607 factor
= self
._factors
[unit
]
609 raise AttributeError('Size object has not attribute "%s"' % unit
)
610 return formatter(float(self
._b
) / factor
)
613 def human_readable_size(size
):
615 Take a size in bytes, and transform it into a human readable size with up
616 to two decimals of precision.
618 suffixes
= ['B', 'KB', 'MB', 'GB', 'TB']
623 return "{size:.2f} {suffix}".format(
625 suffix
=suffixes
[suffix_index
])
628 def get_partitions_facts(sys_block_path
):
629 partition_metadata
= {}
630 for folder
in os
.listdir(sys_block_path
):
631 folder_path
= os
.path
.join(sys_block_path
, folder
)
632 if os
.path
.exists(os
.path
.join(folder_path
, 'partition')):
633 contents
= get_file_contents(os
.path
.join(folder_path
, 'partition'))
637 part_sys_block_path
= os
.path
.join(sys_block_path
, partname
)
639 part
['start'] = get_file_contents(part_sys_block_path
+ "/start", 0)
640 part
['sectors'] = get_file_contents(part_sys_block_path
+ "/size", 0)
642 part
['sectorsize'] = get_file_contents(
643 part_sys_block_path
+ "/queue/logical_block_size")
644 if not part
['sectorsize']:
645 part
['sectorsize'] = get_file_contents(
646 part_sys_block_path
+ "/queue/hw_sector_size", 512)
647 part
['size'] = human_readable_size(float(part
['sectors']) * 512)
649 partition_metadata
[partname
] = part
650 return partition_metadata
653 def is_mapper_device(device_name
):
654 return device_name
.startswith(('/dev/mapper', '/dev/dm-'))
657 def is_locked_raw_device(disk_path
):
659 A device can be locked by a third party software like a database.
660 To detect that case, the device is opened in Read/Write and exclusive mode
662 open_flags
= (os
.O_RDWR | os
.O_EXCL
)
667 fd
= os
.open(disk_path
, open_flags
, open_mode
)
679 def get_devices(_sys_block_path
='/sys/block', _dev_path
='/dev', _mapper_path
='/dev/mapper'):
681 Captures all available devices from /sys/block/, including its partitions,
682 along with interesting metadata like sectors, size, vendor,
683 solid/rotational, etc...
685 Returns a dictionary, where keys are the full paths to devices.
687 ..note:: dmapper devices get their path updated to what they link from, if
688 /dev/dm-0 is linked by /dev/mapper/ceph-data, then the latter gets
691 ..note:: loop devices, removable media, and logical volumes are never included.
693 # Portions of this detection process are inspired by some of the fact
694 # gathering done by Ansible in module_utils/facts/hardware/linux.py. The
695 # processing of metadata and final outcome *is very different* and fully
696 # imcompatible. There are ignored devices, and paths get resolved depending
697 # on dm devices, loop, and removable media
701 block_devs
= get_block_devs(_sys_block_path
)
702 dev_devs
= get_dev_devs(_dev_path
)
703 mapper_devs
= get_mapper_devs(_mapper_path
)
705 for block
in block_devs
:
706 sysdir
= os
.path
.join(_sys_block_path
, block
)
709 # Ensure that the diskname is an absolute path and that it never points
710 # to a /dev/dm-* device
711 diskname
= mapper_devs
.get(block
) or dev_devs
.get(block
)
715 # If the mapper device is a logical volume it gets excluded
716 if is_mapper_device(diskname
):
717 if lvm
.is_lv(diskname
):
720 metadata
['removable'] = get_file_contents(os
.path
.join(sysdir
, 'removable'))
721 # Is the device read-only ?
722 metadata
['ro'] = get_file_contents(os
.path
.join(sysdir
, 'ro'))
725 for key
in ['vendor', 'model', 'rev', 'sas_address', 'sas_device_handle']:
726 metadata
[key
] = get_file_contents(sysdir
+ "/device/" + key
)
728 for key
in ['sectors', 'size']:
729 metadata
[key
] = get_file_contents(os
.path
.join(sysdir
, key
), 0)
731 for key
, _file
in [('support_discard', '/queue/discard_granularity')]:
732 metadata
[key
] = get_file_contents(os
.path
.join(sysdir
, _file
))
734 metadata
['partitions'] = get_partitions_facts(sysdir
)
736 for key
in ['rotational', 'nr_requests']:
737 metadata
[key
] = get_file_contents(sysdir
+ "/queue/" + key
)
739 metadata
['scheduler_mode'] = ""
740 scheduler
= get_file_contents(sysdir
+ "/queue/scheduler")
741 if scheduler
is not None:
742 m
= re
.match(r
".*?(\[(.*)\])", scheduler
)
744 metadata
['scheduler_mode'] = m
.group(2)
746 if not metadata
['sectors']:
747 metadata
['sectors'] = 0
748 size
= metadata
['sectors'] or metadata
['size']
749 metadata
['sectorsize'] = get_file_contents(sysdir
+ "/queue/logical_block_size")
750 if not metadata
['sectorsize']:
751 metadata
['sectorsize'] = get_file_contents(sysdir
+ "/queue/hw_sector_size", 512)
752 metadata
['human_readable_size'] = human_readable_size(float(size
) * 512)
753 metadata
['size'] = float(size
) * 512
754 metadata
['path'] = diskname
755 metadata
['locked'] = is_locked_raw_device(metadata
['path'])
757 device_facts
[diskname
] = metadata