]>
Commit | Line | Data |
---|---|---|
1adf2230 | 1 | import logging |
3efd9988 | 2 | import os |
1adf2230 | 3 | import re |
3efd9988 | 4 | import stat |
39ae355f | 5 | import time |
181888fb | 6 | from ceph_volume import process |
1adf2230 AA |
7 | from ceph_volume.api import lvm |
8 | from ceph_volume.util.system import get_file_contents | |
9 | ||
10 | ||
11 | logger = logging.getLogger(__name__) | |
181888fb FG |
12 | |
13 | ||
b32b8144 FG |
14 | # The blkid CLI tool has some oddities which prevents having one common call |
15 | # to extract the information instead of having separate utilities. The `udev` | |
16 | # type of output is needed in older versions of blkid (v 2.23) that will not | |
17 | # work correctly with just the ``-p`` flag to bypass the cache for example. | |
18 | # Xenial doesn't have this problem as it uses a newer blkid version. | |
19 | ||
20 | ||
181888fb FG |
21 | def get_partuuid(device): |
22 | """ | |
23 | If a device is a partition, it will probably have a PARTUUID on it that | |
24 | will persist and can be queried against `blkid` later to detect the actual | |
25 | device | |
26 | """ | |
27 | out, err, rc = process.call( | |
f67539c2 | 28 | ['blkid', '-c', '/dev/null', '-s', 'PARTUUID', '-o', 'value', device] |
181888fb FG |
29 | ) |
30 | return ' '.join(out).strip() | |
31 | ||
32 | ||
91327a77 AA |
33 | def _blkid_parser(output): |
34 | """ | |
35 | Parses the output from a system ``blkid`` call, requires output to be | |
36 | produced using the ``-p`` flag which bypasses the cache, mangling the | |
37 | names. These names are corrected to what it would look like without the | |
38 | ``-p`` flag. | |
39 | ||
40 | Normal output:: | |
41 | ||
42 | /dev/sdb1: UUID="62416664-cbaf-40bd-9689-10bd337379c3" TYPE="xfs" [...] | |
43 | """ | |
44 | # first spaced separated item is garbage, gets tossed: | |
45 | output = ' '.join(output.split()[1:]) | |
46 | # split again, respecting possible whitespace in quoted values | |
47 | pairs = output.split('" ') | |
48 | raw = {} | |
49 | processed = {} | |
50 | mapping = { | |
51 | 'UUID': 'UUID', | |
52 | 'TYPE': 'TYPE', | |
53 | 'PART_ENTRY_NAME': 'PARTLABEL', | |
54 | 'PART_ENTRY_UUID': 'PARTUUID', | |
494da23a | 55 | 'PART_ENTRY_TYPE': 'PARTTYPE', |
91327a77 AA |
56 | 'PTTYPE': 'PTTYPE', |
57 | } | |
58 | ||
59 | for pair in pairs: | |
60 | try: | |
61 | column, value = pair.split('=') | |
62 | except ValueError: | |
63 | continue | |
64 | raw[column] = value.strip().strip().strip('"') | |
65 | ||
66 | for key, value in raw.items(): | |
67 | new_key = mapping.get(key) | |
68 | if not new_key: | |
69 | continue | |
70 | processed[new_key] = value | |
71 | ||
72 | return processed | |
73 | ||
74 | ||
75 | def blkid(device): | |
76 | """ | |
77 | The blkid interface to its CLI, creating an output similar to what is | |
78 | expected from ``lsblk``. In most cases, ``lsblk()`` should be the preferred | |
79 | method for extracting information about a device. There are some corner | |
80 | cases where it might provide information that is otherwise unavailable. | |
81 | ||
82 | The system call uses the ``-p`` flag which bypasses the cache, the caveat | |
83 | being that the keys produced are named completely different to expected | |
84 | names. | |
85 | ||
86 | For example, instead of ``PARTLABEL`` it provides a ``PART_ENTRY_NAME``. | |
87 | A bit of translation between these known keys is done, which is why | |
88 | ``lsblk`` should always be preferred: the output provided here is not as | |
89 | rich, given that a translation of keys is required for a uniform interface | |
90 | with the ``-p`` flag. | |
91 | ||
92 | Label name to expected output chart: | |
93 | ||
94 | cache bypass name expected name | |
95 | ||
96 | UUID UUID | |
97 | TYPE TYPE | |
98 | PART_ENTRY_NAME PARTLABEL | |
99 | PART_ENTRY_UUID PARTUUID | |
100 | """ | |
101 | out, err, rc = process.call( | |
f67539c2 | 102 | ['blkid', '-c', '/dev/null', '-p', device] |
91327a77 AA |
103 | ) |
104 | return _blkid_parser(' '.join(out)) | |
105 | ||
106 | ||
b32b8144 FG |
107 | def get_part_entry_type(device): |
108 | """ | |
109 | Parses the ``ID_PART_ENTRY_TYPE`` from the "low level" (bypasses the cache) | |
110 | output that uses the ``udev`` type of output. This output is intended to be | |
111 | used for udev rules, but it is useful in this case as it is the only | |
112 | consistent way to retrieve the GUID used by ceph-disk to identify devices. | |
113 | """ | |
f67539c2 | 114 | out, err, rc = process.call(['blkid', '-c', '/dev/null', '-p', '-o', 'udev', device]) |
b32b8144 FG |
115 | for line in out: |
116 | if 'ID_PART_ENTRY_TYPE=' in line: | |
117 | return line.split('=')[-1].strip() | |
118 | return '' | |
119 | ||
120 | ||
181888fb FG |
121 | def get_device_from_partuuid(partuuid): |
122 | """ | |
123 | If a device has a partuuid, query blkid so that it can tell us what that | |
124 | device is | |
125 | """ | |
126 | out, err, rc = process.call( | |
f67539c2 | 127 | ['blkid', '-c', '/dev/null', '-t', 'PARTUUID="%s"' % partuuid, '-o', 'device'] |
181888fb FG |
128 | ) |
129 | return ' '.join(out).strip() | |
3efd9988 FG |
130 | |
131 | ||
f64942e4 AA |
132 | def remove_partition(device): |
133 | """ | |
134 | Removes a partition using parted | |
135 | ||
136 | :param device: A ``Device()`` object | |
137 | """ | |
39ae355f TL |
138 | # Sometimes there's a race condition that makes 'ID_PART_ENTRY_NUMBER' be not present |
139 | # in the output of `udevadm info --query=property`. | |
140 | # Probably not ideal and not the best fix but this allows to get around that issue. | |
141 | # The idea is to make it retry multiple times before actually failing. | |
142 | for i in range(10): | |
143 | udev_info = udevadm_property(device.path) | |
144 | partition_number = udev_info.get('ID_PART_ENTRY_NUMBER') | |
145 | if partition_number: | |
146 | break | |
147 | time.sleep(0.2) | |
f64942e4 | 148 | if not partition_number: |
2a845540 | 149 | raise RuntimeError('Unable to detect the partition number for device: %s' % device.path) |
f64942e4 AA |
150 | |
151 | process.run( | |
522d829b | 152 | ['parted', device.parent_device, '--script', '--', 'rm', partition_number] |
f64942e4 AA |
153 | ) |
154 | ||
155 | ||
3efd9988 FG |
156 | def _stat_is_device(stat_obj): |
157 | """ | |
158 | Helper function that will interpret ``os.stat`` output directly, so that other | |
159 | functions can call ``os.stat`` once and interpret that result several times | |
160 | """ | |
161 | return stat.S_ISBLK(stat_obj) | |
162 | ||
163 | ||
b32b8144 FG |
164 | def _lsblk_parser(line): |
165 | """ | |
166 | Parses lines in lsblk output. Requires output to be in pair mode (``-P`` flag). Lines | |
167 | need to be whole strings, the line gets split when processed. | |
168 | ||
169 | :param line: A string, with the full line from lsblk output | |
170 | """ | |
171 | # parse the COLUMN="value" output to construct the dictionary | |
172 | pairs = line.split('" ') | |
173 | parsed = {} | |
174 | for pair in pairs: | |
175 | try: | |
176 | column, value = pair.split('=') | |
177 | except ValueError: | |
178 | continue | |
179 | parsed[column] = value.strip().strip().strip('"') | |
180 | return parsed | |
181 | ||
182 | ||
183 | def device_family(device): | |
184 | """ | |
185 | Returns a list of associated devices. It assumes that ``device`` is | |
186 | a parent device. It is up to the caller to ensure that the device being | |
187 | used is a parent, not a partition. | |
188 | """ | |
189 | labels = ['NAME', 'PARTLABEL', 'TYPE'] | |
190 | command = ['lsblk', '-P', '-p', '-o', ','.join(labels), device] | |
191 | out, err, rc = process.call(command) | |
192 | devices = [] | |
193 | for line in out: | |
194 | devices.append(_lsblk_parser(line)) | |
195 | ||
196 | return devices | |
197 | ||
198 | ||
f64942e4 AA |
199 | def udevadm_property(device, properties=[]): |
200 | """ | |
201 | Query udevadm for information about device properties. | |
202 | Optionally pass a list of properties to return. A requested property might | |
203 | not be returned if not present. | |
204 | ||
205 | Expected output format:: | |
206 | # udevadm info --query=property --name=/dev/sda :( | |
207 | DEVNAME=/dev/sda | |
208 | DEVTYPE=disk | |
209 | ID_ATA=1 | |
210 | ID_BUS=ata | |
211 | ID_MODEL=SK_hynix_SC311_SATA_512GB | |
212 | ID_PART_TABLE_TYPE=gpt | |
213 | ID_PART_TABLE_UUID=c8f91d57-b26c-4de1-8884-0c9541da288c | |
214 | ID_PATH=pci-0000:00:17.0-ata-3 | |
215 | ID_PATH_TAG=pci-0000_00_17_0-ata-3 | |
216 | ID_REVISION=70000P10 | |
217 | ID_SERIAL=SK_hynix_SC311_SATA_512GB_MS83N71801150416A | |
218 | TAGS=:systemd: | |
219 | USEC_INITIALIZED=16117769 | |
220 | ... | |
221 | """ | |
222 | out = _udevadm_info(device) | |
223 | ret = {} | |
224 | for line in out: | |
225 | p, v = line.split('=', 1) | |
226 | if not properties or p in properties: | |
227 | ret[p] = v | |
228 | return ret | |
229 | ||
230 | ||
231 | def _udevadm_info(device): | |
232 | """ | |
233 | Call udevadm and return the output | |
234 | """ | |
235 | cmd = ['udevadm', 'info', '--query=property', device] | |
236 | out, _err, _rc = process.call(cmd) | |
237 | return out | |
238 | ||
239 | ||
b32b8144 | 240 | def lsblk(device, columns=None, abspath=False): |
39ae355f TL |
241 | result = [] |
242 | if not os.path.isdir(device): | |
243 | result = lsblk_all(device=device, | |
244 | columns=columns, | |
245 | abspath=abspath) | |
246 | if not result: | |
247 | logger.debug(f"{device} not found is lsblk report") | |
248 | return {} | |
249 | ||
250 | return result[0] | |
2a845540 TL |
251 | |
252 | def lsblk_all(device='', columns=None, abspath=False): | |
3efd9988 FG |
253 | """ |
254 | Create a dictionary of identifying values for a device using ``lsblk``. | |
255 | Each supported column is a key, in its *raw* format (all uppercase | |
256 | usually). ``lsblk`` has support for certain "columns" (in blkid these | |
257 | would be labels), and these columns vary between distributions and | |
258 | ``lsblk`` versions. The newer versions support a richer set of columns, | |
259 | while older ones were a bit limited. | |
260 | ||
b32b8144 | 261 | These are a subset of lsblk columns which are known to work on both CentOS 7 and Xenial: |
3efd9988 FG |
262 | |
263 | NAME device name | |
264 | KNAME internal kernel device name | |
2a845540 | 265 | PKNAME internal kernel parent device name |
3efd9988 FG |
266 | MAJ:MIN major:minor device number |
267 | FSTYPE filesystem type | |
268 | MOUNTPOINT where the device is mounted | |
269 | LABEL filesystem LABEL | |
270 | UUID filesystem UUID | |
271 | RO read-only device | |
272 | RM removable device | |
273 | MODEL device identifier | |
274 | SIZE size of the device | |
275 | STATE state of the device | |
276 | OWNER user name | |
277 | GROUP group name | |
278 | MODE device node permissions | |
279 | ALIGNMENT alignment offset | |
280 | MIN-IO minimum I/O size | |
281 | OPT-IO optimal I/O size | |
282 | PHY-SEC physical sector size | |
283 | LOG-SEC logical sector size | |
284 | ROTA rotational device | |
285 | SCHED I/O scheduler name | |
286 | RQ-SIZE request queue size | |
287 | TYPE device type | |
b32b8144 | 288 | PKNAME internal parent kernel device name |
3efd9988 FG |
289 | DISC-ALN discard alignment offset |
290 | DISC-GRAN discard granularity | |
291 | DISC-MAX discard max bytes | |
292 | DISC-ZERO discard zeroes data | |
293 | ||
294 | There is a bug in ``lsblk`` where using all the available (supported) | |
295 | columns will result in no output (!), in order to workaround this the | |
296 | following columns have been removed from the default reporting columns: | |
297 | ||
298 | * RQ-SIZE (request queue size) | |
299 | * MIN-IO minimum I/O size | |
300 | * OPT-IO optimal I/O size | |
301 | ||
302 | These should be available however when using `columns`. For example:: | |
303 | ||
304 | >>> lsblk('/dev/sda1', columns=['OPT-IO']) | |
305 | {'OPT-IO': '0'} | |
306 | ||
307 | Normal CLI output, as filtered by the flags in this function will look like :: | |
308 | ||
2a845540 | 309 | $ lsblk -P -o NAME,KNAME,PKNAME,MAJ:MIN,FSTYPE,MOUNTPOINT |
3efd9988 FG |
310 | NAME="sda1" KNAME="sda1" MAJ:MIN="8:1" FSTYPE="ext4" MOUNTPOINT="/" |
311 | ||
312 | :param columns: A list of columns to report as keys in its original form. | |
b32b8144 | 313 | :param abspath: Set the flag for absolute paths on the report |
3efd9988 FG |
314 | """ |
315 | default_columns = [ | |
2a845540 TL |
316 | 'NAME', 'KNAME', 'PKNAME', 'MAJ:MIN', 'FSTYPE', 'MOUNTPOINT', 'LABEL', |
317 | 'UUID', 'RO', 'RM', 'MODEL', 'SIZE', 'STATE', 'OWNER', 'GROUP', 'MODE', | |
3efd9988 | 318 | 'ALIGNMENT', 'PHY-SEC', 'LOG-SEC', 'ROTA', 'SCHED', 'TYPE', 'DISC-ALN', |
b32b8144 | 319 | 'DISC-GRAN', 'DISC-MAX', 'DISC-ZERO', 'PKNAME', 'PARTLABEL' |
3efd9988 | 320 | ] |
3efd9988 | 321 | columns = columns or default_columns |
3efd9988 | 322 | # -P -> Produce pairs of COLUMN="value" |
b32b8144 | 323 | # -p -> Return full paths to devices, not just the names, when ``abspath`` is set |
3efd9988 | 324 | # -o -> Use the columns specified or default ones provided by this function |
2a845540 | 325 | base_command = ['lsblk', '-P'] |
b32b8144 FG |
326 | if abspath: |
327 | base_command.append('-p') | |
328 | base_command.append('-o') | |
329 | base_command.append(','.join(columns)) | |
39ae355f TL |
330 | if device: |
331 | base_command.append('--nodeps') | |
332 | base_command.append(device) | |
2a845540 | 333 | |
b32b8144 | 334 | out, err, rc = process.call(base_command) |
3efd9988 FG |
335 | |
336 | if rc != 0: | |
2a845540 TL |
337 | raise RuntimeError(f"Error: {err}") |
338 | ||
339 | result = [] | |
340 | ||
341 | for line in out: | |
342 | result.append(_lsblk_parser(line)) | |
343 | ||
39ae355f | 344 | return result |
3efd9988 FG |
345 | |
346 | ||
3efd9988 FG |
347 | def is_device(dev): |
348 | """ | |
349 | Boolean to determine if a given device is a block device (**not** | |
350 | a partition!) | |
351 | ||
352 | For example: /dev/sda would return True, but not /dev/sdc1 | |
353 | """ | |
354 | if not os.path.exists(dev): | |
355 | return False | |
2a845540 TL |
356 | if not dev.startswith('/dev/'): |
357 | return False | |
358 | if dev[len('/dev/'):].startswith('loop'): | |
359 | if not allow_loop_devices(): | |
360 | return False | |
3efd9988 FG |
361 | |
362 | # fallback to stat | |
363 | return _stat_is_device(os.lstat(dev).st_mode) | |
3efd9988 FG |
364 | |
365 | ||
366 | def is_partition(dev): | |
367 | """ | |
368 | Boolean to determine if a given device is a partition, like /dev/sda1 | |
369 | """ | |
370 | if not os.path.exists(dev): | |
371 | return False | |
372 | # use lsblk first, fall back to using stat | |
373 | TYPE = lsblk(dev).get('TYPE') | |
374 | if TYPE: | |
375 | return TYPE == 'part' | |
376 | ||
377 | # fallback to stat | |
378 | stat_obj = os.stat(dev) | |
379 | if _stat_is_device(stat_obj.st_mode): | |
380 | return False | |
381 | ||
382 | major = os.major(stat_obj.st_rdev) | |
383 | minor = os.minor(stat_obj.st_rdev) | |
384 | if os.path.exists('/sys/dev/block/%d:%d/partition' % (major, minor)): | |
385 | return True | |
386 | return False | |
1adf2230 AA |
387 | |
388 | ||
2a845540 TL |
389 | def is_ceph_rbd(dev): |
390 | """ | |
391 | Boolean to determine if a given device is a ceph RBD device, like /dev/rbd0 | |
392 | """ | |
393 | return dev.startswith(('/dev/rbd')) | |
394 | ||
395 | ||
1adf2230 AA |
396 | class BaseFloatUnit(float): |
397 | """ | |
398 | Base class to support float representations of size values. Suffix is | |
399 | computed on child classes by inspecting the class name | |
400 | """ | |
401 | ||
402 | def __repr__(self): | |
403 | return "<%s(%s)>" % (self.__class__.__name__, self.__float__()) | |
404 | ||
405 | def __str__(self): | |
406 | return "{size:.2f} {suffix}".format( | |
407 | size=self.__float__(), | |
408 | suffix=self.__class__.__name__.split('Float')[-1] | |
409 | ) | |
410 | ||
411 | def as_int(self): | |
412 | return int(self.real) | |
413 | ||
414 | def as_float(self): | |
415 | return self.real | |
416 | ||
417 | ||
418 | class FloatB(BaseFloatUnit): | |
419 | pass | |
420 | ||
421 | ||
422 | class FloatMB(BaseFloatUnit): | |
423 | pass | |
424 | ||
425 | ||
426 | class FloatGB(BaseFloatUnit): | |
427 | pass | |
428 | ||
429 | ||
430 | class FloatKB(BaseFloatUnit): | |
431 | pass | |
432 | ||
433 | ||
434 | class FloatTB(BaseFloatUnit): | |
435 | pass | |
436 | ||
20effc67 TL |
437 | class FloatPB(BaseFloatUnit): |
438 | pass | |
1adf2230 AA |
439 | |
440 | class Size(object): | |
441 | """ | |
442 | Helper to provide an interface for different sizes given a single initial | |
443 | input. Allows for comparison between different size objects, which avoids | |
444 | the need to convert sizes before comparison (e.g. comparing megabytes | |
445 | against gigabytes). | |
446 | ||
447 | Common comparison operators are supported:: | |
448 | ||
449 | >>> hd1 = Size(gb=400) | |
450 | >>> hd2 = Size(gb=500) | |
451 | >>> hd1 > hd2 | |
452 | False | |
453 | >>> hd1 < hd2 | |
454 | True | |
455 | >>> hd1 == hd2 | |
456 | False | |
457 | >>> hd1 == Size(gb=400) | |
458 | True | |
459 | ||
460 | The Size object can also be multiplied or divided:: | |
461 | ||
462 | >>> hd1 | |
463 | <Size(400.00 GB)> | |
464 | >>> hd1 * 2 | |
465 | <Size(800.00 GB)> | |
466 | >>> hd1 | |
467 | <Size(800.00 GB)> | |
468 | ||
469 | Additions and subtractions are only supported between Size objects:: | |
470 | ||
471 | >>> Size(gb=224) - Size(gb=100) | |
472 | <Size(124.00 GB)> | |
473 | >>> Size(gb=1) + Size(mb=300) | |
474 | <Size(1.29 GB)> | |
475 | ||
476 | Can also display a human-readable representation, with automatic detection | |
477 | on best suited unit, or alternatively, specific unit representation:: | |
478 | ||
479 | >>> s = Size(mb=2211) | |
480 | >>> s | |
481 | <Size(2.16 GB)> | |
482 | >>> s.mb | |
483 | <FloatMB(2211.0)> | |
9f95a23c | 484 | >>> print("Total size: %s" % s.mb) |
1adf2230 | 485 | Total size: 2211.00 MB |
9f95a23c | 486 | >>> print("Total size: %s" % s) |
1adf2230 AA |
487 | Total size: 2.16 GB |
488 | """ | |
489 | ||
92f5a8d4 TL |
490 | @classmethod |
491 | def parse(cls, size): | |
492 | if (len(size) > 2 and | |
20effc67 | 493 | size[-2].lower() in ['k', 'm', 'g', 't', 'p'] and |
92f5a8d4 TL |
494 | size[-1].lower() == 'b'): |
495 | return cls(**{size[-2:].lower(): float(size[0:-2])}) | |
20effc67 | 496 | elif size[-1].lower() in ['b', 'k', 'm', 'g', 't', 'p']: |
92f5a8d4 TL |
497 | return cls(**{size[-1].lower(): float(size[0:-1])}) |
498 | else: | |
499 | return cls(b=float(size)) | |
500 | ||
501 | ||
1adf2230 AA |
502 | def __init__(self, multiplier=1024, **kw): |
503 | self._multiplier = multiplier | |
504 | # create a mapping of units-to-multiplier, skip bytes as that is | |
505 | # calculated initially always and does not need to convert | |
506 | aliases = [ | |
92f5a8d4 TL |
507 | [('k', 'kb', 'kilobytes'), self._multiplier], |
508 | [('m', 'mb', 'megabytes'), self._multiplier ** 2], | |
509 | [('g', 'gb', 'gigabytes'), self._multiplier ** 3], | |
510 | [('t', 'tb', 'terabytes'), self._multiplier ** 4], | |
20effc67 | 511 | [('p', 'pb', 'petabytes'), self._multiplier ** 5] |
1adf2230 AA |
512 | ] |
513 | # and mappings for units-to-formatters, including bytes and aliases for | |
514 | # each | |
515 | format_aliases = [ | |
516 | [('b', 'bytes'), FloatB], | |
517 | [('kb', 'kilobytes'), FloatKB], | |
518 | [('mb', 'megabytes'), FloatMB], | |
519 | [('gb', 'gigabytes'), FloatGB], | |
520 | [('tb', 'terabytes'), FloatTB], | |
20effc67 | 521 | [('pb', 'petabytes'), FloatPB], |
1adf2230 AA |
522 | ] |
523 | self._formatters = {} | |
524 | for key, value in format_aliases: | |
525 | for alias in key: | |
526 | self._formatters[alias] = value | |
527 | self._factors = {} | |
528 | for key, value in aliases: | |
529 | for alias in key: | |
530 | self._factors[alias] = value | |
531 | ||
532 | for k, v in kw.items(): | |
533 | self._convert(v, k) | |
11fdf7f2 | 534 | # only pursue the first occurrence |
1adf2230 AA |
535 | break |
536 | ||
537 | def _convert(self, size, unit): | |
538 | """ | |
539 | Convert any size down to bytes so that other methods can rely on bytes | |
540 | being available always, regardless of what they pass in, avoiding the | |
541 | need for a mapping of every permutation. | |
542 | """ | |
543 | if unit in ['b', 'bytes']: | |
544 | self._b = size | |
545 | return | |
546 | factor = self._factors[unit] | |
547 | self._b = float(size * factor) | |
548 | ||
549 | def _get_best_format(self): | |
550 | """ | |
551 | Go through all the supported units, and use the first one that is less | |
552 | than 1024. This allows to represent size in the most readable format | |
553 | available | |
554 | """ | |
20effc67 | 555 | for unit in ['b', 'kb', 'mb', 'gb', 'tb', 'pb']: |
1adf2230 AA |
556 | if getattr(self, unit) > 1024: |
557 | continue | |
558 | return getattr(self, unit) | |
559 | ||
560 | def __repr__(self): | |
561 | return "<Size(%s)>" % self._get_best_format() | |
562 | ||
563 | def __str__(self): | |
564 | return "%s" % self._get_best_format() | |
565 | ||
11fdf7f2 TL |
566 | def __format__(self, spec): |
567 | return str(self._get_best_format()).__format__(spec) | |
568 | ||
92f5a8d4 TL |
569 | def __int__(self): |
570 | return int(self._b) | |
571 | ||
572 | def __float__(self): | |
573 | return self._b | |
574 | ||
1adf2230 | 575 | def __lt__(self, other): |
92f5a8d4 TL |
576 | if isinstance(other, Size): |
577 | return self._b < other._b | |
578 | else: | |
579 | return self.b < other | |
1adf2230 AA |
580 | |
581 | def __le__(self, other): | |
92f5a8d4 TL |
582 | if isinstance(other, Size): |
583 | return self._b <= other._b | |
584 | else: | |
585 | return self.b <= other | |
1adf2230 AA |
586 | |
587 | def __eq__(self, other): | |
92f5a8d4 TL |
588 | if isinstance(other, Size): |
589 | return self._b == other._b | |
590 | else: | |
591 | return self.b == other | |
1adf2230 AA |
592 | |
593 | def __ne__(self, other): | |
92f5a8d4 TL |
594 | if isinstance(other, Size): |
595 | return self._b != other._b | |
596 | else: | |
597 | return self.b != other | |
1adf2230 AA |
598 | |
599 | def __ge__(self, other): | |
92f5a8d4 TL |
600 | if isinstance(other, Size): |
601 | return self._b >= other._b | |
602 | else: | |
603 | return self.b >= other | |
1adf2230 AA |
604 | |
605 | def __gt__(self, other): | |
92f5a8d4 TL |
606 | if isinstance(other, Size): |
607 | return self._b > other._b | |
608 | else: | |
609 | return self.b > other | |
1adf2230 AA |
610 | |
611 | def __add__(self, other): | |
612 | if isinstance(other, Size): | |
613 | _b = self._b + other._b | |
614 | return Size(b=_b) | |
615 | raise TypeError('Cannot add "Size" object with int') | |
616 | ||
617 | def __sub__(self, other): | |
618 | if isinstance(other, Size): | |
619 | _b = self._b - other._b | |
620 | return Size(b=_b) | |
621 | raise TypeError('Cannot subtract "Size" object from int') | |
622 | ||
623 | def __mul__(self, other): | |
624 | if isinstance(other, Size): | |
625 | raise TypeError('Cannot multiply with "Size" object') | |
626 | _b = self._b * other | |
627 | return Size(b=_b) | |
628 | ||
629 | def __truediv__(self, other): | |
630 | if isinstance(other, Size): | |
631 | return self._b / other._b | |
91327a77 AA |
632 | _b = self._b / other |
633 | return Size(b=_b) | |
1adf2230 AA |
634 | |
635 | def __div__(self, other): | |
636 | if isinstance(other, Size): | |
637 | return self._b / other._b | |
91327a77 AA |
638 | _b = self._b / other |
639 | return Size(b=_b) | |
1adf2230 | 640 | |
f91f0fd5 TL |
641 | def __bool__(self): |
642 | return self.b != 0 | |
643 | ||
644 | def __nonzero__(self): | |
645 | return self.__bool__() | |
646 | ||
1adf2230 AA |
647 | def __getattr__(self, unit): |
648 | """ | |
649 | Calculate units on the fly, relies on the fact that ``bytes`` has been | |
650 | converted at instantiation. Units that don't exist will trigger an | |
651 | ``AttributeError`` | |
652 | """ | |
653 | try: | |
654 | formatter = self._formatters[unit] | |
655 | except KeyError: | |
656 | raise AttributeError('Size object has not attribute "%s"' % unit) | |
657 | if unit in ['b', 'bytes']: | |
658 | return formatter(self._b) | |
659 | try: | |
660 | factor = self._factors[unit] | |
661 | except KeyError: | |
662 | raise AttributeError('Size object has not attribute "%s"' % unit) | |
663 | return formatter(float(self._b) / factor) | |
664 | ||
665 | ||
666 | def human_readable_size(size): | |
667 | """ | |
668 | Take a size in bytes, and transform it into a human readable size with up | |
669 | to two decimals of precision. | |
670 | """ | |
20effc67 TL |
671 | suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] |
672 | for suffix in suffixes: | |
673 | if size >= 1024: | |
674 | size = size / 1024 | |
675 | else: | |
676 | break | |
1adf2230 AA |
677 | return "{size:.2f} {suffix}".format( |
678 | size=size, | |
20effc67 | 679 | suffix=suffix) |
1adf2230 AA |
680 | |
681 | ||
81eedcae TL |
682 | def size_from_human_readable(s): |
683 | """ | |
684 | Takes a human readable string and converts into a Size. If no unit is | |
685 | passed, bytes is assumed. | |
686 | """ | |
687 | s = s.replace(' ', '') | |
688 | if s[-1].isdigit(): | |
689 | return Size(b=float(s)) | |
690 | n = float(s[:-1]) | |
20effc67 TL |
691 | if s[-1].lower() == 'p': |
692 | return Size(pb=n) | |
81eedcae TL |
693 | if s[-1].lower() == 't': |
694 | return Size(tb=n) | |
695 | if s[-1].lower() == 'g': | |
696 | return Size(gb=n) | |
697 | if s[-1].lower() == 'm': | |
698 | return Size(mb=n) | |
699 | if s[-1].lower() == 'k': | |
700 | return Size(kb=n) | |
701 | return None | |
702 | ||
703 | ||
1adf2230 AA |
704 | def get_partitions_facts(sys_block_path): |
705 | partition_metadata = {} | |
706 | for folder in os.listdir(sys_block_path): | |
707 | folder_path = os.path.join(sys_block_path, folder) | |
708 | if os.path.exists(os.path.join(folder_path, 'partition')): | |
709 | contents = get_file_contents(os.path.join(folder_path, 'partition')) | |
f64942e4 | 710 | if contents: |
1adf2230 AA |
711 | part = {} |
712 | partname = folder | |
713 | part_sys_block_path = os.path.join(sys_block_path, partname) | |
714 | ||
715 | part['start'] = get_file_contents(part_sys_block_path + "/start", 0) | |
716 | part['sectors'] = get_file_contents(part_sys_block_path + "/size", 0) | |
717 | ||
718 | part['sectorsize'] = get_file_contents( | |
719 | part_sys_block_path + "/queue/logical_block_size") | |
720 | if not part['sectorsize']: | |
721 | part['sectorsize'] = get_file_contents( | |
722 | part_sys_block_path + "/queue/hw_sector_size", 512) | |
92f5a8d4 TL |
723 | part['size'] = float(part['sectors']) * 512 |
724 | part['human_readable_size'] = human_readable_size(float(part['sectors']) * 512) | |
f64942e4 AA |
725 | part['holders'] = [] |
726 | for holder in os.listdir(part_sys_block_path + '/holders'): | |
727 | part['holders'].append(holder) | |
1adf2230 AA |
728 | |
729 | partition_metadata[partname] = part | |
730 | return partition_metadata | |
731 | ||
732 | ||
733 | def is_mapper_device(device_name): | |
734 | return device_name.startswith(('/dev/mapper', '/dev/dm-')) | |
735 | ||
736 | ||
91327a77 AA |
737 | def is_locked_raw_device(disk_path): |
738 | """ | |
739 | A device can be locked by a third party software like a database. | |
740 | To detect that case, the device is opened in Read/Write and exclusive mode | |
741 | """ | |
742 | open_flags = (os.O_RDWR | os.O_EXCL) | |
743 | open_mode = 0 | |
744 | fd = None | |
745 | ||
746 | try: | |
747 | fd = os.open(disk_path, open_flags, open_mode) | |
748 | except OSError: | |
749 | return 1 | |
750 | ||
751 | try: | |
752 | os.close(fd) | |
753 | except OSError: | |
754 | return 1 | |
755 | ||
756 | return 0 | |
757 | ||
758 | ||
2a845540 TL |
759 | class AllowLoopDevices(object): |
760 | allow = False | |
761 | warned = False | |
92f5a8d4 | 762 | |
2a845540 TL |
763 | @classmethod |
764 | def __call__(cls): | |
765 | val = os.environ.get("CEPH_VOLUME_ALLOW_LOOP_DEVICES", "false").lower() | |
766 | if val not in ("false", 'no', '0'): | |
767 | cls.allow = True | |
768 | if not cls.warned: | |
769 | logger.warning( | |
770 | "CEPH_VOLUME_ALLOW_LOOP_DEVICES is set in your " | |
771 | "environment, so we will allow the use of unattached loop" | |
772 | " devices as disks. This feature is intended for " | |
773 | "development purposes only and will never be supported in" | |
774 | " production. Issues filed based on this behavior will " | |
775 | "likely be ignored." | |
776 | ) | |
777 | cls.warned = True | |
778 | return cls.allow | |
779 | ||
780 | ||
781 | allow_loop_devices = AllowLoopDevices() | |
782 | ||
783 | ||
784 | def get_block_devs_sysfs(_sys_block_path='/sys/block', _sys_dev_block_path='/sys/dev/block'): | |
785 | def holder_inner_loop(): | |
786 | for holder in holders: | |
787 | # /sys/block/sdy/holders/dm-8/dm/uuid | |
788 | holder_dm_type = get_file_contents(os.path.join(_sys_block_path, dev, f'holders/{holder}/dm/uuid')).split('-')[0].lower() | |
789 | if holder_dm_type == 'mpath': | |
790 | return True | |
791 | ||
792 | # First, get devices that are _not_ partitions | |
793 | result = list() | |
794 | dev_names = os.listdir(_sys_block_path) | |
795 | for dev in dev_names: | |
796 | name = kname = os.path.join("/dev", dev) | |
797 | if not os.path.exists(name): | |
798 | continue | |
799 | type_ = 'disk' | |
800 | holders = os.listdir(os.path.join(_sys_block_path, dev, 'holders')) | |
801 | if get_file_contents(os.path.join(_sys_block_path, dev, 'removable')) == "1": | |
802 | continue | |
803 | if holder_inner_loop(): | |
804 | continue | |
805 | dm_dir_path = os.path.join(_sys_block_path, dev, 'dm') | |
806 | if os.path.isdir(dm_dir_path): | |
807 | dm_type = get_file_contents(os.path.join(dm_dir_path, 'uuid')) | |
808 | type_ = dm_type.split('-')[0].lower() | |
809 | basename = get_file_contents(os.path.join(dm_dir_path, 'name')) | |
810 | name = os.path.join("/dev/mapper", basename) | |
811 | if dev.startswith('loop'): | |
812 | if not allow_loop_devices(): | |
813 | continue | |
814 | # Skip loop devices that are not attached | |
815 | if not os.path.exists(os.path.join(_sys_block_path, dev, 'loop')): | |
816 | continue | |
817 | type_ = 'loop' | |
818 | result.append([kname, name, type_]) | |
819 | # Next, look for devices that _are_ partitions | |
820 | for item in os.listdir(_sys_dev_block_path): | |
821 | is_part = get_file_contents(os.path.join(_sys_dev_block_path, item, 'partition')) == "1" | |
822 | dev = os.path.basename(os.readlink(os.path.join(_sys_dev_block_path, item))) | |
823 | if not is_part: | |
824 | continue | |
825 | name = kname = os.path.join("/dev", dev) | |
826 | result.append([name, kname, "part"]) | |
827 | return sorted(result, key=lambda x: x[0]) | |
92f5a8d4 TL |
828 | |
829 | ||
2a845540 | 830 | def get_devices(_sys_block_path='/sys/block', device=''): |
1adf2230 | 831 | """ |
92f5a8d4 TL |
832 | Captures all available block devices as reported by lsblk. |
833 | Additional interesting metadata like sectors, size, vendor, | |
834 | solid/rotational, etc. is collected from /sys/block/<device> | |
1adf2230 AA |
835 | |
836 | Returns a dictionary, where keys are the full paths to devices. | |
837 | ||
1adf2230 AA |
838 | ..note:: loop devices, removable media, and logical volumes are never included. |
839 | """ | |
1adf2230 AA |
840 | |
841 | device_facts = {} | |
842 | ||
2a845540 TL |
843 | block_devs = get_block_devs_sysfs(_sys_block_path) |
844 | ||
845 | block_types = ['disk', 'mpath'] | |
846 | if allow_loop_devices(): | |
847 | block_types.append('loop') | |
1adf2230 AA |
848 | |
849 | for block in block_devs: | |
92f5a8d4 TL |
850 | devname = os.path.basename(block[0]) |
851 | diskname = block[1] | |
2a845540 | 852 | if block[2] not in block_types: |
91327a77 | 853 | continue |
92f5a8d4 TL |
854 | sysdir = os.path.join(_sys_block_path, devname) |
855 | metadata = {} | |
1adf2230 | 856 | |
2a845540 TL |
857 | # If the device is ceph rbd it gets excluded |
858 | if is_ceph_rbd(diskname): | |
859 | continue | |
860 | ||
1adf2230 AA |
861 | # If the mapper device is a logical volume it gets excluded |
862 | if is_mapper_device(diskname): | |
cd265ab1 | 863 | if lvm.get_device_lvs(diskname): |
1adf2230 AA |
864 | continue |
865 | ||
92f5a8d4 TL |
866 | # all facts that have no defaults |
867 | # (<name>, <path relative to _sys_block_path>) | |
868 | facts = [('removable', 'removable'), | |
869 | ('ro', 'ro'), | |
870 | ('vendor', 'device/vendor'), | |
871 | ('model', 'device/model'), | |
872 | ('rev', 'device/rev'), | |
873 | ('sas_address', 'device/sas_address'), | |
874 | ('sas_device_handle', 'device/sas_device_handle'), | |
875 | ('support_discard', 'queue/discard_granularity'), | |
876 | ('rotational', 'queue/rotational'), | |
877 | ('nr_requests', 'queue/nr_requests'), | |
878 | ] | |
879 | for key, file_ in facts: | |
880 | metadata[key] = get_file_contents(os.path.join(sysdir, file_)) | |
91327a77 | 881 | |
2a845540 TL |
882 | device_slaves = os.listdir(os.path.join(sysdir, 'slaves')) |
883 | if device_slaves: | |
884 | metadata['device_nodes'] = ','.join(device_slaves) | |
885 | else: | |
886 | metadata['device_nodes'] = devname | |
887 | ||
1adf2230 AA |
888 | metadata['scheduler_mode'] = "" |
889 | scheduler = get_file_contents(sysdir + "/queue/scheduler") | |
890 | if scheduler is not None: | |
891 | m = re.match(r".*?(\[(.*)\])", scheduler) | |
892 | if m: | |
893 | metadata['scheduler_mode'] = m.group(2) | |
894 | ||
92f5a8d4 TL |
895 | metadata['partitions'] = get_partitions_facts(sysdir) |
896 | ||
897 | size = get_file_contents(os.path.join(sysdir, 'size'), 0) | |
898 | ||
899 | metadata['sectors'] = get_file_contents(os.path.join(sysdir, 'sectors'), 0) | |
900 | fallback_sectorsize = get_file_contents(sysdir + "/queue/hw_sector_size", 512) | |
901 | metadata['sectorsize'] = get_file_contents(sysdir + | |
902 | "/queue/logical_block_size", | |
903 | fallback_sectorsize) | |
1adf2230 | 904 | metadata['size'] = float(size) * 512 |
92f5a8d4 | 905 | metadata['human_readable_size'] = human_readable_size(metadata['size']) |
1adf2230 | 906 | metadata['path'] = diskname |
91327a77 | 907 | metadata['locked'] = is_locked_raw_device(metadata['path']) |
2a845540 | 908 | metadata['type'] = block[2] |
1adf2230 AA |
909 | |
910 | device_facts[diskname] = metadata | |
911 | return device_facts | |
522d829b TL |
912 | |
913 | def has_bluestore_label(device_path): | |
914 | isBluestore = False | |
915 | bluestoreDiskSignature = 'bluestore block device' # 22 bytes long | |
916 | ||
917 | # throws OSError on failure | |
918 | logger.info("opening device {} to check for BlueStore label".format(device_path)) | |
2a845540 TL |
919 | try: |
920 | with open(device_path, "rb") as fd: | |
921 | # read first 22 bytes looking for bluestore disk signature | |
922 | signature = fd.read(22) | |
923 | if signature.decode('ascii', 'replace') == bluestoreDiskSignature: | |
924 | isBluestore = True | |
925 | except IsADirectoryError: | |
926 | logger.info(f'{device_path} is a directory, skipping.') | |
522d829b TL |
927 | |
928 | return isBluestore |