ceph/src/ceph-volume/ceph_volume/util/device.py

   1 # -*- coding: utf-8 -*-
   2
   3 import os
   4 from functools import total_ordering
   5 from ceph_volume import sys_info
   6 from ceph_volume.api import lvm
   7 from ceph_volume.util import disk
   8
   9 report_template = """
  10 {dev:<25} {size:<12} {rot!s:<7} {available!s:<9} {model}"""
  11
  12
  13 def encryption_status(abspath):
  14     """
  15     Helper function to run ``encryption.status()``. It is done here to avoid
  16     a circular import issue (encryption module imports from this module) and to
  17     ease testing by allowing monkeypatching of this function.
  18     """
  19     from ceph_volume.util import encryption
  20     return encryption.status(abspath)
  21
  22
  23 class Devices(object):
  24     """
  25     A container for Device instances with reporting
  26     """
  27
  28     def __init__(self, devices=None):
  29         if not sys_info.devices:
  30             sys_info.devices = disk.get_devices()
  31         self.devices = [Device(k) for k in
  32                             sys_info.devices.keys()]
  33
  34     def pretty_report(self, all=True):
  35         output = [
  36             report_template.format(
  37                 dev='Device Path',
  38                 size='Size',
  39                 rot='rotates',
  40                 model='Model name',
  41                 available='available',
  42             )]
  43         for device in sorted(self.devices):
  44             output.append(device.report())
  45         return ''.join(output)
  46
  47     def json_report(self):
  48         output = []
  49         for device in sorted(self.devices):
  50             output.append(device.json_report())
  51         return output
  52
  53 @total_ordering
  54 class Device(object):
  55
  56     pretty_template = """
  57      {attr:<25} {value}"""
  58
  59     report_fields = [
  60         'rejected_reasons',
  61         'available',
  62         'path',
  63         'sys_api',
  64     ]
  65     pretty_report_sys_fields = [
  66         'human_readable_size',
  67         'model',
  68         'removable',
  69         'ro',
  70         'rotational',
  71         'sas_address',
  72         'scheduler_mode',
  73         'vendor',
  74     ]
  75
  76     def __init__(self, path):
  77         self.path = path
  78         # LVs can have a vg/lv path, while disks will have /dev/sda
  79         self.abspath = path
  80         self.lv_api = None
  81         self.lvs = []
  82         self.vg_name = None
  83         self.lv_name = None
  84         self.pvs_api = []
  85         self.disk_api = {}
  86         self.blkid_api = {}
  87         self.sys_api = {}
  88         self._exists = None
  89         self._is_lvm_member = None
  90         self._parse()
  91         self.available, self.rejected_reasons = self._check_reject_reasons()
  92         self.device_id = self._get_device_id()
  93
  94     def __lt__(self, other):
  95         '''
  96         Implementing this method and __eq__ allows the @total_ordering
  97         decorator to turn the Device class into a totally ordered type.
  98         This can slower then implementing all comparison operations.
  99         This sorting should put available devices before unavailable devices
 100         and sort on the path otherwise (str sorting).
 101         '''
 102         if self.available == other.available:
 103             return self.path < other.path
 104         return self.available and not other.available
 105
 106     def __eq__(self, other):
 107         return self.path == other.path
 108
 109     def _parse(self):
 110         if not sys_info.devices:
 111             sys_info.devices = disk.get_devices()
 112         self.sys_api = sys_info.devices.get(self.abspath, {})
 113
 114         # start with lvm since it can use an absolute or relative path
 115         lv = lvm.get_lv_from_argument(self.path)
 116         if lv:
 117             self.lv_api = lv
 118             self.lvs = [lv]
 119             self.abspath = lv.lv_path
 120             self.vg_name = lv.vg_name
 121             self.lv_name = lv.name
 122         else:
 123             dev = disk.lsblk(self.path)
 124             self.blkid_api = disk.blkid(self.path)
 125             self.disk_api = dev
 126             device_type = dev.get('TYPE', '')
 127             # always check is this is an lvm member
 128             if device_type in ['part', 'disk']:
 129                 self._set_lvm_membership()
 130
 131         self.ceph_disk = CephDiskDevice(self)
 132
 133     def __repr__(self):
 134         prefix = 'Unknown'
 135         if self.is_lv:
 136             prefix = 'LV'
 137         elif self.is_partition:
 138             prefix = 'Partition'
 139         elif self.is_device:
 140             prefix = 'Raw Device'
 141         return '<%s: %s>' % (prefix, self.abspath)
 142
 143     def pretty_report(self):
 144         def format_value(v):
 145             if isinstance(v, list):
 146                 return ', '.join(v)
 147             else:
 148                 return v
 149         def format_key(k):
 150             return k.strip('_').replace('_', ' ')
 151         output = ['\n====== Device report {} ======\n'.format(self.path)]
 152         output.extend(
 153             [self.pretty_template.format(
 154                 attr=format_key(k),
 155                 value=format_value(v)) for k, v in vars(self).items() if k in
 156                 self.report_fields and k != 'disk_api' and k != 'sys_api'] )
 157         output.extend(
 158             [self.pretty_template.format(
 159                 attr=format_key(k),
 160                 value=format_value(v)) for k, v in self.sys_api.items() if k in
 161                 self.pretty_report_sys_fields])
 162         for lv in self.lvs:
 163             output.append("""
 164     --- Logical Volume ---""")
 165             output.extend(
 166                 [self.pretty_template.format(
 167                     attr=format_key(k),
 168                     value=format_value(v)) for k, v in lv.report().items()])
 169         return ''.join(output)
 170
 171     def report(self):
 172         return report_template.format(
 173             dev=self.abspath,
 174             size=self.size_human,
 175             rot=self.rotational,
 176             available=self.available,
 177             model=self.model,
 178         )
 179
 180     def json_report(self):
 181         output = {k.strip('_'): v for k, v in vars(self).items() if k in
 182                   self.report_fields}
 183         output['lvs'] = [lv.report() for lv in self.lvs]
 184         return output
 185
 186     def _get_device_id(self):
 187         """
 188         Please keep this implementation in sync with get_device_id() in
 189         src/common/blkdev.cc
 190         """
 191         props = ['ID_VENDOR','ID_MODEL','ID_SERIAL_SHORT', 'ID_SERIAL',
 192                  'ID_SCSI_SERIAL']
 193         p = disk.udevadm_property(self.abspath, props)
 194         if 'ID_VENDOR' in p and 'ID_MODEL' in p and 'ID_SCSI_SERIAL' in p:
 195             dev_id = '_'.join([p['ID_VENDOR'], p['ID_MODEL'],
 196                               p['ID_SCSI_SERIAL']])
 197         elif 'ID_MODEL' in p and 'ID_SERIAL_SHORT' in p:
 198             dev_id = '_'.join([p['ID_MODEL'], p['ID_SERIAL_SHORT']])
 199         elif 'ID_SERIAL' in p:
 200             dev_id = p['ID_SERIAL']
 201             if dev_id.startswith('MTFD'):
 202                 # Micron NVMes hide the vendor
 203                 dev_id = 'Micron_' + dev_id
 204         else:
 205             # the else branch should fallback to using sysfs and ioctl to
 206             # retrieve device_id on FreeBSD. Still figuring out if/how the
 207             # python ioctl implementation does that on FreeBSD
 208             dev_id = ''
 209         dev_id.replace(' ', '_')
 210         return dev_id
 211
 212     def _set_lvm_membership(self):
 213         if self._is_lvm_member is None:
 214             # this is contentious, if a PV is recognized by LVM but has no
 215             # VGs, should we consider it as part of LVM? We choose not to
 216             # here, because most likely, we need to use VGs from this PV.
 217             self._is_lvm_member = False
 218             for path in self._get_pv_paths():
 219                 # check if there was a pv created with the
 220                 # name of device
 221                 pvs = lvm.PVolumes()
 222                 pvs.filter(pv_name=path)
 223                 has_vgs = [pv.vg_name for pv in pvs if pv.vg_name]
 224                 if has_vgs:
 225                     self.vgs = list(set(has_vgs))
 226                     # a pv can only be in one vg, so this should be safe
 227                     self.vg_name = has_vgs[0]
 228                     self._is_lvm_member = True
 229                     self.pvs_api = pvs
 230                     for pv in pvs:
 231                         if pv.vg_name and pv.lv_uuid:
 232                             lv = lvm.get_lv(vg_name=pv.vg_name, lv_uuid=pv.lv_uuid)
 233                             if lv:
 234                                 self.lvs.append(lv)
 235                 else:
 236                     self.vgs = []
 237         return self._is_lvm_member
 238
 239     def _get_pv_paths(self):
 240         """
 241         For block devices LVM can reside on the raw block device or on a
 242         partition. Return a list of paths to be checked for a pv.
 243         """
 244         paths = [self.abspath]
 245         path_dir = os.path.dirname(self.abspath)
 246         for part in self.sys_api.get('partitions', {}).keys():
 247             paths.append(os.path.join(path_dir, part))
 248         return paths
 249
 250     @property
 251     def exists(self):
 252         return os.path.exists(self.abspath)
 253
 254     @property
 255     def has_gpt_headers(self):
 256         return self.blkid_api.get("PTTYPE") == "gpt"
 257
 258     @property
 259     def rotational(self):
 260         return self.sys_api['rotational'] == '1'
 261
 262     @property
 263     def model(self):
 264         return self.sys_api['model']
 265
 266     @property
 267     def size_human(self):
 268         return self.sys_api['human_readable_size']
 269
 270     @property
 271     def size(self):
 272             return self.sys_api['size']
 273
 274     @property
 275     def is_lvm_member(self):
 276         if self._is_lvm_member is None:
 277             self._set_lvm_membership()
 278         return self._is_lvm_member
 279
 280     @property
 281     def is_ceph_disk_member(self):
 282         is_member = self.ceph_disk.is_member
 283         if self.sys_api.get("partitions"):
 284             for part in self.sys_api.get("partitions").keys():
 285                 part = Device("/dev/%s" % part)
 286                 if part.is_ceph_disk_member:
 287                     is_member = True
 288                     break
 289         return is_member
 290
 291     @property
 292     def is_mapper(self):
 293         return self.path.startswith(('/dev/mapper', '/dev/dm-'))
 294
 295     @property
 296     def is_lv(self):
 297         return self.lv_api is not None
 298
 299     @property
 300     def is_partition(self):
 301         if self.disk_api:
 302             return self.disk_api['TYPE'] == 'part'
 303         return False
 304
 305     @property
 306     def is_device(self):
 307         if self.disk_api:
 308             is_device = self.disk_api['TYPE'] == 'device'
 309             is_disk = self.disk_api['TYPE'] == 'disk'
 310             if is_device or is_disk:
 311                 return True
 312         return False
 313
 314     @property
 315     def is_encrypted(self):
 316         """
 317         Only correct for LVs, device mappers, and partitions. Will report a ``None``
 318         for raw devices.
 319         """
 320         crypt_reports = [self.blkid_api.get('TYPE', ''), self.disk_api.get('FSTYPE', '')]
 321         if self.is_lv:
 322             # if disk APIs are reporting this is encrypted use that:
 323             if 'crypto_LUKS' in crypt_reports:
 324                 return True
 325             # if ceph-volume created this, then a tag would let us know
 326             elif self.lv_api.encrypted:
 327                 return True
 328             return False
 329         elif self.is_partition:
 330             return 'crypto_LUKS' in crypt_reports
 331         elif self.is_mapper:
 332             active_mapper = encryption_status(self.abspath)
 333             if active_mapper:
 334                 # normalize a bit to ensure same values regardless of source
 335                 encryption_type = active_mapper['type'].lower().strip('12')  # turn LUKS1 or LUKS2 into luks
 336                 return True if encryption_type in ['plain', 'luks'] else False
 337             else:
 338                 return False
 339         else:
 340             return None
 341
 342     @property
 343     def used_by_ceph(self):
 344         # only filter out data devices as journals could potentially be reused
 345         osd_ids = [lv.tags.get("ceph.osd_id") is not None for lv in self.lvs
 346                    if lv.tags.get("ceph.type") in ["data", "block"]]
 347         return any(osd_ids)
 348
 349     def _check_reject_reasons(self):
 350         """
 351         This checks a number of potential reject reasons for a drive and
 352         returns a tuple (boolean, list). The first element denotes whether a
 353         drive is available or not, the second element lists reasons in case a
 354         drive is not available.
 355         """
 356         reasons = [
 357             ('removable', 1, 'removable'),
 358             ('ro', 1, 'read-only'),
 359             ('locked', 1, 'locked'),
 360         ]
 361         rejected = [reason for (k, v, reason) in reasons if
 362                     self.sys_api.get(k, '') == v]
 363         if self.is_ceph_disk_member:
 364             rejected.append("Used by ceph-disk")
 365
 366         return len(rejected) == 0, rejected
 367
 368
 369 class CephDiskDevice(object):
 370     """
 371     Detect devices that have been created by ceph-disk, report their type
 372     (journal, data, etc..). Requires a ``Device`` object as input.
 373     """
 374
 375     def __init__(self, device):
 376         self.device = device
 377         self._is_ceph_disk_member = None
 378
 379     @property
 380     def partlabel(self):
 381         """
 382         In containers, the 'PARTLABEL' attribute might not be detected
 383         correctly via ``lsblk``, so we poke at the value with ``lsblk`` first,
 384         falling back to ``blkid`` (which works correclty in containers).
 385         """
 386         lsblk_partlabel = self.device.disk_api.get('PARTLABEL')
 387         if lsblk_partlabel:
 388             return lsblk_partlabel
 389         return self.device.blkid_api.get('PARTLABEL', '')
 390
 391     @property
 392     def is_member(self):
 393         if self._is_ceph_disk_member is None:
 394             if 'ceph' in self.partlabel:
 395                 self._is_ceph_disk_member = True
 396                 return True
 397             return False
 398         return self._is_ceph_disk_member
 399
 400     @property
 401     def type(self):
 402         types = [
 403             'data', 'wal', 'db', 'lockbox', 'journal',
 404             # ceph-disk uses 'ceph block' when placing data in bluestore, but
 405             # keeps the regular OSD files in 'ceph data' :( :( :( :(
 406             'block',
 407         ]
 408         for t in types:
 409             if t in self.partlabel:
 410                 return t
 411         return 'unknown'