ceph/src/ceph-volume/ceph_volume/tests/devices/raw/test_list.py

   1 import pytest
   2 from mock.mock import patch
   3 from ceph_volume.devices import raw
   4
   5 # Sample lsblk output is below that overviews the test scenario. (--json output for reader clarity)
   6 #  - sda and all its children are used for the OS
   7 #  - sdb is a bluestore OSD with phantom Atari partitions
   8 #  - sdc is an empty disk
   9 #  - sdd has 2 LVM device children
  10 # > lsblk --paths --json
  11 #   {
  12 #      "blockdevices": [
  13 #         {"name": "/dev/sda", "maj:min": "8:0", "rm": "0", "size": "128G", "ro": "0", "type": "disk", "mountpoint": null,
  14 #            "children": [
  15 #               {"name": "/dev/sda1", "maj:min": "8:1", "rm": "0", "size": "487M", "ro": "0", "type": "part", "mountpoint": null},
  16 #               {"name": "/dev/sda2", "maj:min": "8:2", "rm": "0", "size": "1.9G", "ro": "0", "type": "part", "mountpoint": null},
  17 #               {"name": "/dev/sda3", "maj:min": "8:3", "rm": "0", "size": "125.6G", "ro": "0", "type": "part", "mountpoint": "/etc/hosts"}
  18 #            ]
  19 #         },
  20 #         {"name": "/dev/sdb", "maj:min": "8:16", "rm": "0", "size": "1T", "ro": "0", "type": "disk", "mountpoint": null,
  21 #            "children": [
  22 #               {"name": "/dev/sdb2", "maj:min": "8:18", "rm": "0", "size": "48G", "ro": "0", "type": "part", "mountpoint": null},
  23 #               {"name": "/dev/sdb3", "maj:min": "8:19", "rm": "0", "size": "6M", "ro": "0", "type": "part", "mountpoint": null}
  24 #            ]
  25 #         },
  26 #         {"name": "/dev/sdc", "maj:min": "8:32", "rm": "0", "size": "1T", "ro": "0", "type": "disk", "mountpoint": null},
  27 #         {"name": "/dev/sdd", "maj:min": "8:48", "rm": "0", "size": "1T", "ro": "0", "type": "disk", "mountpoint": null,
  28 #            "children": [
  29 #               {"name": "/dev/mapper/ceph--osd--block--1", "maj:min": "253:0", "rm": "0", "size": "512G", "ro": "0", "type": "lvm", "mountpoint": null},
  30 #               {"name": "/dev/mapper/ceph--osd--block--2", "maj:min": "253:1", "rm": "0", "size": "512G", "ro": "0", "type": "lvm", "mountpoint": null}
  31 #            ]
  32 #         }
  33 #      ]
  34 #   }
  35
  36 def _devices_side_effect():
  37     return {
  38         "/dev/sda": {},
  39         "/dev/sda1": {},
  40         "/dev/sda2": {},
  41         "/dev/sda3": {},
  42         "/dev/sdb": {},
  43         "/dev/sdb2": {},
  44         "/dev/sdb3": {},
  45         "/dev/sdc": {},
  46         "/dev/sdd": {},
  47         "/dev/mapper/ceph--osd--block--1": {},
  48         "/dev/mapper/ceph--osd--block--2": {},
  49     }
  50
  51 def _lsblk_list_output():
  52     return [
  53         '/dev/sda',
  54         '/dev/sda1',
  55         '/dev/sda2',
  56         '/dev/sda3',
  57         '/dev/sdb',
  58         '/dev/sdb2',
  59         '/dev/sdb3',
  60         '/dev/sdc',
  61         '/dev/sdd',
  62         '/dev/mapper/ceph--osd--block--1',
  63         '/dev/mapper/ceph--osd--block--2',
  64     ]
  65
  66 # dummy lsblk output for device with optional parent output
  67 def _lsblk_output(dev, parent=None):
  68     if parent is None:
  69         parent = ""
  70     ret = 'NAME="{}" KNAME="{}" PKNAME="{}"'.format(dev, dev, parent)
  71     return [ret] # needs to be in a list form
  72
  73 def _bluestore_tool_label_output_sdb():
  74     return '''{
  75     "/dev/sdb": {
  76         "osd_uuid": "sdb-uuid",
  77         "size": 1099511627776,
  78         "btime": "2021-07-23T16:02:22.809186+0000",
  79         "description": "main",
  80         "bfm_blocks": "268435456",
  81         "bfm_blocks_per_key": "128",
  82         "bfm_bytes_per_block": "4096",
  83         "bfm_size": "1099511627776",
  84         "bluefs": "1",
  85         "ceph_fsid": "sdb-fsid",
  86         "kv_backend": "rocksdb",
  87         "magic": "ceph osd volume v026",
  88         "mkfs_done": "yes",
  89         "osd_key": "AQAO6PpgK+y4CBAAixq/X7OVimbaezvwD/cDmg==",
  90         "ready": "ready",
  91         "require_osd_release": "16",
  92         "whoami": "0"
  93     }
  94 }'''
  95
  96 def _bluestore_tool_label_output_sdb2():
  97     return '''{
  98     "/dev/sdb2": {
  99         "osd_uuid": "sdb2-uuid",
 100         "size": 1099511627776,
 101         "btime": "2021-07-23T16:02:22.809186+0000",
 102         "description": "main",
 103         "bfm_blocks": "268435456",
 104         "bfm_blocks_per_key": "128",
 105         "bfm_bytes_per_block": "4096",
 106         "bfm_size": "1099511627776",
 107         "bluefs": "1",
 108         "ceph_fsid": "sdb2-fsid",
 109         "kv_backend": "rocksdb",
 110         "magic": "ceph osd volume v026",
 111         "mkfs_done": "yes",
 112         "osd_key": "AQAO6PpgK+y4CBAAixq/X7OVimbaezvwD/cDmg==",
 113         "ready": "ready",
 114         "require_osd_release": "16",
 115         "whoami": "2"
 116     }
 117 }'''
 118
 119 def _bluestore_tool_label_output_dm_okay():
 120     return '''{
 121     "/dev/mapper/ceph--osd--block--1": {
 122         "osd_uuid": "lvm-1-uuid",
 123         "size": 549751619584,
 124         "btime": "2021-07-23T16:04:37.881060+0000",
 125         "description": "main",
 126         "bfm_blocks": "134216704",
 127         "bfm_blocks_per_key": "128",
 128         "bfm_bytes_per_block": "4096",
 129         "bfm_size": "549751619584",
 130         "bluefs": "1",
 131         "ceph_fsid": "lvm-1-fsid",
 132         "kv_backend": "rocksdb",
 133         "magic": "ceph osd volume v026",
 134         "mkfs_done": "yes",
 135         "osd_key": "AQCU6Ppgz+UcIRAAh6IUjtPjiXBlEXfwO8ixzw==",
 136         "ready": "ready",
 137         "require_osd_release": "16",
 138         "whoami": "2"
 139     }
 140 }'''
 141
 142 def _process_call_side_effect(command, **kw):
 143     if "lsblk" in command:
 144         if "/dev/" in command[-1]:
 145             dev = command[-1]
 146             if dev == "/dev/sda1" or dev == "/dev/sda2" or dev == "/dev/sda3":
 147                 return _lsblk_output(dev, parent="/dev/sda"), '', 0
 148             if dev == "/dev/sdb2" or dev == "/dev/sdb3":
 149                 return _lsblk_output(dev, parent="/dev/sdb"), '', 0
 150             if dev == "/dev/sda" or dev == "/dev/sdb" or dev == "/dev/sdc" or dev == "/dev/sdd":
 151                 return _lsblk_output(dev), '', 0
 152             if "mapper" in dev:
 153                 return _lsblk_output(dev, parent="/dev/sdd"), '', 0
 154             pytest.fail('dev {} needs behavior specified for it'.format(dev))
 155         if "/dev/" not in command:
 156             return _lsblk_list_output(), '', 0
 157         pytest.fail('command {} needs behavior specified for it'.format(command))
 158
 159     if "ceph-bluestore-tool" in command:
 160         if "/dev/sdb" in command:
 161             # sdb is a bluestore OSD
 162             return _bluestore_tool_label_output_sdb(), '', 0
 163         if "/dev/sdb2" in command:
 164             # sdb2 is a phantom atari partition that appears to have some valid bluestore info
 165             return _bluestore_tool_label_output_sdb2(), '', 0
 166         if "/dev/mapper/ceph--osd--block--1" in command:
 167             # dm device 1 is a valid bluestore OSD (the other is corrupted/invalid)
 168             return _bluestore_tool_label_output_dm_okay(), '', 0
 169         # sda and children, sdb's children, sdc, sdd, dm device 2 all do NOT have bluestore OSD data
 170         return [], 'fake No such file or directory error', 1
 171     pytest.fail('command {} needs behavior specified for it'.format(command))
 172
 173 def _has_bluestore_label_side_effect(disk_path):
 174     if "/dev/sda" in disk_path:
 175         return False # disk and all children are for the OS
 176     if disk_path == "/dev/sdb":
 177         return True # sdb is a valid bluestore OSD
 178     if disk_path == "/dev/sdb2":
 179         return True # sdb2 appears to be a valid bluestore OSD even though it should not be
 180     if disk_path == "/dev/sdc":
 181         return False # empty disk
 182     if disk_path == "/dev/sdd":
 183         return False # has LVM subdevices
 184     if disk_path == "/dev/mapper/ceph--osd--block--1":
 185         return True # good OSD
 186     if disk_path == "/dev/mapper/ceph--osd--block--2":
 187         return False # corrupted
 188     pytest.fail('device {} needs behavior specified for it'.format(disk_path))
 189
 190 class TestList(object):
 191
 192     @patch('ceph_volume.util.device.disk.get_devices')
 193     @patch('ceph_volume.util.disk.has_bluestore_label')
 194     @patch('ceph_volume.process.call')
 195     def test_raw_list(self, patched_call, patched_bluestore_label, patched_get_devices):
 196         raw.list.logger.setLevel("DEBUG")
 197         patched_call.side_effect = _process_call_side_effect
 198         patched_bluestore_label.side_effect = _has_bluestore_label_side_effect
 199         patched_get_devices.side_effect = _devices_side_effect
 200
 201         result = raw.list.List([]).generate()
 202         patched_call.assert_any_call(['lsblk', '--paths', '--output=NAME', '--noheadings', '--list'])
 203         assert len(result) == 2
 204
 205         sdb = result['sdb-uuid']
 206         assert sdb['osd_uuid'] == 'sdb-uuid'
 207         assert sdb['osd_id'] == 0
 208         assert sdb['device'] == '/dev/sdb'
 209         assert sdb['ceph_fsid'] == 'sdb-fsid'
 210         assert sdb['type'] == 'bluestore'
 211
 212         lvm1 = result['lvm-1-uuid']
 213         assert lvm1['osd_uuid'] == 'lvm-1-uuid'
 214         assert lvm1['osd_id'] == 2
 215         assert lvm1['device'] == '/dev/mapper/ceph--osd--block--1'
 216         assert lvm1['ceph_fsid'] == 'lvm-1-fsid'
 217         assert lvm1['type'] == 'bluestore'
 218
 219     @patch('ceph_volume.util.device.disk.get_devices')
 220     @patch('ceph_volume.util.disk.has_bluestore_label')
 221     @patch('ceph_volume.process.call')
 222     def test_raw_list_with_OSError(self, patched_call, patched_bluestore_label, patched_get_devices):
 223         def _has_bluestore_label_side_effect_with_OSError(device_path):
 224             if device_path == "/dev/sdd":
 225                 raise OSError('fake OSError')
 226             return _has_bluestore_label_side_effect(device_path)
 227
 228         raw.list.logger.setLevel("DEBUG")
 229         patched_call.side_effect = _process_call_side_effect
 230         patched_bluestore_label.side_effect = _has_bluestore_label_side_effect_with_OSError
 231         patched_get_devices.side_effect = _devices_side_effect
 232
 233         result = raw.list.List([]).generate()
 234         assert len(result) == 1
 235         assert 'sdb-uuid' in result