ceph/src/python-common/ceph/tests/test_drive_group.py

   1 # flake8: noqa
   2 import re
   3
   4 import pytest
   5 import yaml
   6
   7 from ceph.deployment import drive_selection, translate
   8 from ceph.deployment.hostspec import HostSpec, SpecValidationError
   9 from ceph.deployment.inventory import Device
  10 from ceph.deployment.service_spec import PlacementSpec
  11 from ceph.tests.utils import _mk_inventory, _mk_device
  12 from ceph.deployment.drive_group import DriveGroupSpec, DeviceSelection, \
  13     DriveGroupValidationError
  14
  15 @pytest.mark.parametrize("test_input",
  16 [
  17     (  # new style json
  18         """service_type: osd
  19 service_id: testing_drivegroup
  20 placement:
  21   host_pattern: hostname
  22 data_devices:
  23   paths:
  24   - /dev/sda
  25 """
  26     ),
  27     (
  28         """service_type: osd
  29 service_id: testing_drivegroup
  30 placement:
  31   host_pattern: hostname
  32 data_devices:
  33   paths:
  34   - /dev/sda"""
  35     ),
  36 ])
  37 def test_DriveGroup(test_input):
  38     dg = DriveGroupSpec.from_json(yaml.safe_load(test_input))
  39     assert dg.service_id == 'testing_drivegroup'
  40     assert all([isinstance(x, Device) for x in dg.data_devices.paths])
  41     assert dg.data_devices.paths[0].path == '/dev/sda'
  42
  43 @pytest.mark.parametrize("match,test_input",
  44 [
  45     (
  46         re.escape('Service Spec is not an (JSON or YAML) object. got "None"'),
  47         ''
  48     ),
  49     (
  50         'Failed to validate OSD spec "<unnamed>": `placement` required',
  51         """data_devices:
  52   all: True
  53 """
  54     ),
  55     (
  56         'Failed to validate OSD spec "mydg.data_devices": device selection cannot be empty', """
  57 service_type: osd
  58 service_id: mydg
  59 placement:
  60   host_pattern: '*'
  61 data_devices:
  62   limit: 1
  63 """
  64     ),
  65     (
  66         'Failed to validate OSD spec "mydg": filter_logic must be either <AND> or <OR>', """
  67 service_type: osd
  68 service_id: mydg
  69 placement:
  70   host_pattern: '*'
  71 data_devices:
  72   all: True
  73 filter_logic: XOR
  74 """
  75     ),
  76     (
  77         'Failed to validate OSD spec "mydg": `data_devices` element is required.', """
  78 service_type: osd
  79 service_id: mydg
  80 placement:
  81   host_pattern: '*'
  82 spec:
  83   db_devices:
  84     model: model
  85 """
  86     ),
  87     (
  88         'Failed to validate OSD spec "mydg.db_devices": Filtering for `unknown_key` is not supported', """
  89 service_type: osd
  90 service_id: mydg
  91 placement:
  92   host_pattern: '*'
  93 spec:
  94   db_devices:
  95     unknown_key: 1
  96 """
  97     ),
  98     (
  99         'Failed to validate OSD spec "mydg": Feature `unknown_key` is not supported', """
 100 service_type: osd
 101 service_id: mydg
 102 placement:
 103   host_pattern: '*'
 104 spec:
 105   db_devices:
 106     all: true
 107   unknown_key: 1
 108 """
 109     ),
 110 ])
 111 def test_DriveGroup_fail(match, test_input):
 112     with pytest.raises(SpecValidationError, match=match):
 113         osd_spec = DriveGroupSpec.from_json(yaml.safe_load(test_input))
 114         osd_spec.validate()
 115
 116
 117 def test_drivegroup_pattern():
 118     dg = DriveGroupSpec(
 119         PlacementSpec(host_pattern='node[1-3]'),
 120         service_id='foobar',
 121         data_devices=DeviceSelection(all=True))
 122     assert dg.placement.filter_matching_hostspecs([HostSpec('node{}'.format(i)) for i in range(10)]) == ['node1', 'node2', 'node3']
 123
 124
 125 def test_drive_selection():
 126     devs = DeviceSelection(paths=['/dev/sda'])
 127     spec = DriveGroupSpec(
 128         PlacementSpec('node_name'),
 129         service_id='foobar',
 130         data_devices=devs)
 131     assert all([isinstance(x, Device) for x in spec.data_devices.paths])
 132     assert spec.data_devices.paths[0].path == '/dev/sda'
 133
 134     with pytest.raises(DriveGroupValidationError, match='exclusive'):
 135         ds = DeviceSelection(paths=['/dev/sda'], rotational=False)
 136         ds.validate('')
 137
 138
 139 def test_ceph_volume_command_0():
 140     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
 141                           service_id='foobar',
 142                           data_devices=DeviceSelection(all=True)
 143                           )
 144     spec.validate()
 145     inventory = _mk_inventory(_mk_device()*2)
 146     sel = drive_selection.DriveSelection(spec, inventory)
 147     cmds = translate.to_ceph_volume(sel, []).run()
 148     assert all(cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}'
 149
 150
 151 def test_ceph_volume_command_1():
 152     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
 153                           service_id='foobar',
 154                           data_devices=DeviceSelection(rotational=True),
 155                           db_devices=DeviceSelection(rotational=False)
 156                           )
 157     spec.validate()
 158     inventory = _mk_inventory(_mk_device(rotational=True)*2 + _mk_device(rotational=False)*2)
 159     sel = drive_selection.DriveSelection(spec, inventory)
 160     cmds = translate.to_ceph_volume(sel, []).run()
 161     assert all(cmd == ('lvm batch --no-auto /dev/sda /dev/sdb '
 162                    '--db-devices /dev/sdc /dev/sdd --yes --no-systemd') for cmd in cmds), f'Expected {cmd} in {cmds}'
 163
 164
 165 def test_ceph_volume_command_2():
 166     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
 167                           service_id='foobar',
 168                           data_devices=DeviceSelection(size='200GB:350GB', rotational=True),
 169                           db_devices=DeviceSelection(size='200GB:350GB', rotational=False),
 170                           wal_devices=DeviceSelection(size='10G')
 171                           )
 172     spec.validate()
 173     inventory = _mk_inventory(_mk_device(rotational=True, size="300.00 GB")*2 +
 174                               _mk_device(rotational=False, size="300.00 GB")*2 +
 175                               _mk_device(size="10.0 GB", rotational=False)*2
 176                               )
 177     sel = drive_selection.DriveSelection(spec, inventory)
 178     cmds = translate.to_ceph_volume(sel, []).run()
 179     assert all(cmd == ('lvm batch --no-auto /dev/sda /dev/sdb '
 180                    '--db-devices /dev/sdc /dev/sdd --wal-devices /dev/sde /dev/sdf '
 181                    '--yes --no-systemd') for cmd in cmds), f'Expected {cmd} in {cmds}'
 182
 183
 184 def test_ceph_volume_command_3():
 185     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
 186                           service_id='foobar',
 187                           data_devices=DeviceSelection(size='200GB:350GB', rotational=True),
 188                           db_devices=DeviceSelection(size='200GB:350GB', rotational=False),
 189                           wal_devices=DeviceSelection(size='10G'),
 190                           encrypted=True
 191                           )
 192     spec.validate()
 193     inventory = _mk_inventory(_mk_device(rotational=True, size="300.00 GB")*2 +
 194                               _mk_device(rotational=False, size="300.00 GB")*2 +
 195                               _mk_device(size="10.0 GB", rotational=False)*2
 196                               )
 197     sel = drive_selection.DriveSelection(spec, inventory)
 198     cmds = translate.to_ceph_volume(sel, []).run()
 199     assert all(cmd == ('lvm batch --no-auto /dev/sda /dev/sdb '
 200                    '--db-devices /dev/sdc /dev/sdd '
 201                    '--wal-devices /dev/sde /dev/sdf --dmcrypt '
 202                    '--yes --no-systemd') for cmd in cmds), f'Expected {cmd} in {cmds}'
 203
 204
 205 def test_ceph_volume_command_4():
 206     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
 207                           service_id='foobar',
 208                           data_devices=DeviceSelection(size='200GB:350GB', rotational=True),
 209                           db_devices=DeviceSelection(size='200GB:350GB', rotational=False),
 210                           wal_devices=DeviceSelection(size='10G'),
 211                           block_db_size='500M',
 212                           block_wal_size='500M',
 213                           osds_per_device=3,
 214                           encrypted=True
 215                           )
 216     spec.validate()
 217     inventory = _mk_inventory(_mk_device(rotational=True, size="300.00 GB")*2 +
 218                               _mk_device(rotational=False, size="300.00 GB")*2 +
 219                               _mk_device(size="10.0 GB", rotational=False)*2
 220                               )
 221     sel = drive_selection.DriveSelection(spec, inventory)
 222     cmds = translate.to_ceph_volume(sel, []).run()
 223     assert all(cmd == ('lvm batch --no-auto /dev/sda /dev/sdb '
 224                    '--db-devices /dev/sdc /dev/sdd --wal-devices /dev/sde /dev/sdf '
 225                    '--block-wal-size 500M --block-db-size 500M --dmcrypt '
 226                    '--osds-per-device 3 --yes --no-systemd') for cmd in cmds), f'Expected {cmd} in {cmds}'
 227
 228
 229 def test_ceph_volume_command_5():
 230     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
 231                           service_id='foobar',
 232                           data_devices=DeviceSelection(rotational=True),
 233                           objectstore='filestore'
 234                           )
 235     with pytest.raises(DriveGroupValidationError):
 236         spec.validate()
 237     inventory = _mk_inventory(_mk_device(rotational=True)*2)
 238     sel = drive_selection.DriveSelection(spec, inventory)
 239     cmds = translate.to_ceph_volume(sel, []).run()
 240     assert all(cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --filestore --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}'
 241
 242
 243 def test_ceph_volume_command_6():
 244     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
 245                           service_id='foobar',
 246                           data_devices=DeviceSelection(rotational=False),
 247                           journal_devices=DeviceSelection(rotational=True),
 248                           journal_size='500M',
 249                           objectstore='filestore'
 250                           )
 251     with pytest.raises(DriveGroupValidationError):
 252         spec.validate()
 253     inventory = _mk_inventory(_mk_device(rotational=True)*2 + _mk_device(rotational=False)*2)
 254     sel = drive_selection.DriveSelection(spec, inventory)
 255     cmds = translate.to_ceph_volume(sel, []).run()
 256     assert all(cmd == ('lvm batch --no-auto /dev/sdc /dev/sdd '
 257                    '--journal-size 500M --journal-devices /dev/sda /dev/sdb '
 258                    '--filestore --yes --no-systemd') for cmd in cmds), f'Expected {cmd} in {cmds}'
 259
 260
 261 def test_ceph_volume_command_7():
 262     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
 263                           service_id='foobar',
 264                           data_devices=DeviceSelection(all=True),
 265                           osd_id_claims={'host1': ['0', '1']}
 266                           )
 267     spec.validate()
 268     inventory = _mk_inventory(_mk_device(rotational=True)*2)
 269     sel = drive_selection.DriveSelection(spec, inventory)
 270     cmds = translate.to_ceph_volume(sel, ['0', '1']).run()
 271     assert all(cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --osd-ids 0 1 --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}'
 272
 273
 274 def test_ceph_volume_command_8():
 275     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
 276                           service_id='foobar',
 277                           data_devices=DeviceSelection(rotational=True, model='INTEL SSDS'),
 278                           db_devices=DeviceSelection(model='INTEL SSDP'),
 279                           filter_logic='OR',
 280                           osd_id_claims={}
 281                           )
 282     spec.validate()
 283     inventory = _mk_inventory(_mk_device(rotational=True,  size='1.82 TB',  model='ST2000DM001-1ER1') +  # data
 284                               _mk_device(rotational=False, size="223.0 GB", model='INTEL SSDSC2KG24') +  # data
 285                               _mk_device(rotational=False, size="349.0 GB", model='INTEL SSDPED1K375GA')  # wal/db
 286                               )
 287     sel = drive_selection.DriveSelection(spec, inventory)
 288     cmds = translate.to_ceph_volume(sel, []).run()
 289     assert all(cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --db-devices /dev/sdc --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}'
 290
 291
 292 def test_ceph_volume_command_9():
 293     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
 294                           service_id='foobar',
 295                           data_devices=DeviceSelection(all=True),
 296                           data_allocate_fraction=0.8
 297                           )
 298     spec.validate()
 299     inventory = _mk_inventory(_mk_device()*2)
 300     sel = drive_selection.DriveSelection(spec, inventory)
 301     cmds = translate.to_ceph_volume(sel, []).run()
 302     assert all(cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --data-allocate-fraction 0.8 --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}'
 303
 304 def test_raw_ceph_volume_command_0():
 305     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
 306                           service_id='foobar',
 307                           data_devices=DeviceSelection(rotational=True),
 308                           db_devices=DeviceSelection(rotational=False),
 309                           method='raw',
 310                           )
 311     spec.validate()
 312     inventory = _mk_inventory(_mk_device(rotational=True) +  # data
 313                               _mk_device(rotational=True) +  # data
 314                               _mk_device(rotational=False) +  # db
 315                               _mk_device(rotational=False)  # db
 316                               )
 317     exp_cmds = ['raw prepare --bluestore --data /dev/sda --block.db /dev/sdc', 'raw prepare --bluestore --data /dev/sdb --block.db /dev/sdd']
 318     sel = drive_selection.DriveSelection(spec, inventory)
 319     cmds = translate.to_ceph_volume(sel, []).run()
 320     assert all(cmd in exp_cmds for cmd in cmds), f'Expected {exp_cmds} to match {cmds}'
 321
 322 def test_raw_ceph_volume_command_1():
 323     spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'),
 324                           service_id='foobar',
 325                           data_devices=DeviceSelection(rotational=True),
 326                           db_devices=DeviceSelection(rotational=False),
 327                           method='raw',
 328                           )
 329     spec.validate()
 330     inventory = _mk_inventory(_mk_device(rotational=True) +  # data
 331                               _mk_device(rotational=True) +  # data
 332                               _mk_device(rotational=False) # db
 333                               )
 334     sel = drive_selection.DriveSelection(spec, inventory)
 335     with pytest.raises(ValueError):
 336         cmds = translate.to_ceph_volume(sel, []).run()