ceph/src/ceph-volume/ceph_volume/tests/devices/lvm/test_batch.py

   1 import pytest
   2 import json
   3 import random
   4
   5 from argparse import ArgumentError
   6 from mock import MagicMock, patch
   7
   8 from ceph_volume.devices.lvm import batch
   9 from ceph_volume.util import arg_validators
  10
  11
  12 class TestBatch(object):
  13
  14     def test_batch_instance(self, is_root):
  15         b = batch.Batch([])
  16         b.main()
  17
  18     def test_invalid_osd_ids_passed(self):
  19         with pytest.raises(SystemExit):
  20             batch.Batch(argv=['--osd-ids', '1', 'foo']).main()
  21
  22     def test_disjoint_device_lists(self, factory):
  23         device1 = factory(used_by_ceph=False, available=True, abspath="/dev/sda")
  24         device2 = factory(used_by_ceph=False, available=True, abspath="/dev/sdb")
  25         devices = [device1, device2]
  26         db_devices = [device2]
  27         with pytest.raises(Exception) as disjoint_ex:
  28             batch.ensure_disjoint_device_lists(devices, db_devices)
  29         assert 'Device lists are not disjoint' in str(disjoint_ex.value)
  30
  31     @patch('ceph_volume.util.arg_validators.Device')
  32     def test_reject_partition(self, mocked_device):
  33         mocked_device.return_value = MagicMock(
  34             is_partition=True,
  35             has_fs=False,
  36             is_lvm_member=False,
  37             has_gpt_headers=False,
  38             has_partitions=False,
  39         )
  40         with pytest.raises(ArgumentError):
  41             arg_validators.ValidBatchDevice()('foo')
  42
  43     @pytest.mark.parametrize('format_', ['pretty', 'json', 'json-pretty'])
  44     def test_report(self, format_, factory, conf_ceph_stub, mock_device_generator):
  45         # just ensure reporting works
  46         conf_ceph_stub('[global]\nfsid=asdf-lkjh')
  47         devs = [mock_device_generator() for _ in range(5)]
  48         args = factory(data_slots=1,
  49                        osds_per_device=1,
  50                        osd_ids=[],
  51                        report=True,
  52                        format=format_,
  53                        devices=devs,
  54                        db_devices=[],
  55                        wal_devices=[],
  56                        bluestore=True,
  57                        block_db_size="1G",
  58                        dmcrypt=True,
  59                        data_allocate_fraction=1.0,
  60                       )
  61         b = batch.Batch([])
  62         plan = b.get_plan(args)
  63         b.args = args
  64         b.report(plan)
  65
  66     @pytest.mark.parametrize('format_', ['json', 'json-pretty'])
  67     def test_json_report_valid_empty(self, format_, factory, conf_ceph_stub, mock_device_generator):
  68         # ensure json reports are valid when empty
  69         conf_ceph_stub('[global]\nfsid=asdf-lkjh')
  70         devs = []
  71         args = factory(data_slots=1,
  72                        osds_per_device=1,
  73                        osd_ids=[],
  74                        report=True,
  75                        format=format_,
  76                        devices=devs,
  77                        db_devices=[],
  78                        wal_devices=[],
  79                        bluestore=True,
  80                        block_db_size="1G",
  81                        dmcrypt=True,
  82                        data_allocate_fraction=1.0,
  83                       )
  84         b = batch.Batch([])
  85         plan = b.get_plan(args)
  86         b.args = args
  87         report = b._create_report(plan)
  88         json.loads(report)
  89
  90     @pytest.mark.parametrize('format_', ['json', 'json-pretty'])
  91     def test_json_report_valid_empty_unavailable_fast(self, format_, factory, conf_ceph_stub, mock_device_generator):
  92         # ensure json reports are valid when empty
  93         conf_ceph_stub('[global]\nfsid=asdf-lkjh')
  94         devs = [mock_device_generator() for _ in range(5)]
  95         fast_devs = [mock_device_generator()]
  96         fast_devs[0].available_lvm = False
  97         args = factory(data_slots=1,
  98                        osds_per_device=1,
  99                        osd_ids=[],
 100                        report=True,
 101                        format=format_,
 102                        devices=devs,
 103                        db_devices=fast_devs,
 104                        wal_devices=[],
 105                        bluestore=True,
 106                        block_db_size="1G",
 107                        dmcrypt=True,
 108                        data_allocate_fraction=1.0,
 109                       )
 110         b = batch.Batch([])
 111         plan = b.get_plan(args)
 112         b.args = args
 113         report = b._create_report(plan)
 114         json.loads(report)
 115
 116
 117     @pytest.mark.parametrize('format_', ['json', 'json-pretty'])
 118     def test_json_report_valid_empty_unavailable_very_fast(self, format_, factory, conf_ceph_stub, mock_device_generator):
 119         # ensure json reports are valid when empty
 120         conf_ceph_stub('[global]\nfsid=asdf-lkjh')
 121         devs = [mock_device_generator() for _ in range(5)]
 122         fast_devs = [mock_device_generator()]
 123         very_fast_devs = [mock_device_generator()]
 124         very_fast_devs[0].available_lvm = False
 125         args = factory(data_slots=1,
 126                        osds_per_device=1,
 127                        osd_ids=[],
 128                        report=True,
 129                        format=format_,
 130                        devices=devs,
 131                        db_devices=fast_devs,
 132                        wal_devices=very_fast_devs,
 133                        bluestore=True,
 134                        block_db_size="1G",
 135                        dmcrypt=True,
 136                        data_allocate_fraction=1.0,
 137                       )
 138         b = batch.Batch([])
 139         plan = b.get_plan(args)
 140         b.args = args
 141         report = b._create_report(plan)
 142         json.loads(report)
 143
 144     @pytest.mark.parametrize('rota', [0, 1])
 145     def test_batch_sort_full(self, factory, rota):
 146         device1 = factory(used_by_ceph=False, available=True, rotational=rota, abspath="/dev/sda")
 147         device2 = factory(used_by_ceph=False, available=True, rotational=rota, abspath="/dev/sdb")
 148         device3 = factory(used_by_ceph=False, available=True, rotational=rota, abspath="/dev/sdc")
 149         devices = [device1, device2, device3]
 150         args = factory(report=True,
 151                        devices=devices,
 152                        filestore=False,
 153                       )
 154         b = batch.Batch([])
 155         b.args = args
 156         b._sort_rotational_disks()
 157         assert len(b.args.devices) == 3
 158
 159     @pytest.mark.parametrize('objectstore', ['bluestore', 'filestore'])
 160     def test_batch_sort_mixed(self, factory, objectstore):
 161         device1 = factory(used_by_ceph=False, available=True, rotational=1, abspath="/dev/sda")
 162         device2 = factory(used_by_ceph=False, available=True, rotational=1, abspath="/dev/sdb")
 163         device3 = factory(used_by_ceph=False, available=True, rotational=0, abspath="/dev/sdc")
 164         devices = [device1, device2, device3]
 165         args = factory(report=True,
 166                        devices=devices,
 167                        filestore=False if objectstore == 'bluestore' else True,
 168                       )
 169         b = batch.Batch([])
 170         b.args = args
 171         b._sort_rotational_disks()
 172         assert len(b.args.devices) == 2
 173         if objectstore == 'bluestore':
 174             assert len(b.args.db_devices) == 1
 175         else:
 176             assert len(b.args.journal_devices) == 1
 177
 178     def test_get_physical_osds_return_len(self, factory,
 179                                           mock_devices_available,
 180                                           conf_ceph_stub,
 181                                           osds_per_device):
 182         conf_ceph_stub('[global]\nfsid=asdf-lkjh')
 183         args = factory(data_slots=1, osds_per_device=osds_per_device,
 184                        osd_ids=[], dmcrypt=False,
 185                        data_allocate_fraction=1.0)
 186         osds = batch.get_physical_osds(mock_devices_available, args)
 187         assert len(osds) == len(mock_devices_available) * osds_per_device
 188
 189     def test_get_physical_osds_rel_size(self, factory,
 190                                           mock_devices_available,
 191                                           conf_ceph_stub,
 192                                           osds_per_device,
 193                                           data_allocate_fraction):
 194         args = factory(data_slots=1, osds_per_device=osds_per_device,
 195                        osd_ids=[], dmcrypt=False,
 196                        data_allocate_fraction=data_allocate_fraction)
 197         osds = batch.get_physical_osds(mock_devices_available, args)
 198         for osd in osds:
 199             assert osd.data[1] == data_allocate_fraction / osds_per_device
 200
 201     def test_get_physical_osds_abs_size(self, factory,
 202                                           mock_devices_available,
 203                                           conf_ceph_stub,
 204                                           osds_per_device,
 205                                           data_allocate_fraction):
 206         conf_ceph_stub('[global]\nfsid=asdf-lkjh')
 207         args = factory(data_slots=1, osds_per_device=osds_per_device,
 208                        osd_ids=[], dmcrypt=False,
 209                        data_allocate_fraction=data_allocate_fraction)
 210         osds = batch.get_physical_osds(mock_devices_available, args)
 211         for osd, dev in zip(osds, mock_devices_available):
 212             assert osd.data[2] == int(dev.vg_size[0] * (data_allocate_fraction / osds_per_device))
 213
 214     def test_get_physical_osds_osd_ids(self, factory,
 215                                           mock_devices_available,
 216                                           osds_per_device):
 217         pass
 218
 219     def test_get_physical_fast_allocs_length(self, factory,
 220                                              conf_ceph_stub,
 221                                              mock_devices_available):
 222         conf_ceph_stub('[global]\nfsid=asdf-lkjh')
 223         args = factory(block_db_slots=None, get_block_db_size=None)
 224         fast = batch.get_physical_fast_allocs(mock_devices_available,
 225                                               'block_db', 2, 2, args)
 226         assert len(fast) == 2
 227
 228     def test_batch_fast_allocations_one_block_db_length(self, factory, conf_ceph_stub,
 229                                                   mock_lv_device_generator):
 230         conf_ceph_stub('[global]\nfsid=asdf-lkjh')
 231
 232         b = batch.Batch([])
 233         db_lv_devices = [mock_lv_device_generator()]
 234         fast = b.fast_allocations(db_lv_devices, 1, 0, 'block_db')
 235         assert len(fast) == 1
 236
 237     @pytest.mark.parametrize('occupied_prior', range(7))
 238     @pytest.mark.parametrize('slots,num_devs',
 239                              [l for sub in [list(zip([x]*x, range(1, x + 1))) for x in range(1,7)] for l in sub])
 240     def test_get_physical_fast_allocs_length_existing(self,
 241                                                       num_devs,
 242                                                       slots,
 243                                                       occupied_prior,
 244                                                       factory,
 245                                                       conf_ceph_stub,
 246                                                       mock_device_generator):
 247         conf_ceph_stub('[global]\nfsid=asdf-lkjh')
 248         occupied_prior = min(occupied_prior, slots)
 249         devs = [mock_device_generator() for _ in range(num_devs)]
 250         already_assigned = 0
 251         while already_assigned < occupied_prior:
 252             dev_i = random.randint(0, num_devs - 1)
 253             dev = devs[dev_i]
 254             if len(dev.lvs) < occupied_prior:
 255                 dev.lvs.append('foo')
 256                 dev.path = '/dev/bar'
 257                 already_assigned = sum([len(d.lvs) for d in devs])
 258         args = factory(block_db_slots=None, get_block_db_size=None)
 259         expected_num_osds = max(len(devs) * slots - occupied_prior, 0)
 260         fast = batch.get_physical_fast_allocs(devs,
 261                                               'block_db', slots,
 262                                               expected_num_osds, args)
 263         assert len(fast) == expected_num_osds
 264         expected_assignment_on_used_devices = sum([slots - len(d.lvs) for d in devs if len(d.lvs) > 0])
 265         assert len([f for f in fast if f[0] == '/dev/bar']) == expected_assignment_on_used_devices
 266         assert len([f for f in fast if f[0] != '/dev/bar']) == expected_num_osds - expected_assignment_on_used_devices
 267
 268     def test_get_lvm_osds_return_len(self, factory,
 269                                      mock_lv_device_generator,
 270                                      conf_ceph_stub,
 271                                      osds_per_device):
 272         conf_ceph_stub('[global]\nfsid=asdf-lkjh')
 273         args = factory(data_slots=1, osds_per_device=osds_per_device,
 274                        osd_ids=[], dmcrypt=False)
 275         mock_lvs = [mock_lv_device_generator()]
 276         osds = batch.get_lvm_osds(mock_lvs, args)
 277         assert len(osds) == 1
 278
 279
 280 class TestBatchOsd(object):
 281
 282     def test_osd_class_ctor(self):
 283         osd = batch.Batch.OSD('/dev/data', 1, '5G', 1, 1, None)
 284         assert osd.data == batch.Batch.OSD.VolSpec('/dev/data',
 285                                                    1,
 286                                                    '5G',
 287                                                    1,
 288                                                    'data')
 289     def test_add_fast(self):
 290         osd = batch.Batch.OSD('/dev/data', 1, '5G', 1, 1, None)
 291         osd.add_fast_device('/dev/db', 1, '5G', 1, 'block_db')
 292         assert osd.fast == batch.Batch.OSD.VolSpec('/dev/db',
 293                                                    1,
 294                                                    '5G',
 295                                                    1,
 296                                                    'block_db')
 297
 298     def test_add_very_fast(self):
 299         osd = batch.Batch.OSD('/dev/data', 1, '5G', 1, 1, None)
 300         osd.add_very_fast_device('/dev/wal', 1, '5G', 1)
 301         assert osd.very_fast == batch.Batch.OSD.VolSpec('/dev/wal',
 302                                                         1,
 303                                                         '5G',
 304                                                         1,
 305                                                         'block_wal')