ceph/src/python-common/ceph/tests/test_disk_selector.py

   1 # flake8: noqa
   2 import pytest
   3
   4 from ceph.deployment.drive_selection.matchers import _MatchInvalid
   5 from ceph.deployment.inventory import Devices, Device
   6
   7 from ceph.deployment.drive_group import DriveGroupSpec, DeviceSelection, \
   8         DriveGroupValidationError
   9
  10 from ceph.deployment import drive_selection
  11 from ceph.deployment.service_spec import PlacementSpec
  12 from ceph.tests.factories import InventoryFactory
  13 from ceph.tests.utils import _mk_inventory, _mk_device
  14
  15
  16 class TestMatcher(object):
  17     """ Test Matcher base class
  18     """
  19
  20     def test_get_disk_key_3(self):
  21         """
  22         virtual is False
  23         key is found
  24         retrun value of key is expected
  25         """
  26         disk_map = Device(path='/dev/vdb', sys_api={'foo': 'bar'})
  27         ret = drive_selection.Matcher('foo', 'bar')._get_disk_key(disk_map)
  28         assert ret is disk_map.sys_api.get('foo')
  29
  30     def test_get_disk_key_4(self):
  31         """
  32         virtual is False
  33         key is not found
  34         expect raise Exception
  35         """
  36         disk_map = Device(path='/dev/vdb')
  37         with pytest.raises(Exception):
  38             drive_selection.Matcher('bar', 'foo')._get_disk_key(disk_map)
  39             pytest.fail("No disk_key found for foo or None")
  40
  41
  42 class TestSubstringMatcher(object):
  43     def test_compare(self):
  44         disk_dict = Device(path='/dev/vdb', sys_api=dict(model='samsung'))
  45         matcher = drive_selection.SubstringMatcher('model', 'samsung')
  46         ret = matcher.compare(disk_dict)
  47         assert ret is True
  48
  49     def test_compare_false(self):
  50         disk_dict = Device(path='/dev/vdb', sys_api=dict(model='nothing_matching'))
  51         matcher = drive_selection.SubstringMatcher('model', 'samsung')
  52         ret = matcher.compare(disk_dict)
  53         assert ret is False
  54
  55
  56 class TestEqualityMatcher(object):
  57     def test_compare(self):
  58         disk_dict = Device(path='/dev/vdb', sys_api=dict(rotates='1'))
  59         matcher = drive_selection.EqualityMatcher('rotates', '1')
  60         ret = matcher.compare(disk_dict)
  61         assert ret is True
  62
  63     def test_compare_false(self):
  64         disk_dict = Device(path='/dev/vdb', sys_api=dict(rotates='1'))
  65         matcher = drive_selection.EqualityMatcher('rotates', '0')
  66         ret = matcher.compare(disk_dict)
  67         assert ret is False
  68
  69
  70 class TestAllMatcher(object):
  71     def test_compare(self):
  72         disk_dict = Device(path='/dev/vdb')
  73         matcher = drive_selection.AllMatcher('all', 'True')
  74         ret = matcher.compare(disk_dict)
  75         assert ret is True
  76
  77     def test_compare_value_not_true(self):
  78         disk_dict = Device(path='/dev/vdb')
  79         matcher = drive_selection.AllMatcher('all', 'False')
  80         ret = matcher.compare(disk_dict)
  81         assert ret is True
  82
  83
  84 class TestSizeMatcher(object):
  85     def test_parse_filter_exact(self):
  86         """ Testing exact notation with 20G """
  87         matcher = drive_selection.SizeMatcher('size', '20G')
  88         assert isinstance(matcher.exact, tuple)
  89         assert matcher.exact[0] == '20'
  90         assert matcher.exact[1] == 'GB'
  91
  92     def test_parse_filter_exact_GB_G(self):
  93         """ Testing exact notation with 20G """
  94         matcher = drive_selection.SizeMatcher('size', '20GB')
  95         assert isinstance(matcher.exact, tuple)
  96         assert matcher.exact[0] == '20'
  97         assert matcher.exact[1] == 'GB'
  98
  99     def test_parse_filter_high_low(self):
 100         """ Testing high-low notation with 20G:50G """
 101
 102         matcher = drive_selection.SizeMatcher('size', '20G:50G')
 103         assert isinstance(matcher.exact, tuple)
 104         assert matcher.low[0] == '20'
 105         assert matcher.high[0] == '50'
 106         assert matcher.low[1] == 'GB'
 107         assert matcher.high[1] == 'GB'
 108
 109     def test_parse_filter_max_high(self):
 110         """ Testing high notation with :50G """
 111
 112         matcher = drive_selection.SizeMatcher('size', ':50G')
 113         assert isinstance(matcher.exact, tuple)
 114         assert matcher.high[0] == '50'
 115         assert matcher.high[1] == 'GB'
 116
 117     def test_parse_filter_min_low(self):
 118         """ Testing low notation with 20G: """
 119
 120         matcher = drive_selection.SizeMatcher('size', '50G:')
 121         assert isinstance(matcher.exact, tuple)
 122         assert matcher.low[0] == '50'
 123         assert matcher.low[1] == 'GB'
 124
 125     def test_to_byte_GB(self):
 126         """ Pretty nonesense test.."""
 127
 128         ret = drive_selection.SizeMatcher('size', '10G').to_byte(('10', 'GB'))
 129         assert ret == 10 * 1e+9
 130
 131     def test_to_byte_MB(self):
 132         """ Pretty nonesense test.."""
 133
 134         ret = drive_selection.SizeMatcher('size', '10M').to_byte(('10', 'MB'))
 135         assert ret == 10 * 1e+6
 136
 137     def test_to_byte_TB(self):
 138         """ Pretty nonesense test.."""
 139
 140         ret = drive_selection.SizeMatcher('size', '10T').to_byte(('10', 'TB'))
 141         assert ret == 10 * 1e+12
 142
 143     def test_to_byte_PB(self):
 144         """ Expect to raise """
 145
 146         with pytest.raises(_MatchInvalid):
 147             drive_selection.SizeMatcher('size', '10P').to_byte(('10', 'PB'))
 148         assert 'Unit \'P\' is not supported'
 149
 150     def test_compare_exact(self):
 151
 152         matcher = drive_selection.SizeMatcher('size', '20GB')
 153         disk_dict = Device(path='/dev/vdb', sys_api=dict(size='20.00 GB'))
 154         ret = matcher.compare(disk_dict)
 155         assert ret is True
 156
 157     def test_compare_exact_decimal(self):
 158
 159         matcher = drive_selection.SizeMatcher('size', '20.12GB')
 160         disk_dict = Device(path='/dev/vdb', sys_api=dict(size='20.12 GB'))
 161         ret = matcher.compare(disk_dict)
 162         assert ret is True
 163
 164     @pytest.mark.parametrize("test_input,expected", [
 165         ("1.00 GB", False),
 166         ("20.00 GB", True),
 167         ("50.00 GB", True),
 168         ("100.00 GB", True),
 169         ("101.00 GB", False),
 170         ("1101.00 GB", False),
 171     ])
 172     def test_compare_high_low(self, test_input, expected):
 173
 174         matcher = drive_selection.SizeMatcher('size', '20GB:100GB')
 175         disk_dict = Device(path='/dev/vdb', sys_api=dict(size=test_input))
 176         ret = matcher.compare(disk_dict)
 177         assert ret is expected
 178
 179     @pytest.mark.parametrize("test_input,expected", [
 180         ("1.00 GB", True),
 181         ("20.00 GB", True),
 182         ("50.00 GB", True),
 183         ("100.00 GB", False),
 184         ("101.00 GB", False),
 185         ("1101.00 GB", False),
 186     ])
 187     def test_compare_high(self, test_input, expected):
 188
 189         matcher = drive_selection.SizeMatcher('size', ':50GB')
 190         disk_dict = Device(path='/dev/vdb', sys_api=dict(size=test_input))
 191         ret = matcher.compare(disk_dict)
 192         assert ret is expected
 193
 194     @pytest.mark.parametrize("test_input,expected", [
 195         ("1.00 GB", False),
 196         ("20.00 GB", False),
 197         ("50.00 GB", True),
 198         ("100.00 GB", True),
 199         ("101.00 GB", True),
 200         ("1101.00 GB", True),
 201     ])
 202     def test_compare_low(self, test_input, expected):
 203
 204         matcher = drive_selection.SizeMatcher('size', '50GB:')
 205         disk_dict = Device(path='/dev/vdb', sys_api=dict(size=test_input))
 206         ret = matcher.compare(disk_dict)
 207         assert ret is expected
 208
 209     @pytest.mark.parametrize("test_input,expected", [
 210         ("1.00 GB", False),
 211         ("20.00 GB", False),
 212         ("50.00 GB", False),
 213         ("100.00 GB", False),
 214         ("101.00 GB", False),
 215         ("1101.00 GB", True),
 216         ("9.10 TB", True),
 217     ])
 218     def test_compare_at_least_1TB(self, test_input, expected):
 219
 220         matcher = drive_selection.SizeMatcher('size', '1TB:')
 221         disk_dict = Device(path='/dev/sdz', sys_api=dict(size=test_input))
 222         ret = matcher.compare(disk_dict)
 223         assert ret is expected
 224
 225     def test_compare_raise(self):
 226
 227         matcher = drive_selection.SizeMatcher('size', 'None')
 228         disk_dict = Device(path='/dev/vdb', sys_api=dict(size='20.00 GB'))
 229         with pytest.raises(Exception):
 230             matcher.compare(disk_dict)
 231             pytest.fail("Couldn't parse size")
 232
 233     @pytest.mark.parametrize("test_input,expected", [
 234         ("10G", ('10', 'GB')),
 235         ("20GB", ('20', 'GB')),
 236         ("10g", ('10', 'GB')),
 237         ("20gb", ('20', 'GB')),
 238     ])
 239     def test_get_k_v(self, test_input, expected):
 240         assert drive_selection.SizeMatcher('size', '10G')._get_k_v(test_input) == expected
 241
 242     @pytest.mark.parametrize("test_input,expected", [
 243         ("10G", ('GB')),
 244         ("10g", ('GB')),
 245         ("20GB", ('GB')),
 246         ("20gb", ('GB')),
 247         ("20TB", ('TB')),
 248         ("20tb", ('TB')),
 249         ("20T", ('TB')),
 250         ("20t", ('TB')),
 251         ("20MB", ('MB')),
 252         ("20mb", ('MB')),
 253         ("20M", ('MB')),
 254         ("20m", ('MB')),
 255     ])
 256     def test_parse_suffix(self, test_input, expected):
 257         assert drive_selection.SizeMatcher('size', '10G')._parse_suffix(test_input) == expected
 258
 259     @pytest.mark.parametrize("test_input,expected", [
 260         ("G", 'GB'),
 261         ("GB", 'GB'),
 262         ("TB", 'TB'),
 263         ("T", 'TB'),
 264         ("MB", 'MB'),
 265         ("M", 'MB'),
 266     ])
 267     def test_normalize_suffix(self, test_input, expected):
 268
 269         assert drive_selection.SizeMatcher('10G', 'size')._normalize_suffix(test_input) == expected
 270
 271     def test_normalize_suffix_raises(self):
 272
 273         with pytest.raises(_MatchInvalid):
 274             drive_selection.SizeMatcher('10P', 'size')._normalize_suffix("P")
 275             pytest.fail("Unit 'P' not supported")
 276
 277
 278 class TestDriveGroup(object):
 279     @pytest.fixture(scope='class')
 280     def test_fix(self, empty=None):
 281         def make_sample_data(empty=empty,
 282                              data_limit=0,
 283                              wal_limit=0,
 284                              db_limit=0,
 285                              osds_per_device='',
 286                              disk_format='bluestore'):
 287             raw_sample_bluestore = {
 288                 'service_type': 'osd',
 289                 'service_id': 'foo',
 290                 'placement': {'host_pattern': 'data*'},
 291                 'data_devices': {
 292                     'size': '30G:50G',
 293                     'model': '42-RGB',
 294                     'vendor': 'samsung',
 295                     'limit': data_limit
 296                 },
 297                 'wal_devices': {
 298                     'model': 'fast',
 299                     'limit': wal_limit
 300                 },
 301                 'db_devices': {
 302                     'size': ':20G',
 303                     'limit': db_limit
 304                 },
 305                 'db_slots': 5,
 306                 'wal_slots': 5,
 307                 'block_wal_size': '5G',
 308                 'block_db_size': '10G',
 309                 'objectstore': disk_format,
 310                 'osds_per_device': osds_per_device,
 311                 'encrypted': True,
 312             }
 313             raw_sample_filestore = {
 314                 'service_type': 'osd',
 315                 'service_id': 'foo',
 316                 'placement': {'host_pattern': 'data*'},
 317                 'objectstore': 'filestore',
 318                 'data_devices': {
 319                     'size': '30G:50G',
 320                     'model': 'foo',
 321                     'vendor': '1x',
 322                     'limit': data_limit
 323                 },
 324                 'journal_devices': {
 325                     'size': ':20G'
 326                 },
 327                 'journal_size': '5G',
 328                 'osds_per_device': osds_per_device,
 329                 'encrypted': True,
 330             }
 331             if disk_format == 'filestore':
 332                 raw_sample = raw_sample_filestore
 333             else:
 334                 raw_sample = raw_sample_bluestore
 335
 336             if empty:
 337                 raw_sample = {
 338                     'service_type': 'osd',
 339                     'service_id': 'foo',
 340                     'placement': {'host_pattern': 'data*'},
 341                     'data_devices': {
 342                         'all': True
 343                     },
 344                 }
 345
 346             dgo = DriveGroupSpec.from_json(raw_sample)
 347             return dgo
 348
 349         return make_sample_data
 350
 351     def test_encryption_prop(self, test_fix):
 352         test_fix = test_fix()
 353         assert test_fix.encrypted is True
 354
 355     def test_encryption_prop_empty(self, test_fix):
 356         test_fix = test_fix(empty=True)
 357         assert test_fix.encrypted is False
 358
 359     def test_db_slots_prop(self, test_fix):
 360         test_fix = test_fix()
 361         assert test_fix.db_slots == 5
 362
 363     def test_db_slots_prop_empty(self, test_fix):
 364         test_fix = test_fix(empty=True)
 365         assert test_fix.db_slots is None
 366
 367     def test_wal_slots_prop(self, test_fix):
 368         test_fix = test_fix()
 369         assert test_fix.wal_slots == 5
 370
 371     def test_wal_slots_prop_empty(self, test_fix):
 372         test_fix = test_fix(empty=True)
 373         assert test_fix.wal_slots is None
 374
 375     def test_block_wal_size_prop(self, test_fix):
 376         test_fix = test_fix()
 377         assert test_fix.block_wal_size == '5G'
 378
 379     def test_block_wal_size_prop_empty(self, test_fix):
 380         test_fix = test_fix(empty=True)
 381         assert test_fix.block_wal_size is None
 382
 383     def test_block_db_size_prop(self, test_fix):
 384         test_fix = test_fix()
 385         assert test_fix.block_db_size == '10G'
 386
 387     def test_block_db_size_prop_empty(self, test_fix):
 388         test_fix = test_fix(empty=True)
 389         assert test_fix.block_db_size is None
 390
 391     def test_data_devices_prop(self, test_fix):
 392         test_fix = test_fix()
 393         assert test_fix.data_devices == DeviceSelection(
 394             model='42-RGB',
 395             size='30G:50G',
 396             vendor='samsung',
 397             limit=0,
 398         )
 399
 400     def test_data_devices_prop_empty(self, test_fix):
 401         test_fix = test_fix(empty=True)
 402         assert test_fix.db_devices is None
 403
 404     def test_db_devices_prop(self, test_fix):
 405         test_fix = test_fix()
 406         assert test_fix.db_devices == DeviceSelection(
 407             size=':20G',
 408             limit=0,
 409         )
 410
 411     def test_db_devices_prop_empty(self, test_fix):
 412         test_fix = test_fix(empty=True)
 413         assert test_fix.db_devices is None
 414
 415     def test_wal_device_prop(self, test_fix):
 416         test_fix = test_fix()
 417         assert test_fix.wal_devices == DeviceSelection(
 418             model='fast',
 419             limit=0,
 420         )
 421
 422     def test_wal_device_prop_empty(self, test_fix):
 423         test_fix = test_fix(empty=True)
 424         assert test_fix.wal_devices is None
 425
 426     def test_bluestore_format_prop(self, test_fix):
 427         test_fix = test_fix(disk_format='bluestore')
 428         assert test_fix.objectstore == 'bluestore'
 429
 430     def test_default_format_prop(self, test_fix):
 431         test_fix = test_fix(empty=True)
 432         assert test_fix.objectstore == 'bluestore'
 433
 434     def test_osds_per_device(self, test_fix):
 435         test_fix = test_fix(osds_per_device='3')
 436         assert test_fix.osds_per_device == '3'
 437
 438     def test_osds_per_device_default(self, test_fix):
 439         test_fix = test_fix()
 440         assert test_fix.osds_per_device == ''
 441
 442     def test_journal_size_empty(self, test_fix):
 443         test_fix = test_fix(empty=True)
 444         assert test_fix.journal_size is None
 445
 446     @pytest.fixture
 447     def inventory(self, available=True):
 448         def make_sample_data(available=available,
 449                              data_devices=10,
 450                              wal_devices=0,
 451                              db_devices=2,
 452                              human_readable_size_data='50.00 GB',
 453                              human_readable_size_wal='20.00 GB',
 454                              size=5368709121,
 455                              human_readable_size_db='20.00 GB'):
 456             factory = InventoryFactory()
 457             inventory_sample = []
 458             data_disks = factory.produce(
 459                 pieces=data_devices,
 460                 available=available,
 461                 size=size,
 462                 human_readable_size=human_readable_size_data)
 463             wal_disks = factory.produce(
 464                 pieces=wal_devices,
 465                 human_readable_size=human_readable_size_wal,
 466                 rotational='0',
 467                 model='ssd_type_model',
 468                 size=size,
 469                 available=available)
 470             db_disks = factory.produce(
 471                 pieces=db_devices,
 472                 human_readable_size=human_readable_size_db,
 473                 rotational='0',
 474                 size=size,
 475                 model='ssd_type_model',
 476                 available=available)
 477             inventory_sample.extend(data_disks)
 478             inventory_sample.extend(wal_disks)
 479             inventory_sample.extend(db_disks)
 480
 481             return Devices(devices=inventory_sample)
 482
 483         return make_sample_data
 484
 485
 486 class TestDriveSelection(object):
 487
 488     testdata = [
 489         (
 490             DriveGroupSpec(
 491                 placement=PlacementSpec(host_pattern='*'),
 492                 service_id='foobar',
 493                 data_devices=DeviceSelection(all=True)),
 494             _mk_inventory(_mk_device() * 5),
 495             ['/dev/sda', '/dev/sdb', '/dev/sdc', '/dev/sdd', '/dev/sde'], []
 496         ),
 497         (
 498             DriveGroupSpec(
 499                 placement=PlacementSpec(host_pattern='*'),
 500                 service_id='foobar',
 501                 data_devices=DeviceSelection(all=True, limit=3),
 502                 db_devices=DeviceSelection(all=True)
 503             ),
 504             _mk_inventory(_mk_device() * 5),
 505             ['/dev/sda', '/dev/sdb', '/dev/sdc'], ['/dev/sdd', '/dev/sde']
 506         ),
 507         (
 508             DriveGroupSpec(
 509                 placement=PlacementSpec(host_pattern='*'),
 510                 service_id='foobar',
 511                 data_devices=DeviceSelection(rotational=True),
 512                 db_devices=DeviceSelection(rotational=False)
 513             ),
 514             _mk_inventory(_mk_device(rotational=False) + _mk_device(rotational=True)),
 515             ['/dev/sdb'], ['/dev/sda']
 516         ),
 517         (
 518             DriveGroupSpec(
 519                 placement=PlacementSpec(host_pattern='*'),
 520                 service_id='foobar',
 521                 data_devices=DeviceSelection(rotational=True),
 522                 db_devices=DeviceSelection(rotational=False)
 523             ),
 524             _mk_inventory(_mk_device(rotational=True)*2 + _mk_device(rotational=False)),
 525             ['/dev/sda', '/dev/sdb'], ['/dev/sdc']
 526         ),
 527         (
 528             DriveGroupSpec(
 529                 placement=PlacementSpec(host_pattern='*'),
 530                 service_id='foobar',
 531                 data_devices=DeviceSelection(rotational=True),
 532                 db_devices=DeviceSelection(rotational=False)
 533             ),
 534             _mk_inventory(_mk_device(rotational=True)*2),
 535             ['/dev/sda', '/dev/sdb'], []
 536         ),
 537     ]
 538
 539     @pytest.mark.parametrize("spec,inventory,expected_data,expected_db", testdata)
 540     def test_disk_selection(self, spec, inventory, expected_data, expected_db):
 541         sel = drive_selection.DriveSelection(spec, inventory)
 542         assert [d.path for d in sel.data_devices()] == expected_data
 543         assert [d.path for d in sel.db_devices()] == expected_db
 544
 545     def test_disk_selection_raise(self):
 546         spec = DriveGroupSpec(
 547                 placement=PlacementSpec(host_pattern='*'),
 548                 service_id='foobar',
 549                 data_devices=DeviceSelection(size='wrong'),
 550             )
 551         inventory = _mk_inventory(_mk_device(rotational=True)*2)
 552         m = 'Failed to validate OSD spec "foobar.data_devices": No filters applied'
 553         with pytest.raises(DriveGroupValidationError, match=m):
 554             drive_selection.DriveSelection(spec, inventory)