[ceph.git] / ceph / src / pybind / mgr / cephadm / tests / test_osd_removal.py

import json

from cephadm.services.osd import OSDRemovalQueue, OSD
import pytest
from tests import mock
from .fixtures import with_cephadm_module
from datetime import datetime


class MockOSD:

    def __init__(self, osd_id):
        self.osd_id = osd_id


class TestOSDRemoval:

    @pytest.mark.parametrize(
        "osd_id, osd_df, expected",
        [
            # missing 'nodes' key
            (1, dict(nodes=[]), -1),
            # missing 'pgs' key
            (1, dict(nodes=[dict(id=1)]), -1),
            # id != osd_id
            (1, dict(nodes=[dict(id=999, pgs=1)]), -1),
            # valid
            (1, dict(nodes=[dict(id=1, pgs=1)]), 1),
        ]
    )
    def test_get_pg_count(self, rm_util, osd_id, osd_df, expected):
        with mock.patch("cephadm.services.osd.RemoveUtil.osd_df", return_value=osd_df):
            assert rm_util.get_pg_count(osd_id) == expected

    @pytest.mark.parametrize(
        "osds, ok_to_stop, expected",
        [
            # no osd_ids provided
            ([], [False], []),
            # all osds are ok_to_stop
            ([1, 2], [True], [1, 2]),
            # osds are ok_to_stop after the second iteration
            ([1, 2], [False, True], [2]),
            # osds are never ok_to_stop, (taking the sample size `(len(osd_ids))` into account),
            # expected to get False
            ([1, 2], [False, False], []),
        ]
    )
    def test_find_stop_threshold(self, rm_util, osds, ok_to_stop, expected):
        with mock.patch("cephadm.services.osd.RemoveUtil.ok_to_stop", side_effect=ok_to_stop):
            assert rm_util.find_osd_stop_threshold(osds) == expected

    def test_process_removal_queue(self, rm_util):
        # TODO: !
        # rm_util.process_removal_queue()
        pass

    @pytest.mark.parametrize(
        "max_osd_draining_count, draining_osds, idling_osds, ok_to_stop, expected",
        [
            # drain one at a time, one already draining
            (1, [1], [1], [True], 0),
            # drain one at a time, none draining yet
            (1, [], [1, 2, 3], [True, True, True], 1),
            # drain one at a time, one already draining, none ok-to-stop
            (1, [1], [1], [False], 0),
            # drain one at a time, none draining, one ok-to-stop
            (1, [], [1, 2, 3], [False, False, True], 1),
            # drain three at a time, one already draining, all ok-to-stop
            (3, [1], [1, 2, 3], [True, True, True], 2),
            # drain two at a time, none already draining, none ok-to-stop
            (2, [], [1, 2, 3], [False, False, False], 0),
            # drain two at a time, none already draining, none idling
            (2, [], [], [], 0),
        ]
    )
    def test_ready_to_drain_osds(self, max_osd_draining_count, draining_osds, idling_osds, ok_to_stop, expected):
        with with_cephadm_module({'max_osd_draining_count': max_osd_draining_count}) as m:
            with mock.patch("cephadm.services.osd.OSDRemovalQueue.draining_osds", return_value=draining_osds):
                with mock.patch("cephadm.services.osd.OSDRemovalQueue.idling_osds", return_value=idling_osds):
                    with mock.patch("cephadm.services.osd.RemoveUtil.ok_to_stop", side_effect=ok_to_stop):
                        removal_queue = OSDRemovalQueue(m)
                        assert len(removal_queue._ready_to_drain_osds()) == expected

    def test_ok_to_stop(self, rm_util):
        rm_util.ok_to_stop([MockOSD(1)])
        rm_util._run_mon_cmd.assert_called_with({'prefix': 'osd ok-to-stop', 'ids': ['1']},
                                                error_ok=True)

    def test_safe_to_destroy(self, rm_util):
        rm_util.safe_to_destroy([1])
        rm_util._run_mon_cmd.assert_called_with({'prefix': 'osd safe-to-destroy',
                                                 'ids': ['1']}, error_ok=True)

    def test_destroy_osd(self, rm_util):
        rm_util.destroy_osd(1)
        rm_util._run_mon_cmd.assert_called_with(
            {'prefix': 'osd destroy-actual', 'id': 1, 'yes_i_really_mean_it': True})

    def test_purge_osd(self, rm_util):
        rm_util.purge_osd(1)
        rm_util._run_mon_cmd.assert_called_with(
            {'prefix': 'osd purge-actual', 'id': 1, 'yes_i_really_mean_it': True})

    def test_load(self, cephadm_module, rm_util):
        data = json.dumps([
            {
                "osd_id": 35,
                "started": True,
                "draining": True,
                "stopped": False,
                "replace": False,
                "force": False,
                "zap": False,
                "nodename": "node2",
                "drain_started_at": "2020-09-14T11:41:53.960463",
                "drain_stopped_at": None,
                "drain_done_at": None,
                "process_started_at": "2020-09-14T11:41:52.245832"
            }
        ])
        cephadm_module.set_store('osd_remove_queue', data)
        cephadm_module.to_remove_osds.load_from_store()

        expected = OSDRemovalQueue(cephadm_module)
        expected.osds.add(OSD(osd_id=35, remove_util=rm_util, draining=True))
        assert cephadm_module.to_remove_osds == expected


class TestOSD:

    def test_start(self, osd_obj):
        assert osd_obj.started is False
        osd_obj.start()
        assert osd_obj.started is True
        assert osd_obj.stopped is False

    def test_start_draining_purge(self, osd_obj):
        assert osd_obj.draining is False
        assert osd_obj.drain_started_at is None
        ret = osd_obj.start_draining()
        osd_obj.rm_util.reweight_osd.assert_called_with(osd_obj, 0.0)
        assert isinstance(osd_obj.drain_started_at, datetime)
        assert osd_obj.draining is True
        assert osd_obj.replace is False
        assert ret is True

    def test_start_draining_replace(self, osd_obj):
        assert osd_obj.draining is False
        assert osd_obj.drain_started_at is None
        osd_obj.replace = True
        ret = osd_obj.start_draining()
        osd_obj.rm_util.set_osd_flag.assert_called_with([osd_obj], 'out')
        assert isinstance(osd_obj.drain_started_at, datetime)
        assert osd_obj.draining is True
        assert osd_obj.replace is True
        assert ret is True

    def test_start_draining_stopped(self, osd_obj):
        osd_obj.stopped = True
        ret = osd_obj.start_draining()
        assert osd_obj.drain_started_at is None
        assert ret is False
        assert osd_obj.draining is False

    def test_stop_draining_replace(self, osd_obj):
        osd_obj.replace = True
        ret = osd_obj.stop_draining()
        osd_obj.rm_util.set_osd_flag.assert_called_with([osd_obj], 'in')
        assert isinstance(osd_obj.drain_stopped_at, datetime)
        assert osd_obj.draining is False
        assert ret is True

    def test_stop_draining_purge(self, osd_obj):
        osd_obj.original_weight = 1.0
        ret = osd_obj.stop_draining()
        osd_obj.rm_util.reweight_osd.assert_called_with(osd_obj, 1.0)
        assert isinstance(osd_obj.drain_stopped_at, datetime)
        assert osd_obj.draining is False
        assert ret is True

    @mock.patch('cephadm.services.osd.OSD.stop_draining')
    def test_stop(self, stop_draining_mock, osd_obj):
        osd_obj.stop()
        assert osd_obj.started is False
        assert osd_obj.stopped is True
        stop_draining_mock.assert_called_once()

    @pytest.mark.parametrize(
        "draining, empty, expected",
        [
            # must be !draining! and !not empty! to yield True
            (True, not True, True),
            # not draining and not empty
            (False, not True, False),
            # not draining and empty
            (False, True, False),
            # draining and empty
            (True, True, False),
        ]
    )
    def test_is_draining(self, osd_obj, draining, empty, expected):
        with mock.patch("cephadm.services.osd.OSD.is_empty", new_callable=mock.PropertyMock(return_value=empty)):
            osd_obj.draining = draining
            assert osd_obj.is_draining is expected

    @mock.patch("cephadm.services.osd.RemoveUtil.ok_to_stop")
    def test_is_ok_to_stop(self, _, osd_obj):
        osd_obj.is_ok_to_stop
        osd_obj.rm_util.ok_to_stop.assert_called_once()

    @pytest.mark.parametrize(
        "pg_count, expected",
        [
            (0, True),
            (1, False),
            (9999, False),
            (-1, False),
        ]
    )
    def test_is_empty(self, osd_obj, pg_count, expected):
        with mock.patch("cephadm.services.osd.OSD.get_pg_count", return_value=pg_count):
            assert osd_obj.is_empty is expected

    @mock.patch("cephadm.services.osd.RemoveUtil.safe_to_destroy")
    def test_safe_to_destroy(self, _, osd_obj):
        osd_obj.safe_to_destroy()
        osd_obj.rm_util.safe_to_destroy.assert_called_once()

    @mock.patch("cephadm.services.osd.RemoveUtil.set_osd_flag")
    def test_down(self, _, osd_obj):
        osd_obj.down()
        osd_obj.rm_util.set_osd_flag.assert_called_with([osd_obj], 'down')

    @mock.patch("cephadm.services.osd.RemoveUtil.destroy_osd")
    def test_destroy_osd(self, _, osd_obj):
        osd_obj.destroy()
        osd_obj.rm_util.destroy_osd.assert_called_once()

    @mock.patch("cephadm.services.osd.RemoveUtil.purge_osd")
    def test_purge(self, _, osd_obj):
        osd_obj.purge()
        osd_obj.rm_util.purge_osd.assert_called_once()

    @mock.patch("cephadm.services.osd.RemoveUtil.get_pg_count")
    def test_pg_count(self, _, osd_obj):
        osd_obj.get_pg_count()
        osd_obj.rm_util.get_pg_count.assert_called_once()

    def test_drain_status_human_not_started(self, osd_obj):
        assert osd_obj.drain_status_human() == 'not started'

    def test_drain_status_human_started(self, osd_obj):
        osd_obj.started = True
        assert osd_obj.drain_status_human() == 'started'

    def test_drain_status_human_draining(self, osd_obj):
        osd_obj.started = True
        osd_obj.draining = True
        assert osd_obj.drain_status_human() == 'draining'

    def test_drain_status_human_done(self, osd_obj):
        osd_obj.started = True
        osd_obj.draining = False
        osd_obj.drain_done_at = datetime.utcnow()
        assert osd_obj.drain_status_human() == 'done, waiting for purge'


class TestOSDRemovalQueue:

    def test_queue_size(self, osd_obj):
        q = OSDRemovalQueue(mock.Mock())
        assert q.queue_size() == 0
        q.osds.add(osd_obj)
        assert q.queue_size() == 1

    @mock.patch("cephadm.services.osd.OSD.start")
    @mock.patch("cephadm.services.osd.OSD.exists")
    def test_enqueue(self, exist, start, osd_obj):
        q = OSDRemovalQueue(mock.Mock())
        q.enqueue(osd_obj)
        osd_obj.start.assert_called_once()

    @mock.patch("cephadm.services.osd.OSD.stop")
    @mock.patch("cephadm.services.osd.OSD.exists")
    def test_rm_raise(self, exist, stop, osd_obj):
        q = OSDRemovalQueue(mock.Mock())
        with pytest.raises(KeyError):
            q.rm(osd_obj)
            osd_obj.stop.assert_called_once()

    @mock.patch("cephadm.services.osd.OSD.stop")
    @mock.patch("cephadm.services.osd.OSD.exists")
    def test_rm(self, exist, stop, osd_obj):
        q = OSDRemovalQueue(mock.Mock())
        q.osds.add(osd_obj)
        q.rm(osd_obj)
        osd_obj.stop.assert_called_once()
Commit	Line	Data
f91f0fd5 TL	1	import json
f91f0fd5 TL	2
f67539c2	3	from cephadm.services.osd import OSDRemovalQueue, OSD
f6b5b4d7	4	import pytest
f6b5b4d7	5	from tests import mock
a4b75251	6	from .fixtures import with_cephadm_module
f6b5b4d7 TL	7	from datetime import datetime
	8
	9
	10	class MockOSD:
	11
	12	def __init__(self, osd_id):
	13	self.osd_id = osd_id
	14
f91f0fd5	15
f6b5b4d7 TL	16	class TestOSDRemoval:
	17
	18	@pytest.mark.parametrize(
	19	"osd_id, osd_df, expected",
	20	[
	21	# missing 'nodes' key
	22	(1, dict(nodes=[]), -1),
	23	# missing 'pgs' key
	24	(1, dict(nodes=[dict(id=1)]), -1),
	25	# id != osd_id
	26	(1, dict(nodes=[dict(id=999, pgs=1)]), -1),
	27	# valid
	28	(1, dict(nodes=[dict(id=1, pgs=1)]), 1),
	29	]
	30	)
	31	def test_get_pg_count(self, rm_util, osd_id, osd_df, expected):
	32	with mock.patch("cephadm.services.osd.RemoveUtil.osd_df", return_value=osd_df):
	33	assert rm_util.get_pg_count(osd_id) == expected
	34
	35	@pytest.mark.parametrize(
	36	"osds, ok_to_stop, expected",
	37	[
	38	# no osd_ids provided
	39	([], [False], []),
	40	# all osds are ok_to_stop
	41	([1, 2], [True], [1, 2]),
	42	# osds are ok_to_stop after the second iteration
	43	([1, 2], [False, True], [2]),
	44	# osds are never ok_to_stop, (taking the sample size `(len(osd_ids))` into account),
	45	# expected to get False
	46	([1, 2], [False, False], []),
	47	]
	48	)
	49	def test_find_stop_threshold(self, rm_util, osds, ok_to_stop, expected):
	50	with mock.patch("cephadm.services.osd.RemoveUtil.ok_to_stop", side_effect=ok_to_stop):
	51	assert rm_util.find_osd_stop_threshold(osds) == expected
	52
	53	def test_process_removal_queue(self, rm_util):
	54	# TODO: !
	55	# rm_util.process_removal_queue()
	56	pass
	57
a4b75251 TL	58	@pytest.mark.parametrize(
	59	"max_osd_draining_count, draining_osds, idling_osds, ok_to_stop, expected",
	60	[
	61	# drain one at a time, one already draining
	62	(1, [1], [1], [True], 0),
	63	# drain one at a time, none draining yet
	64	(1, [], [1, 2, 3], [True, True, True], 1),
	65	# drain one at a time, one already draining, none ok-to-stop
	66	(1, [1], [1], [False], 0),
	67	# drain one at a time, none draining, one ok-to-stop
	68	(1, [], [1, 2, 3], [False, False, True], 1),
	69	# drain three at a time, one already draining, all ok-to-stop
	70	(3, [1], [1, 2, 3], [True, True, True], 2),
	71	# drain two at a time, none already draining, none ok-to-stop
	72	(2, [], [1, 2, 3], [False, False, False], 0),
	73	# drain two at a time, none already draining, none idling
	74	(2, [], [], [], 0),
	75	]
	76	)
	77	def test_ready_to_drain_osds(self, max_osd_draining_count, draining_osds, idling_osds, ok_to_stop, expected):
	78	with with_cephadm_module({'max_osd_draining_count': max_osd_draining_count}) as m:
	79	with mock.patch("cephadm.services.osd.OSDRemovalQueue.draining_osds", return_value=draining_osds):
	80	with mock.patch("cephadm.services.osd.OSDRemovalQueue.idling_osds", return_value=idling_osds):
	81	with mock.patch("cephadm.services.osd.RemoveUtil.ok_to_stop", side_effect=ok_to_stop):
	82	removal_queue = OSDRemovalQueue(m)
	83	assert len(removal_queue._ready_to_drain_osds()) == expected
	84
f6b5b4d7 TL	85	def test_ok_to_stop(self, rm_util):
f6b5b4d7 TL	86	rm_util.ok_to_stop([MockOSD(1)])
a4b75251 TL	87	rm_util._run_mon_cmd.assert_called_with({'prefix': 'osd ok-to-stop', 'ids': ['1']},
a4b75251 TL	88	error_ok=True)
f6b5b4d7 TL	89
	90	def test_safe_to_destroy(self, rm_util):
	91	rm_util.safe_to_destroy([1])
a4b75251 TL	92	rm_util._run_mon_cmd.assert_called_with({'prefix': 'osd safe-to-destroy',
a4b75251 TL	93	'ids': ['1']}, error_ok=True)
f6b5b4d7 TL	94
	95	def test_destroy_osd(self, rm_util):
	96	rm_util.destroy_osd(1)
f91f0fd5 TL	97	rm_util._run_mon_cmd.assert_called_with(
f91f0fd5 TL	98	{'prefix': 'osd destroy-actual', 'id': 1, 'yes_i_really_mean_it': True})
f6b5b4d7 TL	99
	100	def test_purge_osd(self, rm_util):
	101	rm_util.purge_osd(1)
f91f0fd5 TL	102	rm_util._run_mon_cmd.assert_called_with(
	103	{'prefix': 'osd purge-actual', 'id': 1, 'yes_i_really_mean_it': True})
	104
adb31ebb	105	def test_load(self, cephadm_module, rm_util):
f91f0fd5 TL	106	data = json.dumps([
	107	{
	108	"osd_id": 35,
	109	"started": True,
	110	"draining": True,
	111	"stopped": False,
	112	"replace": False,
	113	"force": False,
a4b75251	114	"zap": False,
f91f0fd5 TL	115	"nodename": "node2",
	116	"drain_started_at": "2020-09-14T11:41:53.960463",
	117	"drain_stopped_at": None,
	118	"drain_done_at": None,
	119	"process_started_at": "2020-09-14T11:41:52.245832"
	120	}
	121	])
	122	cephadm_module.set_store('osd_remove_queue', data)
adb31ebb	123	cephadm_module.to_remove_osds.load_from_store()
f91f0fd5	124
adb31ebb TL	125	expected = OSDRemovalQueue(cephadm_module)
	126	expected.osds.add(OSD(osd_id=35, remove_util=rm_util, draining=True))
	127	assert cephadm_module.to_remove_osds == expected
f6b5b4d7 TL	128
	129
	130	class TestOSD:
	131
	132	def test_start(self, osd_obj):
	133	assert osd_obj.started is False
	134	osd_obj.start()
	135	assert osd_obj.started is True
	136	assert osd_obj.stopped is False
	137
f67539c2	138	def test_start_draining_purge(self, osd_obj):
f6b5b4d7 TL	139	assert osd_obj.draining is False
	140	assert osd_obj.drain_started_at is None
	141	ret = osd_obj.start_draining()
f67539c2 TL	142	osd_obj.rm_util.reweight_osd.assert_called_with(osd_obj, 0.0)
	143	assert isinstance(osd_obj.drain_started_at, datetime)
	144	assert osd_obj.draining is True
	145	assert osd_obj.replace is False
	146	assert ret is True
	147
	148	def test_start_draining_replace(self, osd_obj):
	149	assert osd_obj.draining is False
	150	assert osd_obj.drain_started_at is None
	151	osd_obj.replace = True
	152	ret = osd_obj.start_draining()
f6b5b4d7 TL	153	osd_obj.rm_util.set_osd_flag.assert_called_with([osd_obj], 'out')
	154	assert isinstance(osd_obj.drain_started_at, datetime)
	155	assert osd_obj.draining is True
f67539c2	156	assert osd_obj.replace is True
f6b5b4d7 TL	157	assert ret is True
	158
	159	def test_start_draining_stopped(self, osd_obj):
	160	osd_obj.stopped = True
	161	ret = osd_obj.start_draining()
	162	assert osd_obj.drain_started_at is None
	163	assert ret is False
	164	assert osd_obj.draining is False
	165
f67539c2 TL	166	def test_stop_draining_replace(self, osd_obj):
f67539c2 TL	167	osd_obj.replace = True
f6b5b4d7 TL	168	ret = osd_obj.stop_draining()
	169	osd_obj.rm_util.set_osd_flag.assert_called_with([osd_obj], 'in')
	170	assert isinstance(osd_obj.drain_stopped_at, datetime)
	171	assert osd_obj.draining is False
	172	assert ret is True
	173
f67539c2 TL	174	def test_stop_draining_purge(self, osd_obj):
	175	osd_obj.original_weight = 1.0
	176	ret = osd_obj.stop_draining()
	177	osd_obj.rm_util.reweight_osd.assert_called_with(osd_obj, 1.0)
	178	assert isinstance(osd_obj.drain_stopped_at, datetime)
	179	assert osd_obj.draining is False
	180	assert ret is True
	181
f6b5b4d7 TL	182	@mock.patch('cephadm.services.osd.OSD.stop_draining')
f6b5b4d7 TL	183	def test_stop(self, stop_draining_mock, osd_obj):
f67539c2	184	osd_obj.stop()
f6b5b4d7 TL	185	assert osd_obj.started is False
	186	assert osd_obj.stopped is True
	187	stop_draining_mock.assert_called_once()
	188
	189	@pytest.mark.parametrize(
	190	"draining, empty, expected",
	191	[
	192	# must be !draining! and !not empty! to yield True
	193	(True, not True, True),
	194	# not draining and not empty
	195	(False, not True, False),
	196	# not draining and empty
	197	(False, True, False),
	198	# draining and empty
	199	(True, True, False),
	200	]
	201	)
	202	def test_is_draining(self, osd_obj, draining, empty, expected):
	203	with mock.patch("cephadm.services.osd.OSD.is_empty", new_callable=mock.PropertyMock(return_value=empty)):
	204	osd_obj.draining = draining
	205	assert osd_obj.is_draining is expected
	206
	207	@mock.patch("cephadm.services.osd.RemoveUtil.ok_to_stop")
	208	def test_is_ok_to_stop(self, _, osd_obj):
f67539c2	209	osd_obj.is_ok_to_stop
f6b5b4d7 TL	210	osd_obj.rm_util.ok_to_stop.assert_called_once()
	211
	212	@pytest.mark.parametrize(
	213	"pg_count, expected",
	214	[
	215	(0, True),
	216	(1, False),
	217	(9999, False),
	218	(-1, False),
	219	]
	220	)
	221	def test_is_empty(self, osd_obj, pg_count, expected):
	222	with mock.patch("cephadm.services.osd.OSD.get_pg_count", return_value=pg_count):
	223	assert osd_obj.is_empty is expected
	224
	225	@mock.patch("cephadm.services.osd.RemoveUtil.safe_to_destroy")
	226	def test_safe_to_destroy(self, _, osd_obj):
f67539c2	227	osd_obj.safe_to_destroy()
f6b5b4d7 TL	228	osd_obj.rm_util.safe_to_destroy.assert_called_once()
	229
	230	@mock.patch("cephadm.services.osd.RemoveUtil.set_osd_flag")
	231	def test_down(self, _, osd_obj):
f67539c2	232	osd_obj.down()
f6b5b4d7 TL	233	osd_obj.rm_util.set_osd_flag.assert_called_with([osd_obj], 'down')
	234
	235	@mock.patch("cephadm.services.osd.RemoveUtil.destroy_osd")
	236	def test_destroy_osd(self, _, osd_obj):
f67539c2	237	osd_obj.destroy()
f6b5b4d7 TL	238	osd_obj.rm_util.destroy_osd.assert_called_once()
	239
	240	@mock.patch("cephadm.services.osd.RemoveUtil.purge_osd")
	241	def test_purge(self, _, osd_obj):
f67539c2	242	osd_obj.purge()
f6b5b4d7 TL	243	osd_obj.rm_util.purge_osd.assert_called_once()
	244
	245	@mock.patch("cephadm.services.osd.RemoveUtil.get_pg_count")
	246	def test_pg_count(self, _, osd_obj):
f67539c2	247	osd_obj.get_pg_count()
f6b5b4d7 TL	248	osd_obj.rm_util.get_pg_count.assert_called_once()
	249
	250	def test_drain_status_human_not_started(self, osd_obj):
	251	assert osd_obj.drain_status_human() == 'not started'
	252
	253	def test_drain_status_human_started(self, osd_obj):
	254	osd_obj.started = True
	255	assert osd_obj.drain_status_human() == 'started'
	256
	257	def test_drain_status_human_draining(self, osd_obj):
	258	osd_obj.started = True
	259	osd_obj.draining = True
	260	assert osd_obj.drain_status_human() == 'draining'
	261
	262	def test_drain_status_human_done(self, osd_obj):
	263	osd_obj.started = True
	264	osd_obj.draining = False
	265	osd_obj.drain_done_at = datetime.utcnow()
	266	assert osd_obj.drain_status_human() == 'done, waiting for purge'
	267
	268
adb31ebb	269	class TestOSDRemovalQueue:
f6b5b4d7 TL	270
f6b5b4d7 TL	271	def test_queue_size(self, osd_obj):
adb31ebb	272	q = OSDRemovalQueue(mock.Mock())
f6b5b4d7	273	assert q.queue_size() == 0
adb31ebb	274	q.osds.add(osd_obj)
f6b5b4d7 TL	275	assert q.queue_size() == 1
	276
	277	@mock.patch("cephadm.services.osd.OSD.start")
	278	@mock.patch("cephadm.services.osd.OSD.exists")
	279	def test_enqueue(self, exist, start, osd_obj):
adb31ebb	280	q = OSDRemovalQueue(mock.Mock())
f6b5b4d7 TL	281	q.enqueue(osd_obj)
	282	osd_obj.start.assert_called_once()
	283
	284	@mock.patch("cephadm.services.osd.OSD.stop")
	285	@mock.patch("cephadm.services.osd.OSD.exists")
	286	def test_rm_raise(self, exist, stop, osd_obj):
adb31ebb	287	q = OSDRemovalQueue(mock.Mock())
f6b5b4d7 TL	288	with pytest.raises(KeyError):
	289	q.rm(osd_obj)
	290	osd_obj.stop.assert_called_once()
	291
	292	@mock.patch("cephadm.services.osd.OSD.stop")
	293	@mock.patch("cephadm.services.osd.OSD.exists")
	294	def test_rm(self, exist, stop, osd_obj):
adb31ebb TL	295	q = OSDRemovalQueue(mock.Mock())
adb31ebb TL	296	q.osds.add(osd_obj)
f6b5b4d7 TL	297	q.rm(osd_obj)
f6b5b4d7 TL	298	osd_obj.stop.assert_called_once()