1 # Disable autopep8 for this file:
5 from typing
import NamedTuple
, List
8 from ceph
.deployment
.hostspec
import HostSpec
9 from ceph
.deployment
.service_spec
import ServiceSpec
, PlacementSpec
, ServiceSpecValidationError
11 from cephadm
.module
import HostAssignment
12 from orchestrator
import DaemonDescription
, OrchestratorValidationError
, OrchestratorError
, HostSpec
16 # some odd thingy to revert the order or arguments
30 def one_of(expected
, *hosts
):
31 if not isinstance(expected
, list):
32 assert False, str(expected
)
33 assert len(expected
) == 1, f
'one_of failed len({expected}) != 1'
34 assert expected
[0] in hosts
38 def two_of(expected
, *hosts
):
39 if not isinstance(expected
, list):
40 assert False, str(expected
)
41 assert len(expected
) == 2, f
'one_of failed len({expected}) != 2'
44 matches
+= int(h
in expected
)
46 assert False, f
'two of {hosts} not in {expected}'
50 def exactly(expected
, *hosts
):
51 assert expected
== list(hosts
)
55 def error(expected
, kind
, match
):
56 assert isinstance(expected
, kind
), (str(expected
), match
)
57 assert str(expected
) == match
, (str(expected
), match
)
61 def _or(expected
, *inners
):
65 except AssertionError as e
:
67 result
= [catch(i
) for i
in inners
]
68 if None not in result
:
69 assert False, f
"_or failed: {expected}"
72 def _always_true(_
): pass
76 return [e
for e
in s
.split(' ') if e
]
79 def get_result(key
, results
):
81 for o
, k
in zip(one
, key
):
82 if o
!= k
and o
!= '*':
85 return [v
for k
, v
in results
89 def mk_spec_and_host(spec_section
, hosts
, explicit_key
, explicit
, count
):
91 if spec_section
== 'hosts':
92 mk_spec
= lambda: ServiceSpec('mgr', placement
=PlacementSpec(
96 elif spec_section
== 'label':
97 mk_spec
= lambda: ServiceSpec('mgr', placement
=PlacementSpec(
101 elif spec_section
== 'host_pattern':
108 mk_spec
= lambda: ServiceSpec('mgr', placement
=PlacementSpec(
109 host_pattern
=pattern
,
116 HostSpec(h
, labels
=['mylabel']) if h
in explicit
else HostSpec(h
)
120 return mk_spec
, hosts
123 def run_scheduler_test(results
, mk_spec
, hosts
, get_daemons_func
, key_elems
):
124 key
= ' '.join('N' if e
is None else str(e
) for e
in key_elems
)
126 assert_res
= get_result(k(key
), results
)
130 host_res
= HostAssignment(
133 get_daemons_func
=get_daemons_func
).place()
134 if isinstance(host_res
, list):
135 e
= ', '.join(repr(h
.hostname
) for h
in host_res
)
136 assert False, f
'`(k("{key}"), exactly({e})),` not found'
137 assert False, f
'`(k("{key}"), ...),` not found'
138 except OrchestratorError
as e
:
139 assert False, f
'`(k("{key}"), error({type(e).__name__}, {repr(str(e))})),` not found'
141 for _
in range(10): # scheduler has a random component
144 host_res
= HostAssignment(
147 get_daemons_func
=get_daemons_func
).place()
149 assert_res(sorted([h
.hostname
for h
in host_res
]))
150 except Exception as e
:
154 # * first match from the top wins
155 # * where e=[], *=any
157 # + list of known hosts available for scheduling (host_key)
158 # | + hosts used for explict placement (explicit_key)
160 # | | | + section (host, label, pattern)
161 # | | | | + expected result
163 test_explicit_scheduler_results
= [
164 (k("* * 0 *"), error(ServiceSpecValidationError
, 'num/count must be > 1')),
165 (k("* e N l"), error(OrchestratorValidationError
, 'Cannot place <ServiceSpec for service_name=mgr>: No matching hosts for label mylabel')),
166 (k("* e N p"), error(OrchestratorValidationError
, 'Cannot place <ServiceSpec for service_name=mgr>: No matching hosts')),
167 (k("* e N h"), error(OrchestratorValidationError
, 'placement spec is empty: no hosts, no label, no pattern, no count')),
168 (k("* e * *"), none
),
169 (k("1 12 * h"), error(OrchestratorValidationError
, "Cannot place <ServiceSpec for service_name=mgr> on 2: Unknown hosts")),
170 (k("1 123 * h"), error(OrchestratorValidationError
, "Cannot place <ServiceSpec for service_name=mgr> on 2, 3: Unknown hosts")),
171 (k("1 * * *"), exactly('1')),
172 (k("12 1 * *"), exactly('1')),
173 (k("12 12 1 *"), one_of('1', '2')),
174 (k("12 12 * *"), exactly('1', '2')),
175 (k("12 123 * h"), error(OrchestratorValidationError
, "Cannot place <ServiceSpec for service_name=mgr> on 3: Unknown hosts")),
176 (k("12 123 1 *"), one_of('1', '2', '3')),
177 (k("12 123 * *"), two_of('1', '2', '3')),
178 (k("123 1 * *"), exactly('1')),
179 (k("123 12 1 *"), one_of('1', '2')),
180 (k("123 12 * *"), exactly('1', '2')),
181 (k("123 123 1 *"), one_of('1', '2', '3')),
182 (k("123 123 2 *"), two_of('1', '2', '3')),
183 (k("123 123 * *"), exactly('1', '2', '3')),
186 @pytest.mark
.parametrize("spec_section_key,spec_section",
190 ('p', 'host_pattern'),
192 @pytest.mark
.parametrize("count",
200 @pytest.mark
.parametrize("explicit_key, explicit",
205 ('123', ['1', '2', '3']),
207 @pytest.mark
.parametrize("host_key, hosts",
211 ('123', ['1', '2', '3']),
213 def test_explicit_scheduler(host_key
, hosts
,
214 explicit_key
, explicit
,
216 spec_section_key
, spec_section
):
218 mk_spec
, hosts
= mk_spec_and_host(spec_section
, hosts
, explicit_key
, explicit
, count
)
220 results
=test_explicit_scheduler_results
,
223 get_daemons_func
=lambda _
: [],
224 key_elems
=(host_key
, explicit_key
, count
, spec_section_key
)
228 # * first match from the top wins
229 # * where e=[], *=any
231 # + list of known hosts available for scheduling (host_key)
232 # | + hosts used for explict placement (explicit_key)
234 # | | | + existing daemons
235 # | | | | + section (host, label, pattern)
236 # | | | | | + expected result
238 test_scheduler_daemons_results
= [
239 (k("* 1 * * *"), exactly('1')),
240 (k("1 123 * * h"), error(OrchestratorValidationError
, 'Cannot place <ServiceSpec for service_name=mgr> on 2, 3: Unknown hosts')),
241 (k("1 123 * * *"), exactly('1')),
242 (k("12 123 * * h"), error(OrchestratorValidationError
, 'Cannot place <ServiceSpec for service_name=mgr> on 3: Unknown hosts')),
243 (k("12 123 N * *"), exactly('1', '2')),
244 (k("12 123 1 * *"), one_of('1', '2')),
245 (k("12 123 2 * *"), exactly('1', '2')),
246 (k("12 123 3 * *"), exactly('1', '2')),
247 (k("123 123 N * *"), exactly('1', '2', '3')),
248 (k("123 123 1 e *"), one_of('1', '2', '3')),
249 (k("123 123 1 1 *"), exactly('1')),
250 (k("123 123 1 3 *"), exactly('3')),
251 (k("123 123 1 12 *"), one_of('1', '2')),
252 (k("123 123 1 112 *"), one_of('1', '2')),
253 (k("123 123 1 23 *"), one_of('2', '3')),
254 (k("123 123 1 123 *"), one_of('1', '2', '3')),
255 (k("123 123 2 e *"), two_of('1', '2', '3')),
256 (k("123 123 2 1 *"), _or(exactly('1', '2'), exactly('1', '3'))),
257 (k("123 123 2 3 *"), _or(exactly('1', '3'), exactly('2', '3'))),
258 (k("123 123 2 12 *"), exactly('1', '2')),
259 (k("123 123 2 112 *"), exactly('1', '2')),
260 (k("123 123 2 23 *"), exactly('2', '3')),
261 (k("123 123 2 123 *"), two_of('1', '2', '3')),
262 (k("123 123 3 * *"), exactly('1', '2', '3')),
266 @pytest.mark
.parametrize("spec_section_key,spec_section",
270 ('p', 'host_pattern'),
272 @pytest.mark
.parametrize("daemons_key, daemons",
278 ('112', ['1', '1', '2']), # deal with existing co-located daemons
280 ('123', ['1', '2', '3']),
282 @pytest.mark
.parametrize("count",
289 @pytest.mark
.parametrize("explicit_key, explicit",
292 ('123', ['1', '2', '3']),
294 @pytest.mark
.parametrize("host_key, hosts",
298 ('123', ['1', '2', '3']),
300 def test_scheduler_daemons(host_key
, hosts
,
301 explicit_key
, explicit
,
303 daemons_key
, daemons
,
304 spec_section_key
, spec_section
):
305 mk_spec
, hosts
= mk_spec_and_host(spec_section
, hosts
, explicit_key
, explicit
, count
)
307 DaemonDescription('mgr', d
, d
)
311 results
=test_scheduler_daemons_results
,
314 get_daemons_func
=lambda _
: dds
,
315 key_elems
=(host_key
, explicit_key
, count
, daemons_key
, spec_section_key
)
319 # =========================
322 class NodeAssignmentTest(NamedTuple
):
324 placement
: PlacementSpec
326 daemons
: List
[DaemonDescription
]
329 @pytest.mark
.parametrize("service_type,placement,hosts,daemons,expected",
334 PlacementSpec(hosts
=['smithi060:[v2:172.21.15.60:3301,v1:172.21.15.60:6790]=c']),
342 PlacementSpec(host_pattern
='*'),
343 'host1 host2 host3'.split(),
345 DaemonDescription('mgr', 'a', 'host1'),
346 DaemonDescription('mgr', 'b', 'host2'),
348 ['host1', 'host2', 'host3']
350 # count that is bigger than the amount of hosts. Truncate to len(hosts)
351 # RGWs should not be co-located to each other.
354 PlacementSpec(count
=4),
355 'host1 host2 host3'.split(),
357 ['host1', 'host2', 'host3']
359 # count + partial host list
362 PlacementSpec(count
=3, hosts
=['host3']),
363 'host1 host2 host3'.split(),
365 DaemonDescription('mgr', 'a', 'host1'),
366 DaemonDescription('mgr', 'b', 'host2'),
370 # count 1 + partial host list
373 PlacementSpec(count
=1, hosts
=['host3']),
374 'host1 host2 host3'.split(),
376 DaemonDescription('mgr', 'a', 'host1'),
377 DaemonDescription('mgr', 'b', 'host2'),
381 # count + partial host list + existing
384 PlacementSpec(count
=2, hosts
=['host3']),
385 'host1 host2 host3'.split(),
387 DaemonDescription('mgr', 'a', 'host1'),
391 # count + partial host list + existing (deterministic)
394 PlacementSpec(count
=2, hosts
=['host1']),
395 'host1 host2'.split(),
397 DaemonDescription('mgr', 'a', 'host1'),
401 # count + partial host list + existing (deterministic)
404 PlacementSpec(count
=2, hosts
=['host1']),
405 'host1 host2'.split(),
407 DaemonDescription('mgr', 'a', 'host2'),
414 PlacementSpec(label
='foo'),
415 'host1 host2 host3'.split(),
417 ['host1', 'host2', 'host3']
422 PlacementSpec(host_pattern
='mgr*'),
423 'mgrhost1 mgrhost2 datahost'.split(),
425 ['mgrhost1', 'mgrhost2']
428 def test_node_assignment(service_type
, placement
, hosts
, daemons
, expected
):
430 if service_type
== 'rgw':
431 service_id
= 'realm.zone'
433 spec
= ServiceSpec(service_type
=service_type
,
434 service_id
=service_id
,
437 hosts
= HostAssignment(
439 hosts
=[HostSpec(h
, labels
=['foo']) for h
in hosts
],
440 get_daemons_func
=lambda _
: daemons
).place()
441 assert sorted([h
.hostname
for h
in hosts
]) == sorted(expected
)
444 class NodeAssignmentTest2(NamedTuple
):
446 placement
: PlacementSpec
448 daemons
: List
[DaemonDescription
]
452 @pytest.mark
.parametrize("service_type,placement,hosts,daemons,expected_len,in_set",
457 PlacementSpec(count
=1),
458 'host1 host2 host3'.split(),
461 ['host1', 'host2', 'host3'],
464 # hosts + (smaller) count
467 PlacementSpec(count
=1, hosts
='host1 host2'.split()),
468 'host1 host2'.split(),
473 # hosts + (smaller) count, existing
476 PlacementSpec(count
=1, hosts
='host1 host2 host3'.split()),
477 'host1 host2 host3'.split(),
478 [DaemonDescription('mgr', 'mgr.a', 'host1'),],
480 ['host1', 'host2', 'host3'],
482 # hosts + (smaller) count, (more) existing
485 PlacementSpec(count
=1, hosts
='host1 host2 host3'.split()),
486 'host1 host2 host3'.split(),
488 DaemonDescription('mgr', 'a', 'host1'),
489 DaemonDescription('mgr', 'b', 'host2'),
494 # count + partial host list
497 PlacementSpec(count
=2, hosts
=['host3']),
498 'host1 host2 host3'.split(),
501 ['host1', 'host2', 'host3']
506 PlacementSpec(count
=1, label
='foo'),
507 'host1 host2 host3'.split(),
510 ['host1', 'host2', 'host3']
513 def test_node_assignment2(service_type
, placement
, hosts
,
514 daemons
, expected_len
, in_set
):
515 hosts
= HostAssignment(
516 spec
=ServiceSpec(service_type
, placement
=placement
),
517 hosts
=[HostSpec(h
, labels
=['foo']) for h
in hosts
],
518 get_daemons_func
=lambda _
: daemons
).place()
519 assert len(hosts
) == expected_len
520 for h
in [h
.hostname
for h
in hosts
]:
523 @pytest.mark
.parametrize("service_type,placement,hosts,daemons,expected_len,must_have",
525 # hosts + (smaller) count, (more) existing
528 PlacementSpec(count
=3, hosts
='host3'.split()),
529 'host1 host2 host3'.split(),
534 # count + partial host list
537 PlacementSpec(count
=2, hosts
=['host3']),
538 'host1 host2 host3'.split(),
544 def test_node_assignment3(service_type
, placement
, hosts
,
545 daemons
, expected_len
, must_have
):
546 hosts
= HostAssignment(
547 spec
=ServiceSpec(service_type
, placement
=placement
),
548 hosts
=[HostSpec(h
) for h
in hosts
],
549 get_daemons_func
=lambda _
: daemons
).place()
550 assert len(hosts
) == expected_len
552 assert h
in [h
.hostname
for h
in hosts
]
555 @pytest.mark
.parametrize("placement",
560 ('hostname12hostname12hostname12hostname12hostname12hostname12hostname12'), # > 63 chars
562 def test_bad_placements(placement
):
564 s
= PlacementSpec
.from_string(placement
.split(' '))
566 except ServiceSpecValidationError
as e
:
570 class NodeAssignmentTestBadSpec(NamedTuple
):
572 placement
: PlacementSpec
574 daemons
: List
[DaemonDescription
]
576 @pytest.mark
.parametrize("service_type,placement,hosts,daemons,expected",
579 NodeAssignmentTestBadSpec(
581 PlacementSpec(hosts
=['unknownhost']),
584 "Cannot place <ServiceSpec for service_name=mgr> on unknownhost: Unknown hosts"
586 # unknown host pattern
587 NodeAssignmentTestBadSpec(
589 PlacementSpec(host_pattern
='unknownhost'),
592 "Cannot place <ServiceSpec for service_name=mgr>: No matching hosts"
595 NodeAssignmentTestBadSpec(
597 PlacementSpec(label
='unknownlabel'),
600 "Cannot place <ServiceSpec for service_name=mgr>: No matching hosts for label unknownlabel"
603 def test_bad_specs(service_type
, placement
, hosts
, daemons
, expected
):
604 with pytest
.raises(OrchestratorValidationError
) as e
:
605 hosts
= HostAssignment(
606 spec
=ServiceSpec(service_type
, placement
=placement
),
607 hosts
=[HostSpec(h
) for h
in hosts
],
608 get_daemons_func
=lambda _
: daemons
).place()
609 assert str(e
.value
) == expected
611 class ActiveAssignmentTest(NamedTuple
):
613 placement
: PlacementSpec
615 daemons
: List
[DaemonDescription
]
616 expected
: List
[List
[str]]
619 @pytest.mark
.parametrize("service_type,placement,hosts,daemons,expected",
621 ActiveAssignmentTest(
623 PlacementSpec(count
=2),
624 'host1 host2 host3'.split(),
626 DaemonDescription('mgr', 'a', 'host1', is_active
=True),
627 DaemonDescription('mgr', 'b', 'host2'),
628 DaemonDescription('mgr', 'c', 'host3'),
630 [['host1', 'host2'], ['host1', 'host3']]
632 ActiveAssignmentTest(
634 PlacementSpec(count
=2),
635 'host1 host2 host3'.split(),
637 DaemonDescription('mgr', 'a', 'host1'),
638 DaemonDescription('mgr', 'b', 'host2'),
639 DaemonDescription('mgr', 'c', 'host3', is_active
=True),
641 [['host1', 'host3'], ['host2', 'host3']]
643 ActiveAssignmentTest(
645 PlacementSpec(count
=1),
646 'host1 host2 host3'.split(),
648 DaemonDescription('mgr', 'a', 'host1'),
649 DaemonDescription('mgr', 'b', 'host2', is_active
=True),
650 DaemonDescription('mgr', 'c', 'host3'),
654 ActiveAssignmentTest(
656 PlacementSpec(count
=1),
657 'host1 host2 host3'.split(),
659 DaemonDescription('mgr', 'a', 'host1'),
660 DaemonDescription('mgr', 'b', 'host2'),
661 DaemonDescription('mgr', 'c', 'host3', is_active
=True),
665 ActiveAssignmentTest(
667 PlacementSpec(count
=1),
668 'host1 host2 host3'.split(),
670 DaemonDescription('mgr', 'a', 'host1', is_active
=True),
671 DaemonDescription('mgr', 'b', 'host2'),
672 DaemonDescription('mgr', 'c', 'host3', is_active
=True),
674 [['host1'], ['host3']]
676 ActiveAssignmentTest(
678 PlacementSpec(count
=2),
679 'host1 host2 host3'.split(),
681 DaemonDescription('mgr', 'a', 'host1'),
682 DaemonDescription('mgr', 'b', 'host2', is_active
=True),
683 DaemonDescription('mgr', 'c', 'host3', is_active
=True),
687 ActiveAssignmentTest(
689 PlacementSpec(count
=1),
690 'host1 host2 host3'.split(),
692 DaemonDescription('mgr', 'a', 'host1', is_active
=True),
693 DaemonDescription('mgr', 'b', 'host2', is_active
=True),
694 DaemonDescription('mgr', 'c', 'host3', is_active
=True),
696 [['host1'], ['host2'], ['host3']]
698 ActiveAssignmentTest(
700 PlacementSpec(count
=1),
701 'host1 host2 host3'.split(),
703 DaemonDescription('mgr', 'a', 'host1', is_active
=True),
704 DaemonDescription('mgr', 'a2', 'host1'),
705 DaemonDescription('mgr', 'b', 'host2'),
706 DaemonDescription('mgr', 'c', 'host3'),
710 ActiveAssignmentTest(
712 PlacementSpec(count
=1),
713 'host1 host2 host3'.split(),
715 DaemonDescription('mgr', 'a', 'host1', is_active
=True),
716 DaemonDescription('mgr', 'a2', 'host1', is_active
=True),
717 DaemonDescription('mgr', 'b', 'host2'),
718 DaemonDescription('mgr', 'c', 'host3'),
722 ActiveAssignmentTest(
724 PlacementSpec(count
=2),
725 'host1 host2 host3'.split(),
727 DaemonDescription('mgr', 'a', 'host1', is_active
=True),
728 DaemonDescription('mgr', 'a2', 'host1'),
729 DaemonDescription('mgr', 'b', 'host2'),
730 DaemonDescription('mgr', 'c', 'host3', is_active
=True),
734 # Explicit placement should override preference for active daemon
735 ActiveAssignmentTest(
737 PlacementSpec(count
=1, hosts
=['host1']),
738 'host1 host2 host3'.split(),
740 DaemonDescription('mgr', 'a', 'host1'),
741 DaemonDescription('mgr', 'b', 'host2'),
742 DaemonDescription('mgr', 'c', 'host3', is_active
=True),
748 def test_active_assignment(service_type
, placement
, hosts
, daemons
, expected
):
750 spec
= ServiceSpec(service_type
=service_type
,
754 hosts
= HostAssignment(
756 hosts
=[HostSpec(h
) for h
in hosts
],
757 get_daemons_func
=lambda _
: daemons
).place()
758 assert sorted([h
.hostname
for h
in hosts
]) in expected
760 class OddMonsTest(NamedTuple
):
762 placement
: PlacementSpec
764 daemons
: List
[DaemonDescription
]
768 @pytest.mark
.parametrize("service_type,placement,hosts,daemons,expected_count",
772 PlacementSpec(count
=5),
773 'host1 host2 host3 host4'.split(),
779 PlacementSpec(count
=4),
780 'host1 host2 host3 host4 host5'.split(),
786 PlacementSpec(count
=5),
787 'host1 host2 host3 host4 host5'.split(),
793 PlacementSpec(hosts
='host1 host2 host3 host4'.split()),
794 'host1 host2 host3 host4 host5'.split(),
800 PlacementSpec(hosts
='host1 host2 host3 host4 host5'.split()),
801 'host1 host2 host3 host4 host5'.split(),
807 PlacementSpec(host_pattern
='*'),
808 'host1 host2 host3 host4'.split(),
814 PlacementSpec(count
=5, hosts
='host1 host2 host3 host4'.split()),
815 'host1 host2 host3 host4 host5'.split(),
821 PlacementSpec(count
=2, hosts
='host1 host2 host3'.split()),
822 'host1 host2 host3 host4 host5'.split(),
828 PlacementSpec(count
=5),
829 'host1 host2 host3 host4'.split(),
831 DaemonDescription('mon', 'a', 'host1'),
832 DaemonDescription('mon', 'b', 'host2'),
833 DaemonDescription('mon', 'c', 'host3'),
839 PlacementSpec(count
=5),
840 'host1 host2 host3 host4'.split(),
842 DaemonDescription('mon', 'a', 'host1'),
843 DaemonDescription('mon', 'b', 'host2'),
849 PlacementSpec(hosts
='host1 host2 host3 host4'.split()),
850 'host1 host2 host3 host4 host5'.split(),
852 DaemonDescription('mon', 'a', 'host1'),
853 DaemonDescription('mon', 'b', 'host2'),
854 DaemonDescription('mon', 'c', 'host3'),
860 def test_odd_mons(service_type
, placement
, hosts
, daemons
, expected_count
):
862 spec
= ServiceSpec(service_type
=service_type
,
866 hosts
= HostAssignment(
868 hosts
=[HostSpec(h
) for h
in hosts
],
869 get_daemons_func
=lambda _
: daemons
).place()
870 assert len(hosts
) == expected_count