]> git.proxmox.com Git - ceph.git/blob - ceph/src/pybind/mgr/cephadm/tests/test_scheduling.py
projects / ceph.git / blob
? search:


82085957ffc47bee443b3b8437c274b2c29d5194
[ceph.git] / ceph / src / pybind / mgr / cephadm / tests / test_scheduling.py
1 # Disable autopep8 for this file:
2
3 # fmt: off
4
5 from typing import NamedTuple, List, Dict
6 import pytest
7
8 from ceph.deployment.hostspec import HostSpec
9 from ceph.deployment.service_spec import ServiceSpec, PlacementSpec, IngressSpec
10 from ceph.deployment.hostspec import SpecValidationError
11
12 from cephadm.module import HostAssignment
13 from cephadm.schedule import DaemonPlacement
14 from orchestrator import DaemonDescription, OrchestratorValidationError, OrchestratorError
15
16
17 def wrapper(func):
18 # some odd thingy to revert the order or arguments
19 def inner(*args):
20 def inner2(expected):
21 func(expected, *args)
22 return inner2
23 return inner
24
25
26 @wrapper
27 def none(expected):
28 assert expected == []
29
30
31 @wrapper
32 def one_of(expected, *hosts):
33 if not isinstance(expected, list):
34 assert False, str(expected)
35 assert len(expected) == 1, f'one_of failed len({expected}) != 1'
36 assert expected[0] in hosts
37
38
39 @wrapper
40 def two_of(expected, *hosts):
41 if not isinstance(expected, list):
42 assert False, str(expected)
43 assert len(expected) == 2, f'one_of failed len({expected}) != 2'
44 matches = 0
45 for h in hosts:
46 matches += int(h in expected)
47 if matches != 2:
48 assert False, f'two of {hosts} not in {expected}'
49
50
51 @wrapper
52 def exactly(expected, *hosts):
53 assert expected == list(hosts)
54
55
56 @wrapper
57 def error(expected, kind, match):
58 assert isinstance(expected, kind), (str(expected), match)
59 assert str(expected) == match, (str(expected), match)
60
61
62 @wrapper
63 def _or(expected, *inners):
64 def catch(inner):
65 try:
66 inner(expected)
67 except AssertionError as e:
68 return e
69 result = [catch(i) for i in inners]
70 if None not in result:
71 assert False, f"_or failed: {expected}"
72
73
74 def _always_true(_):
75 pass
76
77
78 def k(s):
79 return [e for e in s.split(' ') if e]
80
81
82 def get_result(key, results):
83 def match(one):
84 for o, k in zip(one, key):
85 if o != k and o != '*':
86 return False
87 return True
88 return [v for k, v in results if match(k)][0]
89
90
91 def mk_spec_and_host(spec_section, hosts, explicit_key, explicit, count):
92
93 if spec_section == 'hosts':
94 mk_spec = lambda: ServiceSpec('mgr', placement=PlacementSpec( # noqa: E731
95 hosts=explicit,
96 count=count,
97 ))
98 elif spec_section == 'label':
99 mk_spec = lambda: ServiceSpec('mgr', placement=PlacementSpec( # noqa: E731
100 label='mylabel',
101 count=count,
102 ))
103 elif spec_section == 'host_pattern':
104 pattern = {
105 'e': 'notfound',
106 '1': '1',
107 '12': '[1-2]',
108 '123': '*',
109 }[explicit_key]
110 mk_spec = lambda: ServiceSpec('mgr', placement=PlacementSpec( # noqa: E731
111 host_pattern=pattern,
112 count=count,
113 ))
114 else:
115 assert False
116
117 hosts = [
118 HostSpec(h, labels=['mylabel']) if h in explicit else HostSpec(h)
119 for h in hosts
120 ]
121
122 return mk_spec, hosts
123
124
125 def run_scheduler_test(results, mk_spec, hosts, daemons, key_elems):
126 key = ' '.join('N' if e is None else str(e) for e in key_elems)
127 try:
128 assert_res = get_result(k(key), results)
129 except IndexError:
130 try:
131 spec = mk_spec()
132 host_res, to_add, to_remove = HostAssignment(
133 spec=spec,
134 hosts=hosts,
135 daemons=daemons,
136 ).place()
137 if isinstance(host_res, list):
138 e = ', '.join(repr(h.hostname) for h in host_res)
139 assert False, f'`(k("{key}"), exactly({e})),` not found'
140 assert False, f'`(k("{key}"), ...),` not found'
141 except OrchestratorError as e:
142 assert False, f'`(k("{key}"), error({type(e).__name__}, {repr(str(e))})),` not found'
143
144 for _ in range(10): # scheduler has a random component
145 try:
146 spec = mk_spec()
147 host_res, to_add, to_remove = HostAssignment(
148 spec=spec,
149 hosts=hosts,
150 daemons=daemons
151 ).place()
152
153 assert_res(sorted([h.hostname for h in host_res]))
154 except Exception as e:
155 assert_res(e)
156
157
158 # * first match from the top wins
159 # * where e=[], *=any
160 #
161 # + list of known hosts available for scheduling (host_key)
162 # | + hosts used for explict placement (explicit_key)
163 # | | + count
164 # | | | + section (host, label, pattern)
165 # | | | | + expected result
166 # | | | | |
167 test_explicit_scheduler_results = [
168 (k("* * 0 *"), error(SpecValidationError, 'num/count must be > 1')),
169 (k("* e N l"), error(OrchestratorValidationError, 'Cannot place <ServiceSpec for service_name=mgr>: No matching hosts for label mylabel')),
170 (k("* e N p"), error(OrchestratorValidationError, 'Cannot place <ServiceSpec for service_name=mgr>: No matching hosts')),
171 (k("* e N h"), error(OrchestratorValidationError, 'placement spec is empty: no hosts, no label, no pattern, no count')),
172 (k("* e * *"), none),
173 (k("1 12 * h"), error(OrchestratorValidationError, "Cannot place <ServiceSpec for service_name=mgr> on 2: Unknown hosts")),
174 (k("1 123 * h"), error(OrchestratorValidationError, "Cannot place <ServiceSpec for service_name=mgr> on 2, 3: Unknown hosts")),
175 (k("1 * * *"), exactly('1')),
176 (k("12 1 * *"), exactly('1')),
177 (k("12 12 1 *"), one_of('1', '2')),
178 (k("12 12 * *"), exactly('1', '2')),
179 (k("12 123 * h"), error(OrchestratorValidationError, "Cannot place <ServiceSpec for service_name=mgr> on 3: Unknown hosts")),
180 (k("12 123 1 *"), one_of('1', '2', '3')),
181 (k("12 123 * *"), two_of('1', '2', '3')),
182 (k("123 1 * *"), exactly('1')),
183 (k("123 12 1 *"), one_of('1', '2')),
184 (k("123 12 * *"), exactly('1', '2')),
185 (k("123 123 1 *"), one_of('1', '2', '3')),
186 (k("123 123 2 *"), two_of('1', '2', '3')),
187 (k("123 123 * *"), exactly('1', '2', '3')),
188 ]
189
190
191 @pytest.mark.parametrize("dp,n,result",
192 [ # noqa: E128
193 (
194 DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[80]),
195 0,
196 DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[80]),
197 ),
198 (
199 DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[80]),
200 2,
201 DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[82]),
202 ),
203 (
204 DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[80, 90]),
205 2,
206 DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[82, 92]),
207 ),
208 ])
209 def test_daemon_placement_renumber(dp, n, result):
210 assert dp.renumber_ports(n) == result
211
212
213 @pytest.mark.parametrize(
214 'dp,dd,result',
215 [
216 (
217 DaemonPlacement(daemon_type='mgr', hostname='host1'),
218 DaemonDescription('mgr', 'a', 'host1'),
219 True
220 ),
221 (
222 DaemonPlacement(daemon_type='mgr', hostname='host1', name='a'),
223 DaemonDescription('mgr', 'a', 'host1'),
224 True
225 ),
226 (
227 DaemonPlacement(daemon_type='mon', hostname='host1', name='a'),
228 DaemonDescription('mgr', 'a', 'host1'),
229 False
230 ),
231 (
232 DaemonPlacement(daemon_type='mgr', hostname='host1', name='a'),
233 DaemonDescription('mgr', 'b', 'host1'),
234 False
235 ),
236 ])
237 def test_daemon_placement_match(dp, dd, result):
238 assert dp.matches_daemon(dd) == result
239
240
241 @pytest.mark.parametrize("spec_section_key,spec_section",
242 [ # noqa: E128
243 ('h', 'hosts'),
244 ('l', 'label'),
245 ('p', 'host_pattern'),
246 ])
247 @pytest.mark.parametrize("count",
248 [ # noqa: E128
249 None,
250 0,
251 1,
252 2,
253 3,
254 ])
255 @pytest.mark.parametrize("explicit_key, explicit",
256 [ # noqa: E128
257 ('e', []),
258 ('1', ['1']),
259 ('12', ['1', '2']),
260 ('123', ['1', '2', '3']),
261 ])
262 @pytest.mark.parametrize("host_key, hosts",
263 [ # noqa: E128
264 ('1', ['1']),
265 ('12', ['1', '2']),
266 ('123', ['1', '2', '3']),
267 ])
268 def test_explicit_scheduler(host_key, hosts,
269 explicit_key, explicit,
270 count,
271 spec_section_key, spec_section):
272
273 mk_spec, hosts = mk_spec_and_host(spec_section, hosts, explicit_key, explicit, count)
274 run_scheduler_test(
275 results=test_explicit_scheduler_results,
276 mk_spec=mk_spec,
277 hosts=hosts,
278 daemons=[],
279 key_elems=(host_key, explicit_key, count, spec_section_key)
280 )
281
282
283 # * first match from the top wins
284 # * where e=[], *=any
285 #
286 # + list of known hosts available for scheduling (host_key)
287 # | + hosts used for explict placement (explicit_key)
288 # | | + count
289 # | | | + existing daemons
290 # | | | | + section (host, label, pattern)
291 # | | | | | + expected result
292 # | | | | | |
293 test_scheduler_daemons_results = [
294 (k("* 1 * * *"), exactly('1')),
295 (k("1 123 * * h"), error(OrchestratorValidationError, 'Cannot place <ServiceSpec for service_name=mgr> on 2, 3: Unknown hosts')),
296 (k("1 123 * * *"), exactly('1')),
297 (k("12 123 * * h"), error(OrchestratorValidationError, 'Cannot place <ServiceSpec for service_name=mgr> on 3: Unknown hosts')),
298 (k("12 123 N * *"), exactly('1', '2')),
299 (k("12 123 1 * *"), one_of('1', '2')),
300 (k("12 123 2 * *"), exactly('1', '2')),
301 (k("12 123 3 * *"), exactly('1', '2')),
302 (k("123 123 N * *"), exactly('1', '2', '3')),
303 (k("123 123 1 e *"), one_of('1', '2', '3')),
304 (k("123 123 1 1 *"), exactly('1')),
305 (k("123 123 1 3 *"), exactly('3')),
306 (k("123 123 1 12 *"), one_of('1', '2')),
307 (k("123 123 1 112 *"), one_of('1', '2')),
308 (k("123 123 1 23 *"), one_of('2', '3')),
309 (k("123 123 1 123 *"), one_of('1', '2', '3')),
310 (k("123 123 2 e *"), two_of('1', '2', '3')),
311 (k("123 123 2 1 *"), _or(exactly('1', '2'), exactly('1', '3'))),
312 (k("123 123 2 3 *"), _or(exactly('1', '3'), exactly('2', '3'))),
313 (k("123 123 2 12 *"), exactly('1', '2')),
314 (k("123 123 2 112 *"), exactly('1', '2')),
315 (k("123 123 2 23 *"), exactly('2', '3')),
316 (k("123 123 2 123 *"), two_of('1', '2', '3')),
317 (k("123 123 3 * *"), exactly('1', '2', '3')),
318 ]
319
320
321 @pytest.mark.parametrize("spec_section_key,spec_section",
322 [ # noqa: E128
323 ('h', 'hosts'),
324 ('l', 'label'),
325 ('p', 'host_pattern'),
326 ])
327 @pytest.mark.parametrize("daemons_key, daemons",
328 [ # noqa: E128
329 ('e', []),
330 ('1', ['1']),
331 ('3', ['3']),
332 ('12', ['1', '2']),
333 ('112', ['1', '1', '2']), # deal with existing co-located daemons
334 ('23', ['2', '3']),
335 ('123', ['1', '2', '3']),
336 ])
337 @pytest.mark.parametrize("count",
338 [ # noqa: E128
339 None,
340 1,
341 2,
342 3,
343 ])
344 @pytest.mark.parametrize("explicit_key, explicit",
345 [ # noqa: E128
346 ('1', ['1']),
347 ('123', ['1', '2', '3']),
348 ])
349 @pytest.mark.parametrize("host_key, hosts",
350 [ # noqa: E128
351 ('1', ['1']),
352 ('12', ['1', '2']),
353 ('123', ['1', '2', '3']),
354 ])
355 def test_scheduler_daemons(host_key, hosts,
356 explicit_key, explicit,
357 count,
358 daemons_key, daemons,
359 spec_section_key, spec_section):
360 mk_spec, hosts = mk_spec_and_host(spec_section, hosts, explicit_key, explicit, count)
361 dds = [
362 DaemonDescription('mgr', d, d)
363 for d in daemons
364 ]
365 run_scheduler_test(
366 results=test_scheduler_daemons_results,
367 mk_spec=mk_spec,
368 hosts=hosts,
369 daemons=dds,
370 key_elems=(host_key, explicit_key, count, daemons_key, spec_section_key)
371 )
372
373
374 # =========================
375
376
377 class NodeAssignmentTest(NamedTuple):
378 service_type: str
379 placement: PlacementSpec
380 hosts: List[str]
381 daemons: List[DaemonDescription]
382 expected: List[str]
383 expected_add: List[str]
384 expected_remove: List[DaemonDescription]
385
386
387 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected,expected_add,expected_remove",
388 [ # noqa: E128
389 # just hosts
390 NodeAssignmentTest(
391 'mgr',
392 PlacementSpec(hosts=['smithi060']),
393 ['smithi060'],
394 [],
395 ['mgr:smithi060'], ['mgr:smithi060'], []
396 ),
397 # all_hosts
398 NodeAssignmentTest(
399 'mgr',
400 PlacementSpec(host_pattern='*'),
401 'host1 host2 host3'.split(),
402 [
403 DaemonDescription('mgr', 'a', 'host1'),
404 DaemonDescription('mgr', 'b', 'host2'),
405 ],
406 ['mgr:host1', 'mgr:host2', 'mgr:host3'],
407 ['mgr:host3'],
408 []
409 ),
410 # all_hosts + count_per_host
411 NodeAssignmentTest(
412 'mds',
413 PlacementSpec(host_pattern='*', count_per_host=2),
414 'host1 host2 host3'.split(),
415 [
416 DaemonDescription('mds', 'a', 'host1'),
417 DaemonDescription('mds', 'b', 'host2'),
418 ],
419 ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
420 ['mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
421 []
422 ),
423 # count that is bigger than the amount of hosts. Truncate to len(hosts)
424 # mgr should not be co-located to each other.
425 NodeAssignmentTest(
426 'mgr',
427 PlacementSpec(count=4),
428 'host1 host2 host3'.split(),
429 [],
430 ['mgr:host1', 'mgr:host2', 'mgr:host3'],
431 ['mgr:host1', 'mgr:host2', 'mgr:host3'],
432 []
433 ),
434 # count that is bigger than the amount of hosts; wrap around.
435 NodeAssignmentTest(
436 'mds',
437 PlacementSpec(count=6),
438 'host1 host2 host3'.split(),
439 [],
440 ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
441 ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
442 []
443 ),
444 # count + partial host list
445 NodeAssignmentTest(
446 'mgr',
447 PlacementSpec(count=3, hosts=['host3']),
448 'host1 host2 host3'.split(),
449 [
450 DaemonDescription('mgr', 'a', 'host1'),
451 DaemonDescription('mgr', 'b', 'host2'),
452 ],
453 ['mgr:host3'],
454 ['mgr:host3'],
455 ['mgr.a', 'mgr.b']
456 ),
457 # count + partial host list (with colo)
458 NodeAssignmentTest(
459 'mds',
460 PlacementSpec(count=3, hosts=['host3']),
461 'host1 host2 host3'.split(),
462 [
463 DaemonDescription('mds', 'a', 'host1'),
464 DaemonDescription('mds', 'b', 'host2'),
465 ],
466 ['mds:host3', 'mds:host3', 'mds:host3'],
467 ['mds:host3', 'mds:host3', 'mds:host3'],
468 ['mds.a', 'mds.b']
469 ),
470 # count 1 + partial host list
471 NodeAssignmentTest(
472 'mgr',
473 PlacementSpec(count=1, hosts=['host3']),
474 'host1 host2 host3'.split(),
475 [
476 DaemonDescription('mgr', 'a', 'host1'),
477 DaemonDescription('mgr', 'b', 'host2'),
478 ],
479 ['mgr:host3'],
480 ['mgr:host3'],
481 ['mgr.a', 'mgr.b']
482 ),
483 # count + partial host list + existing
484 NodeAssignmentTest(
485 'mgr',
486 PlacementSpec(count=2, hosts=['host3']),
487 'host1 host2 host3'.split(),
488 [
489 DaemonDescription('mgr', 'a', 'host1'),
490 ],
491 ['mgr:host3'],
492 ['mgr:host3'],
493 ['mgr.a']
494 ),
495 # count + partial host list + existing (deterministic)
496 NodeAssignmentTest(
497 'mgr',
498 PlacementSpec(count=2, hosts=['host1']),
499 'host1 host2'.split(),
500 [
501 DaemonDescription('mgr', 'a', 'host1'),
502 ],
503 ['mgr:host1'],
504 [],
505 []
506 ),
507 # count + partial host list + existing (deterministic)
508 NodeAssignmentTest(
509 'mgr',
510 PlacementSpec(count=2, hosts=['host1']),
511 'host1 host2'.split(),
512 [
513 DaemonDescription('mgr', 'a', 'host2'),
514 ],
515 ['mgr:host1'],
516 ['mgr:host1'],
517 ['mgr.a']
518 ),
519 # label only
520 NodeAssignmentTest(
521 'mgr',
522 PlacementSpec(label='foo'),
523 'host1 host2 host3'.split(),
524 [],
525 ['mgr:host1', 'mgr:host2', 'mgr:host3'],
526 ['mgr:host1', 'mgr:host2', 'mgr:host3'],
527 []
528 ),
529 # label + count (truncate to host list)
530 NodeAssignmentTest(
531 'mgr',
532 PlacementSpec(count=4, label='foo'),
533 'host1 host2 host3'.split(),
534 [],
535 ['mgr:host1', 'mgr:host2', 'mgr:host3'],
536 ['mgr:host1', 'mgr:host2', 'mgr:host3'],
537 []
538 ),
539 # label + count (with colo)
540 NodeAssignmentTest(
541 'mds',
542 PlacementSpec(count=6, label='foo'),
543 'host1 host2 host3'.split(),
544 [],
545 ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
546 ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
547 []
548 ),
549 # label only + count_per_hst
550 NodeAssignmentTest(
551 'mds',
552 PlacementSpec(label='foo', count_per_host=3),
553 'host1 host2 host3'.split(),
554 [],
555 ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3',
556 'mds:host1', 'mds:host2', 'mds:host3'],
557 ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3',
558 'mds:host1', 'mds:host2', 'mds:host3'],
559 []
560 ),
561 # host_pattern
562 NodeAssignmentTest(
563 'mgr',
564 PlacementSpec(host_pattern='mgr*'),
565 'mgrhost1 mgrhost2 datahost'.split(),
566 [],
567 ['mgr:mgrhost1', 'mgr:mgrhost2'],
568 ['mgr:mgrhost1', 'mgr:mgrhost2'],
569 []
570 ),
571 # host_pattern + count_per_host
572 NodeAssignmentTest(
573 'mds',
574 PlacementSpec(host_pattern='mds*', count_per_host=3),
575 'mdshost1 mdshost2 datahost'.split(),
576 [],
577 ['mds:mdshost1', 'mds:mdshost2', 'mds:mdshost1', 'mds:mdshost2', 'mds:mdshost1', 'mds:mdshost2'],
578 ['mds:mdshost1', 'mds:mdshost2', 'mds:mdshost1', 'mds:mdshost2', 'mds:mdshost1', 'mds:mdshost2'],
579 []
580 ),
581 # label + count_per_host + ports
582 NodeAssignmentTest(
583 'rgw',
584 PlacementSpec(count=6, label='foo'),
585 'host1 host2 host3'.split(),
586 [],
587 ['rgw:host1(*:80)', 'rgw:host2(*:80)', 'rgw:host3(*:80)',
588 'rgw:host1(*:81)', 'rgw:host2(*:81)', 'rgw:host3(*:81)'],
589 ['rgw:host1(*:80)', 'rgw:host2(*:80)', 'rgw:host3(*:80)',
590 'rgw:host1(*:81)', 'rgw:host2(*:81)', 'rgw:host3(*:81)'],
591 []
592 ),
593 # label + count_per_host + ports (+ xisting)
594 NodeAssignmentTest(
595 'rgw',
596 PlacementSpec(count=6, label='foo'),
597 'host1 host2 host3'.split(),
598 [
599 DaemonDescription('rgw', 'a', 'host1', ports=[81]),
600 DaemonDescription('rgw', 'b', 'host2', ports=[80]),
601 DaemonDescription('rgw', 'c', 'host1', ports=[82]),
602 ],
603 ['rgw:host1(*:80)', 'rgw:host2(*:80)', 'rgw:host3(*:80)',
604 'rgw:host1(*:81)', 'rgw:host2(*:81)', 'rgw:host3(*:81)'],
605 ['rgw:host1(*:80)', 'rgw:host3(*:80)',
606 'rgw:host2(*:81)', 'rgw:host3(*:81)'],
607 ['rgw.c']
608 ),
609 # cephadm.py teuth case
610 NodeAssignmentTest(
611 'mgr',
612 PlacementSpec(count=3, hosts=['host1=y', 'host2=x']),
613 'host1 host2'.split(),
614 [
615 DaemonDescription('mgr', 'y', 'host1'),
616 DaemonDescription('mgr', 'x', 'host2'),
617 ],
618 ['mgr:host1(name=y)', 'mgr:host2(name=x)'],
619 [], []
620 ),
621 ])
622 def test_node_assignment(service_type, placement, hosts, daemons,
623 expected, expected_add, expected_remove):
624 service_id = None
625 allow_colo = False
626 if service_type == 'rgw':
627 service_id = 'realm.zone'
628 allow_colo = True
629 elif service_type == 'mds':
630 service_id = 'myfs'
631 allow_colo = True
632
633 spec = ServiceSpec(service_type=service_type,
634 service_id=service_id,
635 placement=placement)
636
637 all_slots, to_add, to_remove = HostAssignment(
638 spec=spec,
639 hosts=[HostSpec(h, labels=['foo']) for h in hosts],
640 daemons=daemons,
641 allow_colo=allow_colo,
642 ).place()
643
644 got = [str(p) for p in all_slots]
645 num_wildcard = 0
646 for i in expected:
647 if i == '*':
648 num_wildcard += 1
649 else:
650 assert i in got
651 got.remove(i)
652 assert num_wildcard == len(got)
653
654 got = [str(p) for p in to_add]
655 num_wildcard = 0
656 for i in expected_add:
657 if i == '*':
658 num_wildcard += 1
659 else:
660 assert i in got
661 got.remove(i)
662 assert num_wildcard == len(got)
663
664 assert sorted([d.name() for d in to_remove]) == sorted(expected_remove)
665
666
667 class NodeAssignmentTest2(NamedTuple):
668 service_type: str
669 placement: PlacementSpec
670 hosts: List[str]
671 daemons: List[DaemonDescription]
672 expected_len: int
673 in_set: List[str]
674
675
676 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected_len,in_set",
677 [ # noqa: E128
678 # just count
679 NodeAssignmentTest2(
680 'mgr',
681 PlacementSpec(count=1),
682 'host1 host2 host3'.split(),
683 [],
684 1,
685 ['host1', 'host2', 'host3'],
686 ),
687
688 # hosts + (smaller) count
689 NodeAssignmentTest2(
690 'mgr',
691 PlacementSpec(count=1, hosts='host1 host2'.split()),
692 'host1 host2'.split(),
693 [],
694 1,
695 ['host1', 'host2'],
696 ),
697 # hosts + (smaller) count, existing
698 NodeAssignmentTest2(
699 'mgr',
700 PlacementSpec(count=1, hosts='host1 host2 host3'.split()),
701 'host1 host2 host3'.split(),
702 [DaemonDescription('mgr', 'mgr.a', 'host1')],
703 1,
704 ['host1', 'host2', 'host3'],
705 ),
706 # hosts + (smaller) count, (more) existing
707 NodeAssignmentTest2(
708 'mgr',
709 PlacementSpec(count=1, hosts='host1 host2 host3'.split()),
710 'host1 host2 host3'.split(),
711 [
712 DaemonDescription('mgr', 'a', 'host1'),
713 DaemonDescription('mgr', 'b', 'host2'),
714 ],
715 1,
716 ['host1', 'host2']
717 ),
718 # count + partial host list
719 NodeAssignmentTest2(
720 'mgr',
721 PlacementSpec(count=2, hosts=['host3']),
722 'host1 host2 host3'.split(),
723 [],
724 1,
725 ['host1', 'host2', 'host3']
726 ),
727 # label + count
728 NodeAssignmentTest2(
729 'mgr',
730 PlacementSpec(count=1, label='foo'),
731 'host1 host2 host3'.split(),
732 [],
733 1,
734 ['host1', 'host2', 'host3']
735 ),
736 ])
737 def test_node_assignment2(service_type, placement, hosts,
738 daemons, expected_len, in_set):
739 hosts, to_add, to_remove = HostAssignment(
740 spec=ServiceSpec(service_type, placement=placement),
741 hosts=[HostSpec(h, labels=['foo']) for h in hosts],
742 daemons=daemons,
743 ).place()
744 assert len(hosts) == expected_len
745 for h in [h.hostname for h in hosts]:
746 assert h in in_set
747
748
749 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected_len,must_have",
750 [ # noqa: E128
751 # hosts + (smaller) count, (more) existing
752 NodeAssignmentTest2(
753 'mgr',
754 PlacementSpec(count=3, hosts='host3'.split()),
755 'host1 host2 host3'.split(),
756 [],
757 1,
758 ['host3']
759 ),
760 # count + partial host list
761 NodeAssignmentTest2(
762 'mgr',
763 PlacementSpec(count=2, hosts=['host3']),
764 'host1 host2 host3'.split(),
765 [],
766 1,
767 ['host3']
768 ),
769 ])
770 def test_node_assignment3(service_type, placement, hosts,
771 daemons, expected_len, must_have):
772 hosts, to_add, to_remove = HostAssignment(
773 spec=ServiceSpec(service_type, placement=placement),
774 hosts=[HostSpec(h) for h in hosts],
775 daemons=daemons,
776 ).place()
777 assert len(hosts) == expected_len
778 for h in must_have:
779 assert h in [h.hostname for h in hosts]
780
781
782 class NodeAssignmentTest4(NamedTuple):
783 spec: ServiceSpec
784 networks: Dict[str, Dict[str, Dict[str, List[str]]]]
785 daemons: List[DaemonDescription]
786 expected: List[str]
787 expected_add: List[str]
788 expected_remove: List[DaemonDescription]
789
790
791 @pytest.mark.parametrize("spec,networks,daemons,expected,expected_add,expected_remove",
792 [ # noqa: E128
793 NodeAssignmentTest4(
794 ServiceSpec(
795 service_type='rgw',
796 service_id='foo',
797 placement=PlacementSpec(count=6, label='foo'),
798 networks=['10.0.0.0/8'],
799 ),
800 {
801 'host1': {'10.0.0.0/8': {'eth0': ['10.0.0.1']}},
802 'host2': {'10.0.0.0/8': {'eth0': ['10.0.0.2']}},
803 'host3': {'192.168.0.0/16': {'eth0': ['192.168.0.1']}},
804 },
805 [],
806 ['rgw:host1(10.0.0.1:80)', 'rgw:host2(10.0.0.2:80)',
807 'rgw:host1(10.0.0.1:81)', 'rgw:host2(10.0.0.2:81)',
808 'rgw:host1(10.0.0.1:82)', 'rgw:host2(10.0.0.2:82)'],
809 ['rgw:host1(10.0.0.1:80)', 'rgw:host2(10.0.0.2:80)',
810 'rgw:host1(10.0.0.1:81)', 'rgw:host2(10.0.0.2:81)',
811 'rgw:host1(10.0.0.1:82)', 'rgw:host2(10.0.0.2:82)'],
812 []
813 ),
814 NodeAssignmentTest4(
815 IngressSpec(
816 service_type='ingress',
817 service_id='rgw.foo',
818 frontend_port=443,
819 monitor_port=8888,
820 virtual_ip='10.0.0.20/8',
821 backend_service='rgw.foo',
822 placement=PlacementSpec(label='foo'),
823 networks=['10.0.0.0/8'],
824 ),
825 {
826 'host1': {'10.0.0.0/8': {'eth0': ['10.0.0.1']}},
827 'host2': {'10.0.0.0/8': {'eth1': ['10.0.0.2']}},
828 'host3': {'192.168.0.0/16': {'eth2': ['192.168.0.1']}},
829 },
830 [],
831 ['haproxy:host1(10.0.0.1:443,8888)', 'haproxy:host2(10.0.0.2:443,8888)',
832 'keepalived:host1', 'keepalived:host2'],
833 ['haproxy:host1(10.0.0.1:443,8888)', 'haproxy:host2(10.0.0.2:443,8888)',
834 'keepalived:host1', 'keepalived:host2'],
835 []
836 ),
837 NodeAssignmentTest4(
838 IngressSpec(
839 service_type='ingress',
840 service_id='rgw.foo',
841 frontend_port=443,
842 monitor_port=8888,
843 virtual_ip='10.0.0.20/8',
844 backend_service='rgw.foo',
845 placement=PlacementSpec(label='foo'),
846 networks=['10.0.0.0/8'],
847 ),
848 {
849 'host1': {'10.0.0.0/8': {'eth0': ['10.0.0.1']}},
850 'host2': {'10.0.0.0/8': {'eth1': ['10.0.0.2']}},
851 'host3': {'192.168.0.0/16': {'eth2': ['192.168.0.1']}},
852 },
853 [
854 DaemonDescription('haproxy', 'a', 'host1', ip='10.0.0.1',
855 ports=[443, 8888]),
856 DaemonDescription('keepalived', 'b', 'host2'),
857 DaemonDescription('keepalived', 'c', 'host3'),
858 ],
859 ['haproxy:host1(10.0.0.1:443,8888)', 'haproxy:host2(10.0.0.2:443,8888)',
860 'keepalived:host1', 'keepalived:host2'],
861 ['haproxy:host2(10.0.0.2:443,8888)',
862 'keepalived:host1'],
863 ['keepalived.c']
864 ),
865 ])
866 def test_node_assignment4(spec, networks, daemons,
867 expected, expected_add, expected_remove):
868 all_slots, to_add, to_remove = HostAssignment(
869 spec=spec,
870 hosts=[HostSpec(h, labels=['foo']) for h in networks.keys()],
871 daemons=daemons,
872 allow_colo=True,
873 networks=networks,
874 primary_daemon_type='haproxy' if spec.service_type == 'ingress' else spec.service_type,
875 per_host_daemon_type='keepalived' if spec.service_type == 'ingress' else None,
876 ).place()
877
878 got = [str(p) for p in all_slots]
879 num_wildcard = 0
880 for i in expected:
881 if i == '*':
882 num_wildcard += 1
883 else:
884 assert i in got
885 got.remove(i)
886 assert num_wildcard == len(got)
887
888 got = [str(p) for p in to_add]
889 num_wildcard = 0
890 for i in expected_add:
891 if i == '*':
892 num_wildcard += 1
893 else:
894 assert i in got
895 got.remove(i)
896 assert num_wildcard == len(got)
897
898 assert sorted([d.name() for d in to_remove]) == sorted(expected_remove)
899
900
901 @pytest.mark.parametrize("placement",
902 [ # noqa: E128
903 ('1 *'),
904 ('* label:foo'),
905 ('* host1 host2'),
906 ('hostname12hostname12hostname12hostname12hostname12hostname12hostname12'), # > 63 chars
907 ])
908 def test_bad_placements(placement):
909 try:
910 PlacementSpec.from_string(placement.split(' '))
911 assert False
912 except SpecValidationError:
913 pass
914
915
916 class NodeAssignmentTestBadSpec(NamedTuple):
917 service_type: str
918 placement: PlacementSpec
919 hosts: List[str]
920 daemons: List[DaemonDescription]
921 expected: str
922
923
924 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected",
925 [ # noqa: E128
926 # unknown host
927 NodeAssignmentTestBadSpec(
928 'mgr',
929 PlacementSpec(hosts=['unknownhost']),
930 ['knownhost'],
931 [],
932 "Cannot place <ServiceSpec for service_name=mgr> on unknownhost: Unknown hosts"
933 ),
934 # unknown host pattern
935 NodeAssignmentTestBadSpec(
936 'mgr',
937 PlacementSpec(host_pattern='unknownhost'),
938 ['knownhost'],
939 [],
940 "Cannot place <ServiceSpec for service_name=mgr>: No matching hosts"
941 ),
942 # unknown label
943 NodeAssignmentTestBadSpec(
944 'mgr',
945 PlacementSpec(label='unknownlabel'),
946 [],
947 [],
948 "Cannot place <ServiceSpec for service_name=mgr>: No matching hosts for label unknownlabel"
949 ),
950 ])
951 def test_bad_specs(service_type, placement, hosts, daemons, expected):
952 with pytest.raises(OrchestratorValidationError) as e:
953 hosts, to_add, to_remove = HostAssignment(
954 spec=ServiceSpec(service_type, placement=placement),
955 hosts=[HostSpec(h) for h in hosts],
956 daemons=daemons,
957 ).place()
958 assert str(e.value) == expected
959
960
961 class ActiveAssignmentTest(NamedTuple):
962 service_type: str
963 placement: PlacementSpec
964 hosts: List[str]
965 daemons: List[DaemonDescription]
966 expected: List[List[str]]
967 expected_add: List[List[str]]
968 expected_remove: List[List[str]]
969
970
971 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected,expected_add,expected_remove",
972 [
973 ActiveAssignmentTest(
974 'mgr',
975 PlacementSpec(count=2),
976 'host1 host2 host3'.split(),
977 [
978 DaemonDescription('mgr', 'a', 'host1', is_active=True),
979 DaemonDescription('mgr', 'b', 'host2'),
980 DaemonDescription('mgr', 'c', 'host3'),
981 ],
982 [['host1', 'host2'], ['host1', 'host3']],
983 [[]],
984 [['mgr.b'], ['mgr.c']]
985 ),
986 ActiveAssignmentTest(
987 'mgr',
988 PlacementSpec(count=2),
989 'host1 host2 host3'.split(),
990 [
991 DaemonDescription('mgr', 'a', 'host1'),
992 DaemonDescription('mgr', 'b', 'host2'),
993 DaemonDescription('mgr', 'c', 'host3', is_active=True),
994 ],
995 [['host1', 'host3'], ['host2', 'host3']],
996 [[]],
997 [['mgr.a'], ['mgr.b']]
998 ),
999 ActiveAssignmentTest(
1000 'mgr',
1001 PlacementSpec(count=1),
1002 'host1 host2 host3'.split(),
1003 [
1004 DaemonDescription('mgr', 'a', 'host1'),
1005 DaemonDescription('mgr', 'b', 'host2', is_active=True),
1006 DaemonDescription('mgr', 'c', 'host3'),
1007 ],
1008 [['host2']],
1009 [[]],
1010 [['mgr.a', 'mgr.c']]
1011 ),
1012 ActiveAssignmentTest(
1013 'mgr',
1014 PlacementSpec(count=1),
1015 'host1 host2 host3'.split(),
1016 [
1017 DaemonDescription('mgr', 'a', 'host1'),
1018 DaemonDescription('mgr', 'b', 'host2'),
1019 DaemonDescription('mgr', 'c', 'host3', is_active=True),
1020 ],
1021 [['host3']],
1022 [[]],
1023 [['mgr.a', 'mgr.b']]
1024 ),
1025 ActiveAssignmentTest(
1026 'mgr',
1027 PlacementSpec(count=1),
1028 'host1 host2 host3'.split(),
1029 [
1030 DaemonDescription('mgr', 'a', 'host1', is_active=True),
1031 DaemonDescription('mgr', 'b', 'host2'),
1032 DaemonDescription('mgr', 'c', 'host3', is_active=True),
1033 ],
1034 [['host1'], ['host3']],
1035 [[]],
1036 [['mgr.a', 'mgr.b'], ['mgr.b', 'mgr.c']]
1037 ),
1038 ActiveAssignmentTest(
1039 'mgr',
1040 PlacementSpec(count=2),
1041 'host1 host2 host3'.split(),
1042 [
1043 DaemonDescription('mgr', 'a', 'host1'),
1044 DaemonDescription('mgr', 'b', 'host2', is_active=True),
1045 DaemonDescription('mgr', 'c', 'host3', is_active=True),
1046 ],
1047 [['host2', 'host3']],
1048 [[]],
1049 [['mgr.a']]
1050 ),
1051 ActiveAssignmentTest(
1052 'mgr',
1053 PlacementSpec(count=1),
1054 'host1 host2 host3'.split(),
1055 [
1056 DaemonDescription('mgr', 'a', 'host1', is_active=True),
1057 DaemonDescription('mgr', 'b', 'host2', is_active=True),
1058 DaemonDescription('mgr', 'c', 'host3', is_active=True),
1059 ],
1060 [['host1'], ['host2'], ['host3']],
1061 [[]],
1062 [['mgr.a', 'mgr.b'], ['mgr.b', 'mgr.c'], ['mgr.a', 'mgr.c']]
1063 ),
1064 ActiveAssignmentTest(
1065 'mgr',
1066 PlacementSpec(count=1),
1067 'host1 host2 host3'.split(),
1068 [
1069 DaemonDescription('mgr', 'a', 'host1', is_active=True),
1070 DaemonDescription('mgr', 'a2', 'host1'),
1071 DaemonDescription('mgr', 'b', 'host2'),
1072 DaemonDescription('mgr', 'c', 'host3'),
1073 ],
1074 [['host1']],
1075 [[]],
1076 [['mgr.a2', 'mgr.b', 'mgr.c']]
1077 ),
1078 ActiveAssignmentTest(
1079 'mgr',
1080 PlacementSpec(count=1),
1081 'host1 host2 host3'.split(),
1082 [
1083 DaemonDescription('mgr', 'a', 'host1', is_active=True),
1084 DaemonDescription('mgr', 'a2', 'host1', is_active=True),
1085 DaemonDescription('mgr', 'b', 'host2'),
1086 DaemonDescription('mgr', 'c', 'host3'),
1087 ],
1088 [['host1']],
1089 [[]],
1090 [['mgr.a', 'mgr.b', 'mgr.c'], ['mgr.a2', 'mgr.b', 'mgr.c']]
1091 ),
1092 ActiveAssignmentTest(
1093 'mgr',
1094 PlacementSpec(count=2),
1095 'host1 host2 host3'.split(),
1096 [
1097 DaemonDescription('mgr', 'a', 'host1', is_active=True),
1098 DaemonDescription('mgr', 'a2', 'host1'),
1099 DaemonDescription('mgr', 'b', 'host2'),
1100 DaemonDescription('mgr', 'c', 'host3', is_active=True),
1101 ],
1102 [['host1', 'host3']],
1103 [[]],
1104 [['mgr.a2', 'mgr.b']]
1105 ),
1106 # Explicit placement should override preference for active daemon
1107 ActiveAssignmentTest(
1108 'mgr',
1109 PlacementSpec(count=1, hosts=['host1']),
1110 'host1 host2 host3'.split(),
1111 [
1112 DaemonDescription('mgr', 'a', 'host1'),
1113 DaemonDescription('mgr', 'b', 'host2'),
1114 DaemonDescription('mgr', 'c', 'host3', is_active=True),
1115 ],
1116 [['host1']],
1117 [[]],
1118 [['mgr.b', 'mgr.c']]
1119 ),
1120
1121 ])
1122 def test_active_assignment(service_type, placement, hosts, daemons, expected, expected_add, expected_remove):
1123
1124 spec = ServiceSpec(service_type=service_type,
1125 service_id=None,
1126 placement=placement)
1127
1128 hosts, to_add, to_remove = HostAssignment(
1129 spec=spec,
1130 hosts=[HostSpec(h) for h in hosts],
1131 daemons=daemons,
1132 ).place()
1133 assert sorted([h.hostname for h in hosts]) in expected
1134 assert sorted([h.hostname for h in to_add]) in expected_add
1135 assert sorted([h.name() for h in to_remove]) in expected_remove
Ceph