ceph/src/pybind/mgr/cephadm/tests/test_scheduling.py

   1 # Disable autopep8 for this file:
   2
   3 # fmt: off
   4
   5 from typing import NamedTuple, List, Dict, Optional
   6 import pytest
   7
   8 from ceph.deployment.hostspec import HostSpec
   9 from ceph.deployment.service_spec import ServiceSpec, PlacementSpec, IngressSpec
  10 from ceph.deployment.hostspec import SpecValidationError
  11
  12 from cephadm.module import HostAssignment
  13 from cephadm.schedule import DaemonPlacement
  14 from orchestrator import DaemonDescription, OrchestratorValidationError, OrchestratorError
  15
  16
  17 def wrapper(func):
  18     # some odd thingy to revert the order or arguments
  19     def inner(*args):
  20         def inner2(expected):
  21             func(expected, *args)
  22         return inner2
  23     return inner
  24
  25
  26 @wrapper
  27 def none(expected):
  28     assert expected == []
  29
  30
  31 @wrapper
  32 def one_of(expected, *hosts):
  33     if not isinstance(expected, list):
  34         assert False, str(expected)
  35     assert len(expected) == 1, f'one_of failed len({expected}) != 1'
  36     assert expected[0] in hosts
  37
  38
  39 @wrapper
  40 def two_of(expected, *hosts):
  41     if not isinstance(expected, list):
  42         assert False, str(expected)
  43     assert len(expected) == 2, f'one_of failed len({expected}) != 2'
  44     matches = 0
  45     for h in hosts:
  46         matches += int(h in expected)
  47     if matches != 2:
  48         assert False, f'two of {hosts} not in {expected}'
  49
  50
  51 @wrapper
  52 def exactly(expected, *hosts):
  53     assert expected == list(hosts)
  54
  55
  56 @wrapper
  57 def error(expected, kind, match):
  58     assert isinstance(expected, kind), (str(expected), match)
  59     assert str(expected) == match, (str(expected), match)
  60
  61
  62 @wrapper
  63 def _or(expected, *inners):
  64     def catch(inner):
  65         try:
  66             inner(expected)
  67         except AssertionError as e:
  68             return e
  69     result = [catch(i) for i in inners]
  70     if None not in result:
  71         assert False, f"_or failed: {expected}"
  72
  73
  74 def _always_true(_):
  75     pass
  76
  77
  78 def k(s):
  79     return [e for e in s.split(' ') if e]
  80
  81
  82 def get_result(key, results):
  83     def match(one):
  84         for o, k in zip(one, key):
  85             if o != k and o != '*':
  86                 return False
  87         return True
  88     return [v for k, v in results if match(k)][0]
  89
  90
  91 def mk_spec_and_host(spec_section, hosts, explicit_key, explicit, count):
  92
  93     if spec_section == 'hosts':
  94         mk_spec = lambda: ServiceSpec('mgr', placement=PlacementSpec(  # noqa: E731
  95             hosts=explicit,
  96             count=count,
  97         ))
  98     elif spec_section == 'label':
  99         mk_spec = lambda: ServiceSpec('mgr', placement=PlacementSpec(  # noqa: E731
 100             label='mylabel',
 101             count=count,
 102         ))
 103     elif spec_section == 'host_pattern':
 104         pattern = {
 105             'e': 'notfound',
 106             '1': '1',
 107             '12': '[1-2]',
 108             '123': '*',
 109         }[explicit_key]
 110         mk_spec = lambda: ServiceSpec('mgr', placement=PlacementSpec(  # noqa: E731
 111             host_pattern=pattern,
 112             count=count,
 113         ))
 114     else:
 115         assert False
 116
 117     hosts = [
 118         HostSpec(h, labels=['mylabel']) if h in explicit else HostSpec(h)
 119         for h in hosts
 120     ]
 121
 122     return mk_spec, hosts
 123
 124
 125 def run_scheduler_test(results, mk_spec, hosts, daemons, key_elems):
 126     key = ' '.join('N' if e is None else str(e) for e in key_elems)
 127     try:
 128         assert_res = get_result(k(key), results)
 129     except IndexError:
 130         try:
 131             spec = mk_spec()
 132             host_res, to_add, to_remove = HostAssignment(
 133                 spec=spec,
 134                 hosts=hosts,
 135                 unreachable_hosts=[],
 136                 daemons=daemons,
 137             ).place()
 138             if isinstance(host_res, list):
 139                 e = ', '.join(repr(h.hostname) for h in host_res)
 140                 assert False, f'`(k("{key}"), exactly({e})),` not found'
 141             assert False, f'`(k("{key}"), ...),` not found'
 142         except OrchestratorError as e:
 143             assert False, f'`(k("{key}"), error({type(e).__name__}, {repr(str(e))})),` not found'
 144
 145     for _ in range(10):  # scheduler has a random component
 146         try:
 147             spec = mk_spec()
 148             host_res, to_add, to_remove = HostAssignment(
 149                 spec=spec,
 150                 hosts=hosts,
 151                 unreachable_hosts=[],
 152                 daemons=daemons
 153             ).place()
 154
 155             assert_res(sorted([h.hostname for h in host_res]))
 156         except Exception as e:
 157             assert_res(e)
 158
 159
 160 # * first match from the top wins
 161 # * where e=[], *=any
 162 #
 163 #       + list of known hosts available for scheduling (host_key)
 164 #       |   + hosts used for explict placement (explicit_key)
 165 #       |   |   + count
 166 #       |   |   | + section (host, label, pattern)
 167 #       |   |   | |     + expected result
 168 #       |   |   | |     |
 169 test_explicit_scheduler_results = [
 170     (k("*   *   0 *"), error(SpecValidationError, 'num/count must be > 1')),
 171     (k("*   e   N l"), error(OrchestratorValidationError, 'Cannot place <ServiceSpec for service_name=mgr>: No matching hosts for label mylabel')),
 172     (k("*   e   N p"), error(OrchestratorValidationError, 'Cannot place <ServiceSpec for service_name=mgr>: No matching hosts')),
 173     (k("*   e   N h"), error(OrchestratorValidationError, 'placement spec is empty: no hosts, no label, no pattern, no count')),
 174     (k("*   e   * *"), none),
 175     (k("1   12  * h"), error(OrchestratorValidationError, "Cannot place <ServiceSpec for service_name=mgr> on 2: Unknown hosts")),
 176     (k("1   123 * h"), error(OrchestratorValidationError, "Cannot place <ServiceSpec for service_name=mgr> on 2, 3: Unknown hosts")),
 177     (k("1   *   * *"), exactly('1')),
 178     (k("12  1   * *"), exactly('1')),
 179     (k("12  12  1 *"), one_of('1', '2')),
 180     (k("12  12  * *"), exactly('1', '2')),
 181     (k("12  123 * h"), error(OrchestratorValidationError, "Cannot place <ServiceSpec for service_name=mgr> on 3: Unknown hosts")),
 182     (k("12  123 1 *"), one_of('1', '2', '3')),
 183     (k("12  123 * *"), two_of('1', '2', '3')),
 184     (k("123 1   * *"), exactly('1')),
 185     (k("123 12  1 *"), one_of('1', '2')),
 186     (k("123 12  * *"), exactly('1', '2')),
 187     (k("123 123 1 *"), one_of('1', '2', '3')),
 188     (k("123 123 2 *"), two_of('1', '2', '3')),
 189     (k("123 123 * *"), exactly('1', '2', '3')),
 190 ]
 191
 192
 193 @pytest.mark.parametrize("dp,n,result",
 194     [   # noqa: E128
 195         (
 196             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[80]),
 197             0,
 198             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[80]),
 199         ),
 200         (
 201             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[80]),
 202             2,
 203             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[82]),
 204         ),
 205         (
 206             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[80, 90]),
 207             2,
 208             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[82, 92]),
 209         ),
 210     ])
 211 def test_daemon_placement_renumber(dp, n, result):
 212     assert dp.renumber_ports(n) == result
 213
 214
 215 @pytest.mark.parametrize(
 216     'dp,dd,result',
 217     [
 218         (
 219             DaemonPlacement(daemon_type='mgr', hostname='host1'),
 220             DaemonDescription('mgr', 'a', 'host1'),
 221             True
 222         ),
 223         (
 224             DaemonPlacement(daemon_type='mgr', hostname='host1', name='a'),
 225             DaemonDescription('mgr', 'a', 'host1'),
 226             True
 227         ),
 228         (
 229             DaemonPlacement(daemon_type='mon', hostname='host1', name='a'),
 230             DaemonDescription('mgr', 'a', 'host1'),
 231             False
 232         ),
 233         (
 234             DaemonPlacement(daemon_type='mgr', hostname='host1', name='a'),
 235             DaemonDescription('mgr', 'b', 'host1'),
 236             False
 237         ),
 238     ])
 239 def test_daemon_placement_match(dp, dd, result):
 240     assert dp.matches_daemon(dd) == result
 241
 242
 243 @pytest.mark.parametrize("spec_section_key,spec_section",
 244     [   # noqa: E128
 245         ('h', 'hosts'),
 246         ('l', 'label'),
 247         ('p', 'host_pattern'),
 248     ])
 249 @pytest.mark.parametrize("count",
 250     [   # noqa: E128
 251         None,
 252         0,
 253         1,
 254         2,
 255         3,
 256     ])
 257 @pytest.mark.parametrize("explicit_key, explicit",
 258     [   # noqa: E128
 259         ('e', []),
 260         ('1', ['1']),
 261         ('12', ['1', '2']),
 262         ('123', ['1', '2', '3']),
 263     ])
 264 @pytest.mark.parametrize("host_key, hosts",
 265     [   # noqa: E128
 266         ('1', ['1']),
 267         ('12', ['1', '2']),
 268         ('123', ['1', '2', '3']),
 269     ])
 270 def test_explicit_scheduler(host_key, hosts,
 271                             explicit_key, explicit,
 272                             count,
 273                             spec_section_key, spec_section):
 274
 275     mk_spec, hosts = mk_spec_and_host(spec_section, hosts, explicit_key, explicit, count)
 276     run_scheduler_test(
 277         results=test_explicit_scheduler_results,
 278         mk_spec=mk_spec,
 279         hosts=hosts,
 280         daemons=[],
 281         key_elems=(host_key, explicit_key, count, spec_section_key)
 282     )
 283
 284
 285 # * first match from the top wins
 286 # * where e=[], *=any
 287 #
 288 #       + list of known hosts available for scheduling (host_key)
 289 #       |   + hosts used for explict placement (explicit_key)
 290 #       |   |   + count
 291 #       |   |   | + existing daemons
 292 #       |   |   | |     + section (host, label, pattern)
 293 #       |   |   | |     |   + expected result
 294 #       |   |   | |     |   |
 295 test_scheduler_daemons_results = [
 296     (k("*   1   * *   *"), exactly('1')),
 297     (k("1   123 * *   h"), error(OrchestratorValidationError, 'Cannot place <ServiceSpec for service_name=mgr> on 2, 3: Unknown hosts')),
 298     (k("1   123 * *   *"), exactly('1')),
 299     (k("12  123 * *   h"), error(OrchestratorValidationError, 'Cannot place <ServiceSpec for service_name=mgr> on 3: Unknown hosts')),
 300     (k("12  123 N *   *"), exactly('1', '2')),
 301     (k("12  123 1 *   *"), one_of('1', '2')),
 302     (k("12  123 2 *   *"), exactly('1', '2')),
 303     (k("12  123 3 *   *"), exactly('1', '2')),
 304     (k("123 123 N *   *"), exactly('1', '2', '3')),
 305     (k("123 123 1 e   *"), one_of('1', '2', '3')),
 306     (k("123 123 1 1   *"), exactly('1')),
 307     (k("123 123 1 3   *"), exactly('3')),
 308     (k("123 123 1 12  *"), one_of('1', '2')),
 309     (k("123 123 1 112 *"), one_of('1', '2')),
 310     (k("123 123 1 23  *"), one_of('2', '3')),
 311     (k("123 123 1 123 *"), one_of('1', '2', '3')),
 312     (k("123 123 2 e   *"), two_of('1', '2', '3')),
 313     (k("123 123 2 1   *"), _or(exactly('1', '2'), exactly('1', '3'))),
 314     (k("123 123 2 3   *"), _or(exactly('1', '3'), exactly('2', '3'))),
 315     (k("123 123 2 12  *"), exactly('1', '2')),
 316     (k("123 123 2 112 *"), exactly('1', '2')),
 317     (k("123 123 2 23  *"), exactly('2', '3')),
 318     (k("123 123 2 123 *"), two_of('1', '2', '3')),
 319     (k("123 123 3 *   *"), exactly('1', '2', '3')),
 320 ]
 321
 322
 323 @pytest.mark.parametrize("spec_section_key,spec_section",
 324     [   # noqa: E128
 325         ('h', 'hosts'),
 326         ('l', 'label'),
 327         ('p', 'host_pattern'),
 328     ])
 329 @pytest.mark.parametrize("daemons_key, daemons",
 330     [   # noqa: E128
 331         ('e', []),
 332         ('1', ['1']),
 333         ('3', ['3']),
 334         ('12', ['1', '2']),
 335         ('112', ['1', '1', '2']),  # deal with existing co-located daemons
 336         ('23', ['2', '3']),
 337         ('123', ['1', '2', '3']),
 338     ])
 339 @pytest.mark.parametrize("count",
 340     [   # noqa: E128
 341         None,
 342         1,
 343         2,
 344         3,
 345     ])
 346 @pytest.mark.parametrize("explicit_key, explicit",
 347     [   # noqa: E128
 348         ('1', ['1']),
 349         ('123', ['1', '2', '3']),
 350     ])
 351 @pytest.mark.parametrize("host_key, hosts",
 352     [   # noqa: E128
 353         ('1', ['1']),
 354         ('12', ['1', '2']),
 355         ('123', ['1', '2', '3']),
 356     ])
 357 def test_scheduler_daemons(host_key, hosts,
 358                            explicit_key, explicit,
 359                            count,
 360                            daemons_key, daemons,
 361                            spec_section_key, spec_section):
 362     mk_spec, hosts = mk_spec_and_host(spec_section, hosts, explicit_key, explicit, count)
 363     dds = [
 364         DaemonDescription('mgr', d, d)
 365         for d in daemons
 366     ]
 367     run_scheduler_test(
 368         results=test_scheduler_daemons_results,
 369         mk_spec=mk_spec,
 370         hosts=hosts,
 371         daemons=dds,
 372         key_elems=(host_key, explicit_key, count, daemons_key, spec_section_key)
 373     )
 374
 375
 376 # =========================
 377
 378
 379 class NodeAssignmentTest(NamedTuple):
 380     service_type: str
 381     placement: PlacementSpec
 382     hosts: List[str]
 383     daemons: List[DaemonDescription]
 384     rank_map: Optional[Dict[int, Dict[int, Optional[str]]]]
 385     post_rank_map: Optional[Dict[int, Dict[int, Optional[str]]]]
 386     expected: List[str]
 387     expected_add: List[str]
 388     expected_remove: List[DaemonDescription]
 389
 390
 391 @pytest.mark.parametrize("service_type,placement,hosts,daemons,rank_map,post_rank_map,expected,expected_add,expected_remove",
 392     [   # noqa: E128
 393         # just hosts
 394         NodeAssignmentTest(
 395             'mgr',
 396             PlacementSpec(hosts=['smithi060']),
 397             ['smithi060'],
 398             [],
 399             None, None,
 400             ['mgr:smithi060'], ['mgr:smithi060'], []
 401         ),
 402         # all_hosts
 403         NodeAssignmentTest(
 404             'mgr',
 405             PlacementSpec(host_pattern='*'),
 406             'host1 host2 host3'.split(),
 407             [
 408                 DaemonDescription('mgr', 'a', 'host1'),
 409                 DaemonDescription('mgr', 'b', 'host2'),
 410             ],
 411             None, None,
 412             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 413             ['mgr:host3'],
 414             []
 415         ),
 416         # all_hosts + count_per_host
 417         NodeAssignmentTest(
 418             'mds',
 419             PlacementSpec(host_pattern='*', count_per_host=2),
 420             'host1 host2 host3'.split(),
 421             [
 422                 DaemonDescription('mds', 'a', 'host1'),
 423                 DaemonDescription('mds', 'b', 'host2'),
 424             ],
 425             None, None,
 426             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 427             ['mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 428             []
 429         ),
 430         # count that is bigger than the amount of hosts. Truncate to len(hosts)
 431         # mgr should not be co-located to each other.
 432         NodeAssignmentTest(
 433             'mgr',
 434             PlacementSpec(count=4),
 435             'host1 host2 host3'.split(),
 436             [],
 437             None, None,
 438             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 439             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 440             []
 441         ),
 442         # count that is bigger than the amount of hosts; wrap around.
 443         NodeAssignmentTest(
 444             'mds',
 445             PlacementSpec(count=6),
 446             'host1 host2 host3'.split(),
 447             [],
 448             None, None,
 449             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 450             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 451             []
 452         ),
 453         # count + partial host list
 454         NodeAssignmentTest(
 455             'mgr',
 456             PlacementSpec(count=3, hosts=['host3']),
 457             'host1 host2 host3'.split(),
 458             [
 459                 DaemonDescription('mgr', 'a', 'host1'),
 460                 DaemonDescription('mgr', 'b', 'host2'),
 461             ],
 462             None, None,
 463             ['mgr:host3'],
 464             ['mgr:host3'],
 465             ['mgr.a', 'mgr.b']
 466         ),
 467         # count + partial host list (with colo)
 468         NodeAssignmentTest(
 469             'mds',
 470             PlacementSpec(count=3, hosts=['host3']),
 471             'host1 host2 host3'.split(),
 472             [
 473                 DaemonDescription('mds', 'a', 'host1'),
 474                 DaemonDescription('mds', 'b', 'host2'),
 475             ],
 476             None, None,
 477             ['mds:host3', 'mds:host3', 'mds:host3'],
 478             ['mds:host3', 'mds:host3', 'mds:host3'],
 479             ['mds.a', 'mds.b']
 480         ),
 481         # count 1 + partial host list
 482         NodeAssignmentTest(
 483             'mgr',
 484             PlacementSpec(count=1, hosts=['host3']),
 485             'host1 host2 host3'.split(),
 486             [
 487                 DaemonDescription('mgr', 'a', 'host1'),
 488                 DaemonDescription('mgr', 'b', 'host2'),
 489             ],
 490             None, None,
 491             ['mgr:host3'],
 492             ['mgr:host3'],
 493             ['mgr.a', 'mgr.b']
 494         ),
 495         # count + partial host list + existing
 496         NodeAssignmentTest(
 497             'mgr',
 498             PlacementSpec(count=2, hosts=['host3']),
 499             'host1 host2 host3'.split(),
 500             [
 501                 DaemonDescription('mgr', 'a', 'host1'),
 502             ],
 503             None, None,
 504             ['mgr:host3'],
 505             ['mgr:host3'],
 506             ['mgr.a']
 507         ),
 508         # count + partial host list + existing (deterministic)
 509         NodeAssignmentTest(
 510             'mgr',
 511             PlacementSpec(count=2, hosts=['host1']),
 512             'host1 host2'.split(),
 513             [
 514                 DaemonDescription('mgr', 'a', 'host1'),
 515             ],
 516             None, None,
 517             ['mgr:host1'],
 518             [],
 519             []
 520         ),
 521         # count + partial host list + existing (deterministic)
 522         NodeAssignmentTest(
 523             'mgr',
 524             PlacementSpec(count=2, hosts=['host1']),
 525             'host1 host2'.split(),
 526             [
 527                 DaemonDescription('mgr', 'a', 'host2'),
 528             ],
 529             None, None,
 530             ['mgr:host1'],
 531             ['mgr:host1'],
 532             ['mgr.a']
 533         ),
 534         # label only
 535         NodeAssignmentTest(
 536             'mgr',
 537             PlacementSpec(label='foo'),
 538             'host1 host2 host3'.split(),
 539             [],
 540             None, None,
 541             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 542             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 543             []
 544         ),
 545         # label + count (truncate to host list)
 546         NodeAssignmentTest(
 547             'mgr',
 548             PlacementSpec(count=4, label='foo'),
 549             'host1 host2 host3'.split(),
 550             [],
 551             None, None,
 552             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 553             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 554             []
 555         ),
 556         # label + count (with colo)
 557         NodeAssignmentTest(
 558             'mds',
 559             PlacementSpec(count=6, label='foo'),
 560             'host1 host2 host3'.split(),
 561             [],
 562             None, None,
 563             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 564             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 565             []
 566         ),
 567         # label only + count_per_hst
 568         NodeAssignmentTest(
 569             'mds',
 570             PlacementSpec(label='foo', count_per_host=3),
 571             'host1 host2 host3'.split(),
 572             [],
 573             None, None,
 574             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3',
 575              'mds:host1', 'mds:host2', 'mds:host3'],
 576             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3',
 577              'mds:host1', 'mds:host2', 'mds:host3'],
 578             []
 579         ),
 580         # host_pattern
 581         NodeAssignmentTest(
 582             'mgr',
 583             PlacementSpec(host_pattern='mgr*'),
 584             'mgrhost1 mgrhost2 datahost'.split(),
 585             [],
 586             None, None,
 587             ['mgr:mgrhost1', 'mgr:mgrhost2'],
 588             ['mgr:mgrhost1', 'mgr:mgrhost2'],
 589             []
 590         ),
 591         # host_pattern + count_per_host
 592         NodeAssignmentTest(
 593             'mds',
 594             PlacementSpec(host_pattern='mds*', count_per_host=3),
 595             'mdshost1 mdshost2 datahost'.split(),
 596             [],
 597             None, None,
 598             ['mds:mdshost1', 'mds:mdshost2', 'mds:mdshost1', 'mds:mdshost2', 'mds:mdshost1', 'mds:mdshost2'],
 599             ['mds:mdshost1', 'mds:mdshost2', 'mds:mdshost1', 'mds:mdshost2', 'mds:mdshost1', 'mds:mdshost2'],
 600             []
 601         ),
 602         # label + count_per_host + ports
 603         NodeAssignmentTest(
 604             'rgw',
 605             PlacementSpec(count=6, label='foo'),
 606             'host1 host2 host3'.split(),
 607             [],
 608             None, None,
 609             ['rgw:host1(*:80)', 'rgw:host2(*:80)', 'rgw:host3(*:80)',
 610              'rgw:host1(*:81)', 'rgw:host2(*:81)', 'rgw:host3(*:81)'],
 611             ['rgw:host1(*:80)', 'rgw:host2(*:80)', 'rgw:host3(*:80)',
 612              'rgw:host1(*:81)', 'rgw:host2(*:81)', 'rgw:host3(*:81)'],
 613             []
 614         ),
 615         # label + count_per_host + ports (+ xisting)
 616         NodeAssignmentTest(
 617             'rgw',
 618             PlacementSpec(count=6, label='foo'),
 619             'host1 host2 host3'.split(),
 620             [
 621                 DaemonDescription('rgw', 'a', 'host1', ports=[81]),
 622                 DaemonDescription('rgw', 'b', 'host2', ports=[80]),
 623                 DaemonDescription('rgw', 'c', 'host1', ports=[82]),
 624             ],
 625             None, None,
 626             ['rgw:host1(*:80)', 'rgw:host2(*:80)', 'rgw:host3(*:80)',
 627              'rgw:host1(*:81)', 'rgw:host2(*:81)', 'rgw:host3(*:81)'],
 628             ['rgw:host1(*:80)', 'rgw:host3(*:80)',
 629              'rgw:host2(*:81)', 'rgw:host3(*:81)'],
 630             ['rgw.c']
 631         ),
 632         # cephadm.py teuth case
 633         NodeAssignmentTest(
 634             'mgr',
 635             PlacementSpec(count=3, hosts=['host1=y', 'host2=x']),
 636             'host1 host2'.split(),
 637             [
 638                 DaemonDescription('mgr', 'y', 'host1'),
 639                 DaemonDescription('mgr', 'x', 'host2'),
 640             ],
 641             None, None,
 642             ['mgr:host1(name=y)', 'mgr:host2(name=x)'],
 643             [], []
 644         ),
 645
 646         # note: host -> rank mapping is psuedo-random based on svc name, so these
 647         # host/rank pairs may seem random but they match the nfs.mynfs seed used by
 648         # the test.
 649
 650         # ranked, fresh
 651         NodeAssignmentTest(
 652             'nfs',
 653             PlacementSpec(count=3),
 654             'host1 host2 host3'.split(),
 655             [],
 656             {},
 657             {0: {0: None}, 1: {0: None}, 2: {0: None}},
 658             ['nfs:host1(rank=0.0)', 'nfs:host2(rank=1.0)', 'nfs:host3(rank=2.0)'],
 659             ['nfs:host1(rank=0.0)', 'nfs:host2(rank=1.0)', 'nfs:host3(rank=2.0)'],
 660             []
 661         ),
 662         # 21: ranked, exist
 663         NodeAssignmentTest(
 664             'nfs',
 665             PlacementSpec(count=3),
 666             'host1 host2 host3'.split(),
 667             [
 668                 DaemonDescription('nfs', '0.1', 'host1', rank=0, rank_generation=1),
 669             ],
 670             {0: {1: '0.1'}},
 671             {0: {1: '0.1'}, 1: {0: None}, 2: {0: None}},
 672             ['nfs:host1(rank=0.1)', 'nfs:host2(rank=1.0)', 'nfs:host3(rank=2.0)'],
 673             ['nfs:host2(rank=1.0)', 'nfs:host3(rank=2.0)'],
 674             []
 675         ),
 676         # ranked, exist, different ranks
 677         NodeAssignmentTest(
 678             'nfs',
 679             PlacementSpec(count=3),
 680             'host1 host2 host3'.split(),
 681             [
 682                 DaemonDescription('nfs', '0.1', 'host1', rank=0, rank_generation=1),
 683                 DaemonDescription('nfs', '1.1', 'host2', rank=1, rank_generation=1),
 684             ],
 685             {0: {1: '0.1'}, 1: {1: '1.1'}},
 686             {0: {1: '0.1'}, 1: {1: '1.1'}, 2: {0: None}},
 687             ['nfs:host1(rank=0.1)', 'nfs:host2(rank=1.1)', 'nfs:host3(rank=2.0)'],
 688             ['nfs:host3(rank=2.0)'],
 689             []
 690         ),
 691         # ranked, exist, different ranks (2)
 692         NodeAssignmentTest(
 693             'nfs',
 694             PlacementSpec(count=3),
 695             'host1 host2 host3'.split(),
 696             [
 697                 DaemonDescription('nfs', '0.1', 'host1', rank=0, rank_generation=1),
 698                 DaemonDescription('nfs', '1.1', 'host3', rank=1, rank_generation=1),
 699             ],
 700             {0: {1: '0.1'}, 1: {1: '1.1'}},
 701             {0: {1: '0.1'}, 1: {1: '1.1'}, 2: {0: None}},
 702             ['nfs:host1(rank=0.1)', 'nfs:host3(rank=1.1)', 'nfs:host2(rank=2.0)'],
 703             ['nfs:host2(rank=2.0)'],
 704             []
 705         ),
 706         # ranked, exist, extra ranks
 707         NodeAssignmentTest(
 708             'nfs',
 709             PlacementSpec(count=3),
 710             'host1 host2 host3'.split(),
 711             [
 712                 DaemonDescription('nfs', '0.5', 'host1', rank=0, rank_generation=5),
 713                 DaemonDescription('nfs', '1.5', 'host2', rank=1, rank_generation=5),
 714                 DaemonDescription('nfs', '4.5', 'host2', rank=4, rank_generation=5),
 715             ],
 716             {0: {5: '0.5'}, 1: {5: '1.5'}},
 717             {0: {5: '0.5'}, 1: {5: '1.5'}, 2: {0: None}},
 718             ['nfs:host1(rank=0.5)', 'nfs:host2(rank=1.5)', 'nfs:host3(rank=2.0)'],
 719             ['nfs:host3(rank=2.0)'],
 720             ['nfs.4.5']
 721         ),
 722         # 25: ranked, exist, extra ranks (scale down: kill off high rank)
 723         NodeAssignmentTest(
 724             'nfs',
 725             PlacementSpec(count=2),
 726             'host3 host2 host1'.split(),
 727             [
 728                 DaemonDescription('nfs', '0.5', 'host1', rank=0, rank_generation=5),
 729                 DaemonDescription('nfs', '1.5', 'host2', rank=1, rank_generation=5),
 730                 DaemonDescription('nfs', '2.5', 'host3', rank=2, rank_generation=5),
 731             ],
 732             {0: {5: '0.5'}, 1: {5: '1.5'}, 2: {5: '2.5'}},
 733             {0: {5: '0.5'}, 1: {5: '1.5'}, 2: {5: '2.5'}},
 734             ['nfs:host1(rank=0.5)', 'nfs:host2(rank=1.5)'],
 735             [],
 736             ['nfs.2.5']
 737         ),
 738         # ranked, exist, extra ranks (scale down hosts)
 739         NodeAssignmentTest(
 740             'nfs',
 741             PlacementSpec(count=2),
 742             'host1 host3'.split(),
 743             [
 744                 DaemonDescription('nfs', '0.5', 'host1', rank=0, rank_generation=5),
 745                 DaemonDescription('nfs', '1.5', 'host2', rank=1, rank_generation=5),
 746                 DaemonDescription('nfs', '2.5', 'host3', rank=4, rank_generation=5),
 747             ],
 748             {0: {5: '0.5'}, 1: {5: '1.5'}, 2: {5: '2.5'}},
 749             {0: {5: '0.5'}, 1: {5: '1.5', 6: None}, 2: {5: '2.5'}},
 750             ['nfs:host1(rank=0.5)', 'nfs:host3(rank=1.6)'],
 751             ['nfs:host3(rank=1.6)'],
 752             ['nfs.2.5', 'nfs.1.5']
 753         ),
 754         # ranked, exist, duplicate rank
 755         NodeAssignmentTest(
 756             'nfs',
 757             PlacementSpec(count=3),
 758             'host1 host2 host3'.split(),
 759             [
 760                 DaemonDescription('nfs', '0.0', 'host1', rank=0, rank_generation=0),
 761                 DaemonDescription('nfs', '1.1', 'host2', rank=1, rank_generation=1),
 762                 DaemonDescription('nfs', '1.2', 'host3', rank=1, rank_generation=2),
 763             ],
 764             {0: {0: '0.0'}, 1: {2: '1.2'}},
 765             {0: {0: '0.0'}, 1: {2: '1.2'}, 2: {0: None}},
 766             ['nfs:host1(rank=0.0)', 'nfs:host3(rank=1.2)', 'nfs:host2(rank=2.0)'],
 767             ['nfs:host2(rank=2.0)'],
 768             ['nfs.1.1']
 769         ),
 770         # 28: ranked, all gens stale (failure during update cycle)
 771         NodeAssignmentTest(
 772             'nfs',
 773             PlacementSpec(count=2),
 774             'host1 host2 host3'.split(),
 775             [
 776                 DaemonDescription('nfs', '0.2', 'host1', rank=0, rank_generation=2),
 777                 DaemonDescription('nfs', '1.2', 'host2', rank=1, rank_generation=2),
 778             ],
 779             {0: {2: '0.2'}, 1: {2: '1.2', 3: '1.3'}},
 780             {0: {2: '0.2'}, 1: {2: '1.2', 3: '1.3', 4: None}},
 781             ['nfs:host1(rank=0.2)', 'nfs:host2(rank=1.4)'],
 782             ['nfs:host2(rank=1.4)'],
 783             ['nfs.1.2']
 784         ),
 785         # ranked, not enough hosts
 786         NodeAssignmentTest(
 787             'nfs',
 788             PlacementSpec(count=4),
 789             'host1 host2 host3'.split(),
 790             [
 791                 DaemonDescription('nfs', '0.2', 'host1', rank=0, rank_generation=2),
 792                 DaemonDescription('nfs', '1.2', 'host2', rank=1, rank_generation=2),
 793             ],
 794             {0: {2: '0.2'}, 1: {2: '1.2'}},
 795             {0: {2: '0.2'}, 1: {2: '1.2'}, 2: {0: None}},
 796             ['nfs:host1(rank=0.2)', 'nfs:host2(rank=1.2)', 'nfs:host3(rank=2.0)'],
 797             ['nfs:host3(rank=2.0)'],
 798             []
 799         ),
 800         # ranked, scale down
 801         NodeAssignmentTest(
 802             'nfs',
 803             PlacementSpec(hosts=['host2']),
 804             'host1 host2'.split(),
 805             [
 806                 DaemonDescription('nfs', '0.2', 'host1', rank=0, rank_generation=2),
 807                 DaemonDescription('nfs', '1.2', 'host2', rank=1, rank_generation=2),
 808                 DaemonDescription('nfs', '2.2', 'host3', rank=2, rank_generation=2),
 809             ],
 810             {0: {2: '0.2'}, 1: {2: '1.2'}, 2: {2: '2.2'}},
 811             {0: {2: '0.2', 3: None}, 1: {2: '1.2'}, 2: {2: '2.2'}},
 812             ['nfs:host2(rank=0.3)'],
 813             ['nfs:host2(rank=0.3)'],
 814             ['nfs.0.2', 'nfs.1.2', 'nfs.2.2']
 815         ),
 816
 817     ])
 818 def test_node_assignment(service_type, placement, hosts, daemons, rank_map, post_rank_map,
 819                          expected, expected_add, expected_remove):
 820     spec = None
 821     service_id = None
 822     allow_colo = False
 823     if service_type == 'rgw':
 824         service_id = 'realm.zone'
 825         allow_colo = True
 826     elif service_type == 'mds':
 827         service_id = 'myfs'
 828         allow_colo = True
 829     elif service_type == 'nfs':
 830         service_id = 'mynfs'
 831         spec = ServiceSpec(service_type=service_type,
 832                            service_id=service_id,
 833                            placement=placement,
 834                            pool='foo')
 835
 836     if not spec:
 837         spec = ServiceSpec(service_type=service_type,
 838                            service_id=service_id,
 839                            placement=placement)
 840
 841     all_slots, to_add, to_remove = HostAssignment(
 842         spec=spec,
 843         hosts=[HostSpec(h, labels=['foo']) for h in hosts],
 844         unreachable_hosts=[],
 845         daemons=daemons,
 846         allow_colo=allow_colo,
 847         rank_map=rank_map,
 848     ).place()
 849
 850     assert rank_map == post_rank_map
 851
 852     got = [str(p) for p in all_slots]
 853     num_wildcard = 0
 854     for i in expected:
 855         if i == '*':
 856             num_wildcard += 1
 857         else:
 858             assert i in got
 859             got.remove(i)
 860     assert num_wildcard == len(got)
 861
 862     got = [str(p) for p in to_add]
 863     num_wildcard = 0
 864     for i in expected_add:
 865         if i == '*':
 866             num_wildcard += 1
 867         else:
 868             assert i in got
 869             got.remove(i)
 870     assert num_wildcard == len(got)
 871
 872     assert sorted([d.name() for d in to_remove]) == sorted(expected_remove)
 873
 874
 875 class NodeAssignmentTest2(NamedTuple):
 876     service_type: str
 877     placement: PlacementSpec
 878     hosts: List[str]
 879     daemons: List[DaemonDescription]
 880     expected_len: int
 881     in_set: List[str]
 882
 883
 884 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected_len,in_set",
 885     [   # noqa: E128
 886         # just count
 887         NodeAssignmentTest2(
 888             'mgr',
 889             PlacementSpec(count=1),
 890             'host1 host2 host3'.split(),
 891             [],
 892             1,
 893             ['host1', 'host2', 'host3'],
 894         ),
 895
 896         # hosts + (smaller) count
 897         NodeAssignmentTest2(
 898             'mgr',
 899             PlacementSpec(count=1, hosts='host1 host2'.split()),
 900             'host1 host2'.split(),
 901             [],
 902             1,
 903             ['host1', 'host2'],
 904         ),
 905         # hosts + (smaller) count, existing
 906         NodeAssignmentTest2(
 907             'mgr',
 908             PlacementSpec(count=1, hosts='host1 host2 host3'.split()),
 909             'host1 host2 host3'.split(),
 910             [DaemonDescription('mgr', 'mgr.a', 'host1')],
 911             1,
 912             ['host1', 'host2', 'host3'],
 913         ),
 914         # hosts + (smaller) count, (more) existing
 915         NodeAssignmentTest2(
 916             'mgr',
 917             PlacementSpec(count=1, hosts='host1 host2 host3'.split()),
 918             'host1 host2 host3'.split(),
 919             [
 920                 DaemonDescription('mgr', 'a', 'host1'),
 921                 DaemonDescription('mgr', 'b', 'host2'),
 922             ],
 923             1,
 924             ['host1', 'host2']
 925         ),
 926         # count + partial host list
 927         NodeAssignmentTest2(
 928             'mgr',
 929             PlacementSpec(count=2, hosts=['host3']),
 930             'host1 host2 host3'.split(),
 931             [],
 932             1,
 933             ['host1', 'host2', 'host3']
 934         ),
 935         # label + count
 936         NodeAssignmentTest2(
 937             'mgr',
 938             PlacementSpec(count=1, label='foo'),
 939             'host1 host2 host3'.split(),
 940             [],
 941             1,
 942             ['host1', 'host2', 'host3']
 943         ),
 944     ])
 945 def test_node_assignment2(service_type, placement, hosts,
 946                           daemons, expected_len, in_set):
 947     hosts, to_add, to_remove = HostAssignment(
 948         spec=ServiceSpec(service_type, placement=placement),
 949         hosts=[HostSpec(h, labels=['foo']) for h in hosts],
 950         unreachable_hosts=[],
 951         daemons=daemons,
 952     ).place()
 953     assert len(hosts) == expected_len
 954     for h in [h.hostname for h in hosts]:
 955         assert h in in_set
 956
 957
 958 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected_len,must_have",
 959     [   # noqa: E128
 960         # hosts + (smaller) count, (more) existing
 961         NodeAssignmentTest2(
 962             'mgr',
 963             PlacementSpec(count=3, hosts='host3'.split()),
 964             'host1 host2 host3'.split(),
 965             [],
 966             1,
 967             ['host3']
 968         ),
 969         # count + partial host list
 970         NodeAssignmentTest2(
 971             'mgr',
 972             PlacementSpec(count=2, hosts=['host3']),
 973             'host1 host2 host3'.split(),
 974             [],
 975             1,
 976             ['host3']
 977         ),
 978     ])
 979 def test_node_assignment3(service_type, placement, hosts,
 980                           daemons, expected_len, must_have):
 981     hosts, to_add, to_remove = HostAssignment(
 982         spec=ServiceSpec(service_type, placement=placement),
 983         hosts=[HostSpec(h) for h in hosts],
 984         unreachable_hosts=[],
 985         daemons=daemons,
 986     ).place()
 987     assert len(hosts) == expected_len
 988     for h in must_have:
 989         assert h in [h.hostname for h in hosts]
 990
 991
 992 class NodeAssignmentTest4(NamedTuple):
 993     spec: ServiceSpec
 994     networks: Dict[str, Dict[str, Dict[str, List[str]]]]
 995     daemons: List[DaemonDescription]
 996     expected: List[str]
 997     expected_add: List[str]
 998     expected_remove: List[DaemonDescription]
 999
1000
1001 @pytest.mark.parametrize("spec,networks,daemons,expected,expected_add,expected_remove",
1002     [   # noqa: E128
1003         NodeAssignmentTest4(
1004             ServiceSpec(
1005                 service_type='rgw',
1006                 service_id='foo',
1007                 placement=PlacementSpec(count=6, label='foo'),
1008                 networks=['10.0.0.0/8'],
1009             ),
1010             {
1011                 'host1': {'10.0.0.0/8': {'eth0': ['10.0.0.1']}},
1012                 'host2': {'10.0.0.0/8': {'eth0': ['10.0.0.2']}},
1013                 'host3': {'192.168.0.0/16': {'eth0': ['192.168.0.1']}},
1014             },
1015             [],
1016             ['rgw:host1(10.0.0.1:80)', 'rgw:host2(10.0.0.2:80)',
1017              'rgw:host1(10.0.0.1:81)', 'rgw:host2(10.0.0.2:81)',
1018              'rgw:host1(10.0.0.1:82)', 'rgw:host2(10.0.0.2:82)'],
1019             ['rgw:host1(10.0.0.1:80)', 'rgw:host2(10.0.0.2:80)',
1020              'rgw:host1(10.0.0.1:81)', 'rgw:host2(10.0.0.2:81)',
1021              'rgw:host1(10.0.0.1:82)', 'rgw:host2(10.0.0.2:82)'],
1022             []
1023         ),
1024         NodeAssignmentTest4(
1025             IngressSpec(
1026                 service_type='ingress',
1027                 service_id='rgw.foo',
1028                 frontend_port=443,
1029                 monitor_port=8888,
1030                 virtual_ip='10.0.0.20/8',
1031                 backend_service='rgw.foo',
1032                 placement=PlacementSpec(label='foo'),
1033                 networks=['10.0.0.0/8'],
1034             ),
1035             {
1036                 'host1': {'10.0.0.0/8': {'eth0': ['10.0.0.1']}},
1037                 'host2': {'10.0.0.0/8': {'eth1': ['10.0.0.2']}},
1038                 'host3': {'192.168.0.0/16': {'eth2': ['192.168.0.1']}},
1039             },
1040             [],
1041             ['haproxy:host1(10.0.0.1:443,8888)', 'haproxy:host2(10.0.0.2:443,8888)',
1042              'keepalived:host1', 'keepalived:host2'],
1043             ['haproxy:host1(10.0.0.1:443,8888)', 'haproxy:host2(10.0.0.2:443,8888)',
1044              'keepalived:host1', 'keepalived:host2'],
1045             []
1046         ),
1047         NodeAssignmentTest4(
1048             IngressSpec(
1049                 service_type='ingress',
1050                 service_id='rgw.foo',
1051                 frontend_port=443,
1052                 monitor_port=8888,
1053                 virtual_ip='10.0.0.20/8',
1054                 backend_service='rgw.foo',
1055                 placement=PlacementSpec(label='foo'),
1056                 networks=['10.0.0.0/8'],
1057             ),
1058             {
1059                 'host1': {'10.0.0.0/8': {'eth0': ['10.0.0.1']}},
1060                 'host2': {'10.0.0.0/8': {'eth1': ['10.0.0.2']}},
1061                 'host3': {'192.168.0.0/16': {'eth2': ['192.168.0.1']}},
1062             },
1063             [
1064                 DaemonDescription('haproxy', 'a', 'host1', ip='10.0.0.1',
1065                                   ports=[443, 8888]),
1066                 DaemonDescription('keepalived', 'b', 'host2'),
1067                 DaemonDescription('keepalived', 'c', 'host3'),
1068             ],
1069             ['haproxy:host1(10.0.0.1:443,8888)', 'haproxy:host2(10.0.0.2:443,8888)',
1070              'keepalived:host1', 'keepalived:host2'],
1071             ['haproxy:host2(10.0.0.2:443,8888)',
1072              'keepalived:host1'],
1073             ['keepalived.c']
1074         ),
1075     ])
1076 def test_node_assignment4(spec, networks, daemons,
1077                           expected, expected_add, expected_remove):
1078     all_slots, to_add, to_remove = HostAssignment(
1079         spec=spec,
1080         hosts=[HostSpec(h, labels=['foo']) for h in networks.keys()],
1081         unreachable_hosts=[],
1082         daemons=daemons,
1083         allow_colo=True,
1084         networks=networks,
1085         primary_daemon_type='haproxy' if spec.service_type == 'ingress' else spec.service_type,
1086         per_host_daemon_type='keepalived' if spec.service_type == 'ingress' else None,
1087     ).place()
1088
1089     got = [str(p) for p in all_slots]
1090     num_wildcard = 0
1091     for i in expected:
1092         if i == '*':
1093             num_wildcard += 1
1094         else:
1095             assert i in got
1096             got.remove(i)
1097     assert num_wildcard == len(got)
1098
1099     got = [str(p) for p in to_add]
1100     num_wildcard = 0
1101     for i in expected_add:
1102         if i == '*':
1103             num_wildcard += 1
1104         else:
1105             assert i in got
1106             got.remove(i)
1107     assert num_wildcard == len(got)
1108
1109     assert sorted([d.name() for d in to_remove]) == sorted(expected_remove)
1110
1111
1112 @pytest.mark.parametrize("placement",
1113     [   # noqa: E128
1114         ('1 *'),
1115         ('* label:foo'),
1116         ('* host1 host2'),
1117         ('hostname12hostname12hostname12hostname12hostname12hostname12hostname12'),  # > 63 chars
1118     ])
1119 def test_bad_placements(placement):
1120     try:
1121         PlacementSpec.from_string(placement.split(' '))
1122         assert False
1123     except SpecValidationError:
1124         pass
1125
1126
1127 class NodeAssignmentTestBadSpec(NamedTuple):
1128     service_type: str
1129     placement: PlacementSpec
1130     hosts: List[str]
1131     daemons: List[DaemonDescription]
1132     expected: str
1133
1134
1135 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected",
1136     [   # noqa: E128
1137         # unknown host
1138         NodeAssignmentTestBadSpec(
1139             'mgr',
1140             PlacementSpec(hosts=['unknownhost']),
1141             ['knownhost'],
1142             [],
1143             "Cannot place <ServiceSpec for service_name=mgr> on unknownhost: Unknown hosts"
1144         ),
1145         # unknown host pattern
1146         NodeAssignmentTestBadSpec(
1147             'mgr',
1148             PlacementSpec(host_pattern='unknownhost'),
1149             ['knownhost'],
1150             [],
1151             "Cannot place <ServiceSpec for service_name=mgr>: No matching hosts"
1152         ),
1153         # unknown label
1154         NodeAssignmentTestBadSpec(
1155             'mgr',
1156             PlacementSpec(label='unknownlabel'),
1157             [],
1158             [],
1159             "Cannot place <ServiceSpec for service_name=mgr>: No matching hosts for label unknownlabel"
1160         ),
1161     ])
1162 def test_bad_specs(service_type, placement, hosts, daemons, expected):
1163     with pytest.raises(OrchestratorValidationError) as e:
1164         hosts, to_add, to_remove = HostAssignment(
1165             spec=ServiceSpec(service_type, placement=placement),
1166             hosts=[HostSpec(h) for h in hosts],
1167             unreachable_hosts=[],
1168             daemons=daemons,
1169         ).place()
1170     assert str(e.value) == expected
1171
1172
1173 class ActiveAssignmentTest(NamedTuple):
1174     service_type: str
1175     placement: PlacementSpec
1176     hosts: List[str]
1177     daemons: List[DaemonDescription]
1178     expected: List[List[str]]
1179     expected_add: List[List[str]]
1180     expected_remove: List[List[str]]
1181
1182
1183 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected,expected_add,expected_remove",
1184                          [
1185                              ActiveAssignmentTest(
1186                                  'mgr',
1187                                  PlacementSpec(count=2),
1188                                  'host1 host2 host3'.split(),
1189                                  [
1190                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1191                                      DaemonDescription('mgr', 'b', 'host2'),
1192                                      DaemonDescription('mgr', 'c', 'host3'),
1193                                  ],
1194                                  [['host1', 'host2'], ['host1', 'host3']],
1195                                  [[]],
1196                                  [['mgr.b'], ['mgr.c']]
1197                              ),
1198                              ActiveAssignmentTest(
1199                                  'mgr',
1200                                  PlacementSpec(count=2),
1201                                  'host1 host2 host3'.split(),
1202                                  [
1203                                      DaemonDescription('mgr', 'a', 'host1'),
1204                                      DaemonDescription('mgr', 'b', 'host2'),
1205                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1206                                  ],
1207                                  [['host1', 'host3'], ['host2', 'host3']],
1208                                  [[]],
1209                                  [['mgr.a'], ['mgr.b']]
1210                              ),
1211                              ActiveAssignmentTest(
1212                                  'mgr',
1213                                  PlacementSpec(count=1),
1214                                  'host1 host2 host3'.split(),
1215                                  [
1216                                      DaemonDescription('mgr', 'a', 'host1'),
1217                                      DaemonDescription('mgr', 'b', 'host2', is_active=True),
1218                                      DaemonDescription('mgr', 'c', 'host3'),
1219                                  ],
1220                                  [['host2']],
1221                                  [[]],
1222                                  [['mgr.a', 'mgr.c']]
1223                              ),
1224                              ActiveAssignmentTest(
1225                                  'mgr',
1226                                  PlacementSpec(count=1),
1227                                  'host1 host2 host3'.split(),
1228                                  [
1229                                      DaemonDescription('mgr', 'a', 'host1'),
1230                                      DaemonDescription('mgr', 'b', 'host2'),
1231                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1232                                  ],
1233                                  [['host3']],
1234                                  [[]],
1235                                  [['mgr.a', 'mgr.b']]
1236                              ),
1237                              ActiveAssignmentTest(
1238                                  'mgr',
1239                                  PlacementSpec(count=1),
1240                                  'host1 host2 host3'.split(),
1241                                  [
1242                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1243                                      DaemonDescription('mgr', 'b', 'host2'),
1244                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1245                                  ],
1246                                  [['host1'], ['host3']],
1247                                  [[]],
1248                                  [['mgr.a', 'mgr.b'], ['mgr.b', 'mgr.c']]
1249                              ),
1250                              ActiveAssignmentTest(
1251                                  'mgr',
1252                                  PlacementSpec(count=2),
1253                                  'host1 host2 host3'.split(),
1254                                  [
1255                                      DaemonDescription('mgr', 'a', 'host1'),
1256                                      DaemonDescription('mgr', 'b', 'host2', is_active=True),
1257                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1258                                  ],
1259                                  [['host2', 'host3']],
1260                                  [[]],
1261                                  [['mgr.a']]
1262                              ),
1263                              ActiveAssignmentTest(
1264                                  'mgr',
1265                                  PlacementSpec(count=1),
1266                                  'host1 host2 host3'.split(),
1267                                  [
1268                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1269                                      DaemonDescription('mgr', 'b', 'host2', is_active=True),
1270                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1271                                  ],
1272                                  [['host1'], ['host2'], ['host3']],
1273                                  [[]],
1274                                  [['mgr.a', 'mgr.b'], ['mgr.b', 'mgr.c'], ['mgr.a', 'mgr.c']]
1275                              ),
1276                              ActiveAssignmentTest(
1277                                  'mgr',
1278                                  PlacementSpec(count=1),
1279                                  'host1 host2 host3'.split(),
1280                                  [
1281                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1282                                      DaemonDescription('mgr', 'a2', 'host1'),
1283                                      DaemonDescription('mgr', 'b', 'host2'),
1284                                      DaemonDescription('mgr', 'c', 'host3'),
1285                                  ],
1286                                  [['host1']],
1287                                  [[]],
1288                                  [['mgr.a2', 'mgr.b', 'mgr.c']]
1289                              ),
1290                              ActiveAssignmentTest(
1291                                  'mgr',
1292                                  PlacementSpec(count=1),
1293                                  'host1 host2 host3'.split(),
1294                                  [
1295                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1296                                      DaemonDescription('mgr', 'a2', 'host1', is_active=True),
1297                                      DaemonDescription('mgr', 'b', 'host2'),
1298                                      DaemonDescription('mgr', 'c', 'host3'),
1299                                  ],
1300                                  [['host1']],
1301                                  [[]],
1302                                  [['mgr.a', 'mgr.b', 'mgr.c'], ['mgr.a2', 'mgr.b', 'mgr.c']]
1303                              ),
1304                              ActiveAssignmentTest(
1305                                  'mgr',
1306                                  PlacementSpec(count=2),
1307                                  'host1 host2 host3'.split(),
1308                                  [
1309                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1310                                      DaemonDescription('mgr', 'a2', 'host1'),
1311                                      DaemonDescription('mgr', 'b', 'host2'),
1312                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1313                                  ],
1314                                  [['host1', 'host3']],
1315                                  [[]],
1316                                  [['mgr.a2', 'mgr.b']]
1317                              ),
1318                              # Explicit placement should override preference for active daemon
1319                              ActiveAssignmentTest(
1320                                  'mgr',
1321                                  PlacementSpec(count=1, hosts=['host1']),
1322                                  'host1 host2 host3'.split(),
1323                                  [
1324                                      DaemonDescription('mgr', 'a', 'host1'),
1325                                      DaemonDescription('mgr', 'b', 'host2'),
1326                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1327                                  ],
1328                                  [['host1']],
1329                                  [[]],
1330                                  [['mgr.b', 'mgr.c']]
1331                              ),
1332
1333                          ])
1334 def test_active_assignment(service_type, placement, hosts, daemons, expected, expected_add, expected_remove):
1335
1336     spec = ServiceSpec(service_type=service_type,
1337                        service_id=None,
1338                        placement=placement)
1339
1340     hosts, to_add, to_remove = HostAssignment(
1341         spec=spec,
1342         hosts=[HostSpec(h) for h in hosts],
1343         unreachable_hosts=[],
1344         daemons=daemons,
1345     ).place()
1346     assert sorted([h.hostname for h in hosts]) in expected
1347     assert sorted([h.hostname for h in to_add]) in expected_add
1348     assert sorted([h.name() for h in to_remove]) in expected_remove
1349
1350
1351 class UnreachableHostsTest(NamedTuple):
1352     service_type: str
1353     placement: PlacementSpec
1354     hosts: List[str]
1355     unreachables_hosts: List[str]
1356     daemons: List[DaemonDescription]
1357     expected_add: List[List[str]]
1358     expected_remove: List[List[str]]
1359
1360
1361 @pytest.mark.parametrize("service_type,placement,hosts,unreachable_hosts,daemons,expected_add,expected_remove",
1362                          [
1363                              UnreachableHostsTest(
1364                                  'mgr',
1365                                  PlacementSpec(count=3),
1366                                  'host1 host2 host3'.split(),
1367                                  ['host2'],
1368                                  [],
1369                                  [['host1', 'host3']],
1370                                  [[]],
1371                              ),
1372                              UnreachableHostsTest(
1373                                  'mgr',
1374                                  PlacementSpec(hosts=['host3']),
1375                                  'host1 host2 host3'.split(),
1376                                  ['host1'],
1377                                  [
1378                                      DaemonDescription('mgr', 'a', 'host1'),
1379                                      DaemonDescription('mgr', 'b', 'host2'),
1380                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1381                                  ],
1382                                  [[]],
1383                                  [['mgr.b']],
1384                              ),
1385                              UnreachableHostsTest(
1386                                  'mgr',
1387                                  PlacementSpec(count=3),
1388                                  'host1 host2 host3 host4'.split(),
1389                                  ['host1'],
1390                                  [
1391                                      DaemonDescription('mgr', 'a', 'host1'),
1392                                      DaemonDescription('mgr', 'b', 'host2'),
1393                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1394                                  ],
1395                                  [[]],
1396                                  [[]],
1397                              ),
1398                              UnreachableHostsTest(
1399                                  'mgr',
1400                                  PlacementSpec(count=1),
1401                                  'host1 host2 host3 host4'.split(),
1402                                  'host1 host3'.split(),
1403                                  [
1404                                      DaemonDescription('mgr', 'a', 'host1'),
1405                                      DaemonDescription('mgr', 'b', 'host2'),
1406                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1407                                  ],
1408                                  [[]],
1409                                  [['mgr.b']],
1410                              ),
1411                              UnreachableHostsTest(
1412                                  'mgr',
1413                                  PlacementSpec(count=3),
1414                                  'host1 host2 host3 host4'.split(),
1415                                  ['host2'],
1416                                  [],
1417                                  [['host1', 'host3', 'host4']],
1418                                  [[]],
1419                              ),
1420                              UnreachableHostsTest(
1421                                  'mgr',
1422                                  PlacementSpec(count=3),
1423                                  'host1 host2 host3 host4'.split(),
1424                                  'host1 host4'.split(),
1425                                  [],
1426                                  [['host2', 'host3']],
1427                                  [[]],
1428                              ),
1429
1430                          ])
1431 def test_unreachable_host(service_type, placement, hosts, unreachable_hosts, daemons, expected_add, expected_remove):
1432
1433     spec = ServiceSpec(service_type=service_type,
1434                        service_id=None,
1435                        placement=placement)
1436
1437     hosts, to_add, to_remove = HostAssignment(
1438         spec=spec,
1439         hosts=[HostSpec(h) for h in hosts],
1440         unreachable_hosts=[HostSpec(h) for h in unreachable_hosts],
1441         daemons=daemons,
1442     ).place()
1443     assert sorted([h.hostname for h in to_add]) in expected_add
1444     assert sorted([h.name() for h in to_remove]) in expected_remove