ceph/src/pybind/mgr/cephadm/tests/test_scheduling.py

   1 # Disable autopep8 for this file:
   2
   3 # fmt: off
   4
   5 from typing import NamedTuple, List, Dict, Optional
   6 import pytest
   7
   8 from ceph.deployment.hostspec import HostSpec
   9 from ceph.deployment.service_spec import ServiceSpec, PlacementSpec, IngressSpec
  10 from ceph.deployment.hostspec import SpecValidationError
  11
  12 from cephadm.module import HostAssignment
  13 from cephadm.schedule import DaemonPlacement
  14 from orchestrator import DaemonDescription, OrchestratorValidationError, OrchestratorError
  15
  16
  17 def wrapper(func):
  18     # some odd thingy to revert the order or arguments
  19     def inner(*args):
  20         def inner2(expected):
  21             func(expected, *args)
  22         return inner2
  23     return inner
  24
  25
  26 @wrapper
  27 def none(expected):
  28     assert expected == []
  29
  30
  31 @wrapper
  32 def one_of(expected, *hosts):
  33     if not isinstance(expected, list):
  34         assert False, str(expected)
  35     assert len(expected) == 1, f'one_of failed len({expected}) != 1'
  36     assert expected[0] in hosts
  37
  38
  39 @wrapper
  40 def two_of(expected, *hosts):
  41     if not isinstance(expected, list):
  42         assert False, str(expected)
  43     assert len(expected) == 2, f'one_of failed len({expected}) != 2'
  44     matches = 0
  45     for h in hosts:
  46         matches += int(h in expected)
  47     if matches != 2:
  48         assert False, f'two of {hosts} not in {expected}'
  49
  50
  51 @wrapper
  52 def exactly(expected, *hosts):
  53     assert expected == list(hosts)
  54
  55
  56 @wrapper
  57 def error(expected, kind, match):
  58     assert isinstance(expected, kind), (str(expected), match)
  59     assert str(expected) == match, (str(expected), match)
  60
  61
  62 @wrapper
  63 def _or(expected, *inners):
  64     def catch(inner):
  65         try:
  66             inner(expected)
  67         except AssertionError as e:
  68             return e
  69     result = [catch(i) for i in inners]
  70     if None not in result:
  71         assert False, f"_or failed: {expected}"
  72
  73
  74 def _always_true(_):
  75     pass
  76
  77
  78 def k(s):
  79     return [e for e in s.split(' ') if e]
  80
  81
  82 def get_result(key, results):
  83     def match(one):
  84         for o, k in zip(one, key):
  85             if o != k and o != '*':
  86                 return False
  87         return True
  88     return [v for k, v in results if match(k)][0]
  89
  90
  91 def mk_spec_and_host(spec_section, hosts, explicit_key, explicit, count):
  92
  93     if spec_section == 'hosts':
  94         mk_spec = lambda: ServiceSpec('mgr', placement=PlacementSpec(  # noqa: E731
  95             hosts=explicit,
  96             count=count,
  97         ))
  98     elif spec_section == 'label':
  99         mk_spec = lambda: ServiceSpec('mgr', placement=PlacementSpec(  # noqa: E731
 100             label='mylabel',
 101             count=count,
 102         ))
 103     elif spec_section == 'host_pattern':
 104         pattern = {
 105             'e': 'notfound',
 106             '1': '1',
 107             '12': '[1-2]',
 108             '123': '*',
 109         }[explicit_key]
 110         mk_spec = lambda: ServiceSpec('mgr', placement=PlacementSpec(  # noqa: E731
 111             host_pattern=pattern,
 112             count=count,
 113         ))
 114     else:
 115         assert False
 116
 117     hosts = [
 118         HostSpec(h, labels=['mylabel']) if h in explicit else HostSpec(h)
 119         for h in hosts
 120     ]
 121
 122     return mk_spec, hosts
 123
 124
 125 def run_scheduler_test(results, mk_spec, hosts, daemons, key_elems):
 126     key = ' '.join('N' if e is None else str(e) for e in key_elems)
 127     try:
 128         assert_res = get_result(k(key), results)
 129     except IndexError:
 130         try:
 131             spec = mk_spec()
 132             host_res, to_add, to_remove = HostAssignment(
 133                 spec=spec,
 134                 hosts=hosts,
 135                 daemons=daemons,
 136             ).place()
 137             if isinstance(host_res, list):
 138                 e = ', '.join(repr(h.hostname) for h in host_res)
 139                 assert False, f'`(k("{key}"), exactly({e})),` not found'
 140             assert False, f'`(k("{key}"), ...),` not found'
 141         except OrchestratorError as e:
 142             assert False, f'`(k("{key}"), error({type(e).__name__}, {repr(str(e))})),` not found'
 143
 144     for _ in range(10):  # scheduler has a random component
 145         try:
 146             spec = mk_spec()
 147             host_res, to_add, to_remove = HostAssignment(
 148                 spec=spec,
 149                 hosts=hosts,
 150                 daemons=daemons
 151             ).place()
 152
 153             assert_res(sorted([h.hostname for h in host_res]))
 154         except Exception as e:
 155             assert_res(e)
 156
 157
 158 # * first match from the top wins
 159 # * where e=[], *=any
 160 #
 161 #       + list of known hosts available for scheduling (host_key)
 162 #       |   + hosts used for explict placement (explicit_key)
 163 #       |   |   + count
 164 #       |   |   | + section (host, label, pattern)
 165 #       |   |   | |     + expected result
 166 #       |   |   | |     |
 167 test_explicit_scheduler_results = [
 168     (k("*   *   0 *"), error(SpecValidationError, 'num/count must be > 1')),
 169     (k("*   e   N l"), error(OrchestratorValidationError, 'Cannot place <ServiceSpec for service_name=mgr>: No matching hosts for label mylabel')),
 170     (k("*   e   N p"), error(OrchestratorValidationError, 'Cannot place <ServiceSpec for service_name=mgr>: No matching hosts')),
 171     (k("*   e   N h"), error(OrchestratorValidationError, 'placement spec is empty: no hosts, no label, no pattern, no count')),
 172     (k("*   e   * *"), none),
 173     (k("1   12  * h"), error(OrchestratorValidationError, "Cannot place <ServiceSpec for service_name=mgr> on 2: Unknown hosts")),
 174     (k("1   123 * h"), error(OrchestratorValidationError, "Cannot place <ServiceSpec for service_name=mgr> on 2, 3: Unknown hosts")),
 175     (k("1   *   * *"), exactly('1')),
 176     (k("12  1   * *"), exactly('1')),
 177     (k("12  12  1 *"), one_of('1', '2')),
 178     (k("12  12  * *"), exactly('1', '2')),
 179     (k("12  123 * h"), error(OrchestratorValidationError, "Cannot place <ServiceSpec for service_name=mgr> on 3: Unknown hosts")),
 180     (k("12  123 1 *"), one_of('1', '2', '3')),
 181     (k("12  123 * *"), two_of('1', '2', '3')),
 182     (k("123 1   * *"), exactly('1')),
 183     (k("123 12  1 *"), one_of('1', '2')),
 184     (k("123 12  * *"), exactly('1', '2')),
 185     (k("123 123 1 *"), one_of('1', '2', '3')),
 186     (k("123 123 2 *"), two_of('1', '2', '3')),
 187     (k("123 123 * *"), exactly('1', '2', '3')),
 188 ]
 189
 190
 191 @pytest.mark.parametrize("dp,n,result",
 192     [   # noqa: E128
 193         (
 194             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[80]),
 195             0,
 196             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[80]),
 197         ),
 198         (
 199             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[80]),
 200             2,
 201             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[82]),
 202         ),
 203         (
 204             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[80, 90]),
 205             2,
 206             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[82, 92]),
 207         ),
 208     ])
 209 def test_daemon_placement_renumber(dp, n, result):
 210     assert dp.renumber_ports(n) == result
 211
 212
 213 @pytest.mark.parametrize(
 214     'dp,dd,result',
 215     [
 216         (
 217             DaemonPlacement(daemon_type='mgr', hostname='host1'),
 218             DaemonDescription('mgr', 'a', 'host1'),
 219             True
 220         ),
 221         (
 222             DaemonPlacement(daemon_type='mgr', hostname='host1', name='a'),
 223             DaemonDescription('mgr', 'a', 'host1'),
 224             True
 225         ),
 226         (
 227             DaemonPlacement(daemon_type='mon', hostname='host1', name='a'),
 228             DaemonDescription('mgr', 'a', 'host1'),
 229             False
 230         ),
 231         (
 232             DaemonPlacement(daemon_type='mgr', hostname='host1', name='a'),
 233             DaemonDescription('mgr', 'b', 'host1'),
 234             False
 235         ),
 236     ])
 237 def test_daemon_placement_match(dp, dd, result):
 238     assert dp.matches_daemon(dd) == result
 239
 240
 241 @pytest.mark.parametrize("spec_section_key,spec_section",
 242     [   # noqa: E128
 243         ('h', 'hosts'),
 244         ('l', 'label'),
 245         ('p', 'host_pattern'),
 246     ])
 247 @pytest.mark.parametrize("count",
 248     [   # noqa: E128
 249         None,
 250         0,
 251         1,
 252         2,
 253         3,
 254     ])
 255 @pytest.mark.parametrize("explicit_key, explicit",
 256     [   # noqa: E128
 257         ('e', []),
 258         ('1', ['1']),
 259         ('12', ['1', '2']),
 260         ('123', ['1', '2', '3']),
 261     ])
 262 @pytest.mark.parametrize("host_key, hosts",
 263     [   # noqa: E128
 264         ('1', ['1']),
 265         ('12', ['1', '2']),
 266         ('123', ['1', '2', '3']),
 267     ])
 268 def test_explicit_scheduler(host_key, hosts,
 269                             explicit_key, explicit,
 270                             count,
 271                             spec_section_key, spec_section):
 272
 273     mk_spec, hosts = mk_spec_and_host(spec_section, hosts, explicit_key, explicit, count)
 274     run_scheduler_test(
 275         results=test_explicit_scheduler_results,
 276         mk_spec=mk_spec,
 277         hosts=hosts,
 278         daemons=[],
 279         key_elems=(host_key, explicit_key, count, spec_section_key)
 280     )
 281
 282
 283 # * first match from the top wins
 284 # * where e=[], *=any
 285 #
 286 #       + list of known hosts available for scheduling (host_key)
 287 #       |   + hosts used for explict placement (explicit_key)
 288 #       |   |   + count
 289 #       |   |   | + existing daemons
 290 #       |   |   | |     + section (host, label, pattern)
 291 #       |   |   | |     |   + expected result
 292 #       |   |   | |     |   |
 293 test_scheduler_daemons_results = [
 294     (k("*   1   * *   *"), exactly('1')),
 295     (k("1   123 * *   h"), error(OrchestratorValidationError, 'Cannot place <ServiceSpec for service_name=mgr> on 2, 3: Unknown hosts')),
 296     (k("1   123 * *   *"), exactly('1')),
 297     (k("12  123 * *   h"), error(OrchestratorValidationError, 'Cannot place <ServiceSpec for service_name=mgr> on 3: Unknown hosts')),
 298     (k("12  123 N *   *"), exactly('1', '2')),
 299     (k("12  123 1 *   *"), one_of('1', '2')),
 300     (k("12  123 2 *   *"), exactly('1', '2')),
 301     (k("12  123 3 *   *"), exactly('1', '2')),
 302     (k("123 123 N *   *"), exactly('1', '2', '3')),
 303     (k("123 123 1 e   *"), one_of('1', '2', '3')),
 304     (k("123 123 1 1   *"), exactly('1')),
 305     (k("123 123 1 3   *"), exactly('3')),
 306     (k("123 123 1 12  *"), one_of('1', '2')),
 307     (k("123 123 1 112 *"), one_of('1', '2')),
 308     (k("123 123 1 23  *"), one_of('2', '3')),
 309     (k("123 123 1 123 *"), one_of('1', '2', '3')),
 310     (k("123 123 2 e   *"), two_of('1', '2', '3')),
 311     (k("123 123 2 1   *"), _or(exactly('1', '2'), exactly('1', '3'))),
 312     (k("123 123 2 3   *"), _or(exactly('1', '3'), exactly('2', '3'))),
 313     (k("123 123 2 12  *"), exactly('1', '2')),
 314     (k("123 123 2 112 *"), exactly('1', '2')),
 315     (k("123 123 2 23  *"), exactly('2', '3')),
 316     (k("123 123 2 123 *"), two_of('1', '2', '3')),
 317     (k("123 123 3 *   *"), exactly('1', '2', '3')),
 318 ]
 319
 320
 321 @pytest.mark.parametrize("spec_section_key,spec_section",
 322     [   # noqa: E128
 323         ('h', 'hosts'),
 324         ('l', 'label'),
 325         ('p', 'host_pattern'),
 326     ])
 327 @pytest.mark.parametrize("daemons_key, daemons",
 328     [   # noqa: E128
 329         ('e', []),
 330         ('1', ['1']),
 331         ('3', ['3']),
 332         ('12', ['1', '2']),
 333         ('112', ['1', '1', '2']),  # deal with existing co-located daemons
 334         ('23', ['2', '3']),
 335         ('123', ['1', '2', '3']),
 336     ])
 337 @pytest.mark.parametrize("count",
 338     [   # noqa: E128
 339         None,
 340         1,
 341         2,
 342         3,
 343     ])
 344 @pytest.mark.parametrize("explicit_key, explicit",
 345     [   # noqa: E128
 346         ('1', ['1']),
 347         ('123', ['1', '2', '3']),
 348     ])
 349 @pytest.mark.parametrize("host_key, hosts",
 350     [   # noqa: E128
 351         ('1', ['1']),
 352         ('12', ['1', '2']),
 353         ('123', ['1', '2', '3']),
 354     ])
 355 def test_scheduler_daemons(host_key, hosts,
 356                            explicit_key, explicit,
 357                            count,
 358                            daemons_key, daemons,
 359                            spec_section_key, spec_section):
 360     mk_spec, hosts = mk_spec_and_host(spec_section, hosts, explicit_key, explicit, count)
 361     dds = [
 362         DaemonDescription('mgr', d, d)
 363         for d in daemons
 364     ]
 365     run_scheduler_test(
 366         results=test_scheduler_daemons_results,
 367         mk_spec=mk_spec,
 368         hosts=hosts,
 369         daemons=dds,
 370         key_elems=(host_key, explicit_key, count, daemons_key, spec_section_key)
 371     )
 372
 373
 374 # =========================
 375
 376
 377 class NodeAssignmentTest(NamedTuple):
 378     service_type: str
 379     placement: PlacementSpec
 380     hosts: List[str]
 381     daemons: List[DaemonDescription]
 382     rank_map: Optional[Dict[int, Dict[int, Optional[str]]]]
 383     post_rank_map: Optional[Dict[int, Dict[int, Optional[str]]]]
 384     expected: List[str]
 385     expected_add: List[str]
 386     expected_remove: List[DaemonDescription]
 387
 388
 389 @pytest.mark.parametrize("service_type,placement,hosts,daemons,rank_map,post_rank_map,expected,expected_add,expected_remove",
 390     [   # noqa: E128
 391         # just hosts
 392         NodeAssignmentTest(
 393             'mgr',
 394             PlacementSpec(hosts=['smithi060']),
 395             ['smithi060'],
 396             [],
 397             None, None,
 398             ['mgr:smithi060'], ['mgr:smithi060'], []
 399         ),
 400         # all_hosts
 401         NodeAssignmentTest(
 402             'mgr',
 403             PlacementSpec(host_pattern='*'),
 404             'host1 host2 host3'.split(),
 405             [
 406                 DaemonDescription('mgr', 'a', 'host1'),
 407                 DaemonDescription('mgr', 'b', 'host2'),
 408             ],
 409             None, None,
 410             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 411             ['mgr:host3'],
 412             []
 413         ),
 414         # all_hosts + count_per_host
 415         NodeAssignmentTest(
 416             'mds',
 417             PlacementSpec(host_pattern='*', count_per_host=2),
 418             'host1 host2 host3'.split(),
 419             [
 420                 DaemonDescription('mds', 'a', 'host1'),
 421                 DaemonDescription('mds', 'b', 'host2'),
 422             ],
 423             None, None,
 424             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 425             ['mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 426             []
 427         ),
 428         # count that is bigger than the amount of hosts. Truncate to len(hosts)
 429         # mgr should not be co-located to each other.
 430         NodeAssignmentTest(
 431             'mgr',
 432             PlacementSpec(count=4),
 433             'host1 host2 host3'.split(),
 434             [],
 435             None, None,
 436             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 437             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 438             []
 439         ),
 440         # count that is bigger than the amount of hosts; wrap around.
 441         NodeAssignmentTest(
 442             'mds',
 443             PlacementSpec(count=6),
 444             'host1 host2 host3'.split(),
 445             [],
 446             None, None,
 447             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 448             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 449             []
 450         ),
 451         # count + partial host list
 452         NodeAssignmentTest(
 453             'mgr',
 454             PlacementSpec(count=3, hosts=['host3']),
 455             'host1 host2 host3'.split(),
 456             [
 457                 DaemonDescription('mgr', 'a', 'host1'),
 458                 DaemonDescription('mgr', 'b', 'host2'),
 459             ],
 460             None, None,
 461             ['mgr:host3'],
 462             ['mgr:host3'],
 463             ['mgr.a', 'mgr.b']
 464         ),
 465         # count + partial host list (with colo)
 466         NodeAssignmentTest(
 467             'mds',
 468             PlacementSpec(count=3, hosts=['host3']),
 469             'host1 host2 host3'.split(),
 470             [
 471                 DaemonDescription('mds', 'a', 'host1'),
 472                 DaemonDescription('mds', 'b', 'host2'),
 473             ],
 474             None, None,
 475             ['mds:host3', 'mds:host3', 'mds:host3'],
 476             ['mds:host3', 'mds:host3', 'mds:host3'],
 477             ['mds.a', 'mds.b']
 478         ),
 479         # count 1 + partial host list
 480         NodeAssignmentTest(
 481             'mgr',
 482             PlacementSpec(count=1, hosts=['host3']),
 483             'host1 host2 host3'.split(),
 484             [
 485                 DaemonDescription('mgr', 'a', 'host1'),
 486                 DaemonDescription('mgr', 'b', 'host2'),
 487             ],
 488             None, None,
 489             ['mgr:host3'],
 490             ['mgr:host3'],
 491             ['mgr.a', 'mgr.b']
 492         ),
 493         # count + partial host list + existing
 494         NodeAssignmentTest(
 495             'mgr',
 496             PlacementSpec(count=2, hosts=['host3']),
 497             'host1 host2 host3'.split(),
 498             [
 499                 DaemonDescription('mgr', 'a', 'host1'),
 500             ],
 501             None, None,
 502             ['mgr:host3'],
 503             ['mgr:host3'],
 504             ['mgr.a']
 505         ),
 506         # count + partial host list + existing (deterministic)
 507         NodeAssignmentTest(
 508             'mgr',
 509             PlacementSpec(count=2, hosts=['host1']),
 510             'host1 host2'.split(),
 511             [
 512                 DaemonDescription('mgr', 'a', 'host1'),
 513             ],
 514             None, None,
 515             ['mgr:host1'],
 516             [],
 517             []
 518         ),
 519         # count + partial host list + existing (deterministic)
 520         NodeAssignmentTest(
 521             'mgr',
 522             PlacementSpec(count=2, hosts=['host1']),
 523             'host1 host2'.split(),
 524             [
 525                 DaemonDescription('mgr', 'a', 'host2'),
 526             ],
 527             None, None,
 528             ['mgr:host1'],
 529             ['mgr:host1'],
 530             ['mgr.a']
 531         ),
 532         # label only
 533         NodeAssignmentTest(
 534             'mgr',
 535             PlacementSpec(label='foo'),
 536             'host1 host2 host3'.split(),
 537             [],
 538             None, None,
 539             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 540             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 541             []
 542         ),
 543         # label + count (truncate to host list)
 544         NodeAssignmentTest(
 545             'mgr',
 546             PlacementSpec(count=4, label='foo'),
 547             'host1 host2 host3'.split(),
 548             [],
 549             None, None,
 550             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 551             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 552             []
 553         ),
 554         # label + count (with colo)
 555         NodeAssignmentTest(
 556             'mds',
 557             PlacementSpec(count=6, label='foo'),
 558             'host1 host2 host3'.split(),
 559             [],
 560             None, None,
 561             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 562             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 563             []
 564         ),
 565         # label only + count_per_hst
 566         NodeAssignmentTest(
 567             'mds',
 568             PlacementSpec(label='foo', count_per_host=3),
 569             'host1 host2 host3'.split(),
 570             [],
 571             None, None,
 572             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3',
 573              'mds:host1', 'mds:host2', 'mds:host3'],
 574             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3',
 575              'mds:host1', 'mds:host2', 'mds:host3'],
 576             []
 577         ),
 578         # host_pattern
 579         NodeAssignmentTest(
 580             'mgr',
 581             PlacementSpec(host_pattern='mgr*'),
 582             'mgrhost1 mgrhost2 datahost'.split(),
 583             [],
 584             None, None,
 585             ['mgr:mgrhost1', 'mgr:mgrhost2'],
 586             ['mgr:mgrhost1', 'mgr:mgrhost2'],
 587             []
 588         ),
 589         # host_pattern + count_per_host
 590         NodeAssignmentTest(
 591             'mds',
 592             PlacementSpec(host_pattern='mds*', count_per_host=3),
 593             'mdshost1 mdshost2 datahost'.split(),
 594             [],
 595             None, None,
 596             ['mds:mdshost1', 'mds:mdshost2', 'mds:mdshost1', 'mds:mdshost2', 'mds:mdshost1', 'mds:mdshost2'],
 597             ['mds:mdshost1', 'mds:mdshost2', 'mds:mdshost1', 'mds:mdshost2', 'mds:mdshost1', 'mds:mdshost2'],
 598             []
 599         ),
 600         # label + count_per_host + ports
 601         NodeAssignmentTest(
 602             'rgw',
 603             PlacementSpec(count=6, label='foo'),
 604             'host1 host2 host3'.split(),
 605             [],
 606             None, None,
 607             ['rgw:host1(*:80)', 'rgw:host2(*:80)', 'rgw:host3(*:80)',
 608              'rgw:host1(*:81)', 'rgw:host2(*:81)', 'rgw:host3(*:81)'],
 609             ['rgw:host1(*:80)', 'rgw:host2(*:80)', 'rgw:host3(*:80)',
 610              'rgw:host1(*:81)', 'rgw:host2(*:81)', 'rgw:host3(*:81)'],
 611             []
 612         ),
 613         # label + count_per_host + ports (+ xisting)
 614         NodeAssignmentTest(
 615             'rgw',
 616             PlacementSpec(count=6, label='foo'),
 617             'host1 host2 host3'.split(),
 618             [
 619                 DaemonDescription('rgw', 'a', 'host1', ports=[81]),
 620                 DaemonDescription('rgw', 'b', 'host2', ports=[80]),
 621                 DaemonDescription('rgw', 'c', 'host1', ports=[82]),
 622             ],
 623             None, None,
 624             ['rgw:host1(*:80)', 'rgw:host2(*:80)', 'rgw:host3(*:80)',
 625              'rgw:host1(*:81)', 'rgw:host2(*:81)', 'rgw:host3(*:81)'],
 626             ['rgw:host1(*:80)', 'rgw:host3(*:80)',
 627              'rgw:host2(*:81)', 'rgw:host3(*:81)'],
 628             ['rgw.c']
 629         ),
 630         # cephadm.py teuth case
 631         NodeAssignmentTest(
 632             'mgr',
 633             PlacementSpec(count=3, hosts=['host1=y', 'host2=x']),
 634             'host1 host2'.split(),
 635             [
 636                 DaemonDescription('mgr', 'y', 'host1'),
 637                 DaemonDescription('mgr', 'x', 'host2'),
 638             ],
 639             None, None,
 640             ['mgr:host1(name=y)', 'mgr:host2(name=x)'],
 641             [], []
 642         ),
 643
 644         # note: host -> rank mapping is psuedo-random based on svc name, so these
 645         # host/rank pairs may seem random but they match the nfs.mynfs seed used by
 646         # the test.
 647
 648         # ranked, fresh
 649         NodeAssignmentTest(
 650             'nfs',
 651             PlacementSpec(count=3),
 652             'host1 host2 host3'.split(),
 653             [],
 654             {},
 655             {0: {0: None}, 1: {0: None}, 2: {0: None}},
 656             ['nfs:host1(rank=0.0)', 'nfs:host2(rank=1.0)', 'nfs:host3(rank=2.0)'],
 657             ['nfs:host1(rank=0.0)', 'nfs:host2(rank=1.0)', 'nfs:host3(rank=2.0)'],
 658             []
 659         ),
 660         # 21: ranked, exist
 661         NodeAssignmentTest(
 662             'nfs',
 663             PlacementSpec(count=3),
 664             'host1 host2 host3'.split(),
 665             [
 666                 DaemonDescription('nfs', '0.1', 'host1', rank=0, rank_generation=1),
 667             ],
 668             {0: {1: '0.1'}},
 669             {0: {1: '0.1'}, 1: {0: None}, 2: {0: None}},
 670             ['nfs:host1(rank=0.1)', 'nfs:host2(rank=1.0)', 'nfs:host3(rank=2.0)'],
 671             ['nfs:host2(rank=1.0)', 'nfs:host3(rank=2.0)'],
 672             []
 673         ),
 674         # ranked, exist, different ranks
 675         NodeAssignmentTest(
 676             'nfs',
 677             PlacementSpec(count=3),
 678             'host1 host2 host3'.split(),
 679             [
 680                 DaemonDescription('nfs', '0.1', 'host1', rank=0, rank_generation=1),
 681                 DaemonDescription('nfs', '1.1', 'host2', rank=1, rank_generation=1),
 682             ],
 683             {0: {1: '0.1'}, 1: {1: '1.1'}},
 684             {0: {1: '0.1'}, 1: {1: '1.1'}, 2: {0: None}},
 685             ['nfs:host1(rank=0.1)', 'nfs:host2(rank=1.1)', 'nfs:host3(rank=2.0)'],
 686             ['nfs:host3(rank=2.0)'],
 687             []
 688         ),
 689         # ranked, exist, different ranks (2)
 690         NodeAssignmentTest(
 691             'nfs',
 692             PlacementSpec(count=3),
 693             'host1 host2 host3'.split(),
 694             [
 695                 DaemonDescription('nfs', '0.1', 'host1', rank=0, rank_generation=1),
 696                 DaemonDescription('nfs', '1.1', 'host3', rank=1, rank_generation=1),
 697             ],
 698             {0: {1: '0.1'}, 1: {1: '1.1'}},
 699             {0: {1: '0.1'}, 1: {1: '1.1'}, 2: {0: None}},
 700             ['nfs:host1(rank=0.1)', 'nfs:host3(rank=1.1)', 'nfs:host2(rank=2.0)'],
 701             ['nfs:host2(rank=2.0)'],
 702             []
 703         ),
 704         # ranked, exist, extra ranks
 705         NodeAssignmentTest(
 706             'nfs',
 707             PlacementSpec(count=3),
 708             'host1 host2 host3'.split(),
 709             [
 710                 DaemonDescription('nfs', '0.5', 'host1', rank=0, rank_generation=5),
 711                 DaemonDescription('nfs', '1.5', 'host2', rank=1, rank_generation=5),
 712                 DaemonDescription('nfs', '4.5', 'host2', rank=4, rank_generation=5),
 713             ],
 714             {0: {5: '0.5'}, 1: {5: '1.5'}},
 715             {0: {5: '0.5'}, 1: {5: '1.5'}, 2: {0: None}},
 716             ['nfs:host1(rank=0.5)', 'nfs:host2(rank=1.5)', 'nfs:host3(rank=2.0)'],
 717             ['nfs:host3(rank=2.0)'],
 718             ['nfs.4.5']
 719         ),
 720         # 25: ranked, exist, extra ranks (scale down: kill off high rank)
 721         NodeAssignmentTest(
 722             'nfs',
 723             PlacementSpec(count=2),
 724             'host3 host2 host1'.split(),
 725             [
 726                 DaemonDescription('nfs', '0.5', 'host1', rank=0, rank_generation=5),
 727                 DaemonDescription('nfs', '1.5', 'host2', rank=1, rank_generation=5),
 728                 DaemonDescription('nfs', '2.5', 'host3', rank=2, rank_generation=5),
 729             ],
 730             {0: {5: '0.5'}, 1: {5: '1.5'}, 2: {5: '2.5'}},
 731             {0: {5: '0.5'}, 1: {5: '1.5'}, 2: {5: '2.5'}},
 732             ['nfs:host1(rank=0.5)', 'nfs:host2(rank=1.5)'],
 733             [],
 734             ['nfs.2.5']
 735         ),
 736         # ranked, exist, extra ranks (scale down hosts)
 737         NodeAssignmentTest(
 738             'nfs',
 739             PlacementSpec(count=2),
 740             'host1 host3'.split(),
 741             [
 742                 DaemonDescription('nfs', '0.5', 'host1', rank=0, rank_generation=5),
 743                 DaemonDescription('nfs', '1.5', 'host2', rank=1, rank_generation=5),
 744                 DaemonDescription('nfs', '2.5', 'host3', rank=4, rank_generation=5),
 745             ],
 746             {0: {5: '0.5'}, 1: {5: '1.5'}, 2: {5: '2.5'}},
 747             {0: {5: '0.5'}, 1: {5: '1.5', 6: None}, 2: {5: '2.5'}},
 748             ['nfs:host1(rank=0.5)', 'nfs:host3(rank=1.6)'],
 749             ['nfs:host3(rank=1.6)'],
 750             ['nfs.2.5', 'nfs.1.5']
 751         ),
 752         # ranked, exist, duplicate rank
 753         NodeAssignmentTest(
 754             'nfs',
 755             PlacementSpec(count=3),
 756             'host1 host2 host3'.split(),
 757             [
 758                 DaemonDescription('nfs', '0.0', 'host1', rank=0, rank_generation=0),
 759                 DaemonDescription('nfs', '1.1', 'host2', rank=1, rank_generation=1),
 760                 DaemonDescription('nfs', '1.2', 'host3', rank=1, rank_generation=2),
 761             ],
 762             {0: {0: '0.0'}, 1: {2: '1.2'}},
 763             {0: {0: '0.0'}, 1: {2: '1.2'}, 2: {0: None}},
 764             ['nfs:host1(rank=0.0)', 'nfs:host3(rank=1.2)', 'nfs:host2(rank=2.0)'],
 765             ['nfs:host2(rank=2.0)'],
 766             ['nfs.1.1']
 767         ),
 768         # 28: ranked, all gens stale (failure during update cycle)
 769         NodeAssignmentTest(
 770             'nfs',
 771             PlacementSpec(count=2),
 772             'host1 host2 host3'.split(),
 773             [
 774                 DaemonDescription('nfs', '0.2', 'host1', rank=0, rank_generation=2),
 775                 DaemonDescription('nfs', '1.2', 'host2', rank=1, rank_generation=2),
 776             ],
 777             {0: {2: '0.2'}, 1: {2: '1.2', 3: '1.3'}},
 778             {0: {2: '0.2'}, 1: {2: '1.2', 3: '1.3', 4: None}},
 779             ['nfs:host1(rank=0.2)', 'nfs:host2(rank=1.4)'],
 780             ['nfs:host2(rank=1.4)'],
 781             ['nfs.1.2']
 782         ),
 783         # ranked, not enough hosts
 784         NodeAssignmentTest(
 785             'nfs',
 786             PlacementSpec(count=4),
 787             'host1 host2 host3'.split(),
 788             [
 789                 DaemonDescription('nfs', '0.2', 'host1', rank=0, rank_generation=2),
 790                 DaemonDescription('nfs', '1.2', 'host2', rank=1, rank_generation=2),
 791             ],
 792             {0: {2: '0.2'}, 1: {2: '1.2'}},
 793             {0: {2: '0.2'}, 1: {2: '1.2'}, 2: {0: None}},
 794             ['nfs:host1(rank=0.2)', 'nfs:host2(rank=1.2)', 'nfs:host3(rank=2.0)'],
 795             ['nfs:host3(rank=2.0)'],
 796             []
 797         ),
 798         # ranked, scale down
 799         NodeAssignmentTest(
 800             'nfs',
 801             PlacementSpec(hosts=['host2']),
 802             'host1 host2'.split(),
 803             [
 804                 DaemonDescription('nfs', '0.2', 'host1', rank=0, rank_generation=2),
 805                 DaemonDescription('nfs', '1.2', 'host2', rank=1, rank_generation=2),
 806                 DaemonDescription('nfs', '2.2', 'host3', rank=2, rank_generation=2),
 807             ],
 808             {0: {2: '0.2'}, 1: {2: '1.2'}, 2: {2: '2.2'}},
 809             {0: {2: '0.2', 3: None}, 1: {2: '1.2'}, 2: {2: '2.2'}},
 810             ['nfs:host2(rank=0.3)'],
 811             ['nfs:host2(rank=0.3)'],
 812             ['nfs.0.2', 'nfs.1.2', 'nfs.2.2']
 813         ),
 814
 815     ])
 816 def test_node_assignment(service_type, placement, hosts, daemons, rank_map, post_rank_map,
 817                          expected, expected_add, expected_remove):
 818     spec = None
 819     service_id = None
 820     allow_colo = False
 821     if service_type == 'rgw':
 822         service_id = 'realm.zone'
 823         allow_colo = True
 824     elif service_type == 'mds':
 825         service_id = 'myfs'
 826         allow_colo = True
 827     elif service_type == 'nfs':
 828         service_id = 'mynfs'
 829         spec = ServiceSpec(service_type=service_type,
 830                            service_id=service_id,
 831                            placement=placement,
 832                            pool='foo')
 833
 834     if not spec:
 835         spec = ServiceSpec(service_type=service_type,
 836                            service_id=service_id,
 837                            placement=placement)
 838
 839     all_slots, to_add, to_remove = HostAssignment(
 840         spec=spec,
 841         hosts=[HostSpec(h, labels=['foo']) for h in hosts],
 842         daemons=daemons,
 843         allow_colo=allow_colo,
 844         rank_map=rank_map,
 845     ).place()
 846
 847     assert rank_map == post_rank_map
 848
 849     got = [str(p) for p in all_slots]
 850     num_wildcard = 0
 851     for i in expected:
 852         if i == '*':
 853             num_wildcard += 1
 854         else:
 855             assert i in got
 856             got.remove(i)
 857     assert num_wildcard == len(got)
 858
 859     got = [str(p) for p in to_add]
 860     num_wildcard = 0
 861     for i in expected_add:
 862         if i == '*':
 863             num_wildcard += 1
 864         else:
 865             assert i in got
 866             got.remove(i)
 867     assert num_wildcard == len(got)
 868
 869     assert sorted([d.name() for d in to_remove]) == sorted(expected_remove)
 870
 871
 872 class NodeAssignmentTest2(NamedTuple):
 873     service_type: str
 874     placement: PlacementSpec
 875     hosts: List[str]
 876     daemons: List[DaemonDescription]
 877     expected_len: int
 878     in_set: List[str]
 879
 880
 881 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected_len,in_set",
 882     [   # noqa: E128
 883         # just count
 884         NodeAssignmentTest2(
 885             'mgr',
 886             PlacementSpec(count=1),
 887             'host1 host2 host3'.split(),
 888             [],
 889             1,
 890             ['host1', 'host2', 'host3'],
 891         ),
 892
 893         # hosts + (smaller) count
 894         NodeAssignmentTest2(
 895             'mgr',
 896             PlacementSpec(count=1, hosts='host1 host2'.split()),
 897             'host1 host2'.split(),
 898             [],
 899             1,
 900             ['host1', 'host2'],
 901         ),
 902         # hosts + (smaller) count, existing
 903         NodeAssignmentTest2(
 904             'mgr',
 905             PlacementSpec(count=1, hosts='host1 host2 host3'.split()),
 906             'host1 host2 host3'.split(),
 907             [DaemonDescription('mgr', 'mgr.a', 'host1')],
 908             1,
 909             ['host1', 'host2', 'host3'],
 910         ),
 911         # hosts + (smaller) count, (more) existing
 912         NodeAssignmentTest2(
 913             'mgr',
 914             PlacementSpec(count=1, hosts='host1 host2 host3'.split()),
 915             'host1 host2 host3'.split(),
 916             [
 917                 DaemonDescription('mgr', 'a', 'host1'),
 918                 DaemonDescription('mgr', 'b', 'host2'),
 919             ],
 920             1,
 921             ['host1', 'host2']
 922         ),
 923         # count + partial host list
 924         NodeAssignmentTest2(
 925             'mgr',
 926             PlacementSpec(count=2, hosts=['host3']),
 927             'host1 host2 host3'.split(),
 928             [],
 929             1,
 930             ['host1', 'host2', 'host3']
 931         ),
 932         # label + count
 933         NodeAssignmentTest2(
 934             'mgr',
 935             PlacementSpec(count=1, label='foo'),
 936             'host1 host2 host3'.split(),
 937             [],
 938             1,
 939             ['host1', 'host2', 'host3']
 940         ),
 941     ])
 942 def test_node_assignment2(service_type, placement, hosts,
 943                           daemons, expected_len, in_set):
 944     hosts, to_add, to_remove = HostAssignment(
 945         spec=ServiceSpec(service_type, placement=placement),
 946         hosts=[HostSpec(h, labels=['foo']) for h in hosts],
 947         daemons=daemons,
 948     ).place()
 949     assert len(hosts) == expected_len
 950     for h in [h.hostname for h in hosts]:
 951         assert h in in_set
 952
 953
 954 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected_len,must_have",
 955     [   # noqa: E128
 956         # hosts + (smaller) count, (more) existing
 957         NodeAssignmentTest2(
 958             'mgr',
 959             PlacementSpec(count=3, hosts='host3'.split()),
 960             'host1 host2 host3'.split(),
 961             [],
 962             1,
 963             ['host3']
 964         ),
 965         # count + partial host list
 966         NodeAssignmentTest2(
 967             'mgr',
 968             PlacementSpec(count=2, hosts=['host3']),
 969             'host1 host2 host3'.split(),
 970             [],
 971             1,
 972             ['host3']
 973         ),
 974     ])
 975 def test_node_assignment3(service_type, placement, hosts,
 976                           daemons, expected_len, must_have):
 977     hosts, to_add, to_remove = HostAssignment(
 978         spec=ServiceSpec(service_type, placement=placement),
 979         hosts=[HostSpec(h) for h in hosts],
 980         daemons=daemons,
 981     ).place()
 982     assert len(hosts) == expected_len
 983     for h in must_have:
 984         assert h in [h.hostname for h in hosts]
 985
 986
 987 class NodeAssignmentTest4(NamedTuple):
 988     spec: ServiceSpec
 989     networks: Dict[str, Dict[str, Dict[str, List[str]]]]
 990     daemons: List[DaemonDescription]
 991     expected: List[str]
 992     expected_add: List[str]
 993     expected_remove: List[DaemonDescription]
 994
 995
 996 @pytest.mark.parametrize("spec,networks,daemons,expected,expected_add,expected_remove",
 997     [   # noqa: E128
 998         NodeAssignmentTest4(
 999             ServiceSpec(
1000                 service_type='rgw',
1001                 service_id='foo',
1002                 placement=PlacementSpec(count=6, label='foo'),
1003                 networks=['10.0.0.0/8'],
1004             ),
1005             {
1006                 'host1': {'10.0.0.0/8': {'eth0': ['10.0.0.1']}},
1007                 'host2': {'10.0.0.0/8': {'eth0': ['10.0.0.2']}},
1008                 'host3': {'192.168.0.0/16': {'eth0': ['192.168.0.1']}},
1009             },
1010             [],
1011             ['rgw:host1(10.0.0.1:80)', 'rgw:host2(10.0.0.2:80)',
1012              'rgw:host1(10.0.0.1:81)', 'rgw:host2(10.0.0.2:81)',
1013              'rgw:host1(10.0.0.1:82)', 'rgw:host2(10.0.0.2:82)'],
1014             ['rgw:host1(10.0.0.1:80)', 'rgw:host2(10.0.0.2:80)',
1015              'rgw:host1(10.0.0.1:81)', 'rgw:host2(10.0.0.2:81)',
1016              'rgw:host1(10.0.0.1:82)', 'rgw:host2(10.0.0.2:82)'],
1017             []
1018         ),
1019         NodeAssignmentTest4(
1020             IngressSpec(
1021                 service_type='ingress',
1022                 service_id='rgw.foo',
1023                 frontend_port=443,
1024                 monitor_port=8888,
1025                 virtual_ip='10.0.0.20/8',
1026                 backend_service='rgw.foo',
1027                 placement=PlacementSpec(label='foo'),
1028                 networks=['10.0.0.0/8'],
1029             ),
1030             {
1031                 'host1': {'10.0.0.0/8': {'eth0': ['10.0.0.1']}},
1032                 'host2': {'10.0.0.0/8': {'eth1': ['10.0.0.2']}},
1033                 'host3': {'192.168.0.0/16': {'eth2': ['192.168.0.1']}},
1034             },
1035             [],
1036             ['haproxy:host1(10.0.0.1:443,8888)', 'haproxy:host2(10.0.0.2:443,8888)',
1037              'keepalived:host1', 'keepalived:host2'],
1038             ['haproxy:host1(10.0.0.1:443,8888)', 'haproxy:host2(10.0.0.2:443,8888)',
1039              'keepalived:host1', 'keepalived:host2'],
1040             []
1041         ),
1042         NodeAssignmentTest4(
1043             IngressSpec(
1044                 service_type='ingress',
1045                 service_id='rgw.foo',
1046                 frontend_port=443,
1047                 monitor_port=8888,
1048                 virtual_ip='10.0.0.20/8',
1049                 backend_service='rgw.foo',
1050                 placement=PlacementSpec(label='foo'),
1051                 networks=['10.0.0.0/8'],
1052             ),
1053             {
1054                 'host1': {'10.0.0.0/8': {'eth0': ['10.0.0.1']}},
1055                 'host2': {'10.0.0.0/8': {'eth1': ['10.0.0.2']}},
1056                 'host3': {'192.168.0.0/16': {'eth2': ['192.168.0.1']}},
1057             },
1058             [
1059                 DaemonDescription('haproxy', 'a', 'host1', ip='10.0.0.1',
1060                                   ports=[443, 8888]),
1061                 DaemonDescription('keepalived', 'b', 'host2'),
1062                 DaemonDescription('keepalived', 'c', 'host3'),
1063             ],
1064             ['haproxy:host1(10.0.0.1:443,8888)', 'haproxy:host2(10.0.0.2:443,8888)',
1065              'keepalived:host1', 'keepalived:host2'],
1066             ['haproxy:host2(10.0.0.2:443,8888)',
1067              'keepalived:host1'],
1068             ['keepalived.c']
1069         ),
1070     ])
1071 def test_node_assignment4(spec, networks, daemons,
1072                           expected, expected_add, expected_remove):
1073     all_slots, to_add, to_remove = HostAssignment(
1074         spec=spec,
1075         hosts=[HostSpec(h, labels=['foo']) for h in networks.keys()],
1076         daemons=daemons,
1077         allow_colo=True,
1078         networks=networks,
1079         primary_daemon_type='haproxy' if spec.service_type == 'ingress' else spec.service_type,
1080         per_host_daemon_type='keepalived' if spec.service_type == 'ingress' else None,
1081     ).place()
1082
1083     got = [str(p) for p in all_slots]
1084     num_wildcard = 0
1085     for i in expected:
1086         if i == '*':
1087             num_wildcard += 1
1088         else:
1089             assert i in got
1090             got.remove(i)
1091     assert num_wildcard == len(got)
1092
1093     got = [str(p) for p in to_add]
1094     num_wildcard = 0
1095     for i in expected_add:
1096         if i == '*':
1097             num_wildcard += 1
1098         else:
1099             assert i in got
1100             got.remove(i)
1101     assert num_wildcard == len(got)
1102
1103     assert sorted([d.name() for d in to_remove]) == sorted(expected_remove)
1104
1105
1106 @pytest.mark.parametrize("placement",
1107     [   # noqa: E128
1108         ('1 *'),
1109         ('* label:foo'),
1110         ('* host1 host2'),
1111         ('hostname12hostname12hostname12hostname12hostname12hostname12hostname12'),  # > 63 chars
1112     ])
1113 def test_bad_placements(placement):
1114     try:
1115         PlacementSpec.from_string(placement.split(' '))
1116         assert False
1117     except SpecValidationError:
1118         pass
1119
1120
1121 class NodeAssignmentTestBadSpec(NamedTuple):
1122     service_type: str
1123     placement: PlacementSpec
1124     hosts: List[str]
1125     daemons: List[DaemonDescription]
1126     expected: str
1127
1128
1129 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected",
1130     [   # noqa: E128
1131         # unknown host
1132         NodeAssignmentTestBadSpec(
1133             'mgr',
1134             PlacementSpec(hosts=['unknownhost']),
1135             ['knownhost'],
1136             [],
1137             "Cannot place <ServiceSpec for service_name=mgr> on unknownhost: Unknown hosts"
1138         ),
1139         # unknown host pattern
1140         NodeAssignmentTestBadSpec(
1141             'mgr',
1142             PlacementSpec(host_pattern='unknownhost'),
1143             ['knownhost'],
1144             [],
1145             "Cannot place <ServiceSpec for service_name=mgr>: No matching hosts"
1146         ),
1147         # unknown label
1148         NodeAssignmentTestBadSpec(
1149             'mgr',
1150             PlacementSpec(label='unknownlabel'),
1151             [],
1152             [],
1153             "Cannot place <ServiceSpec for service_name=mgr>: No matching hosts for label unknownlabel"
1154         ),
1155     ])
1156 def test_bad_specs(service_type, placement, hosts, daemons, expected):
1157     with pytest.raises(OrchestratorValidationError) as e:
1158         hosts, to_add, to_remove = HostAssignment(
1159             spec=ServiceSpec(service_type, placement=placement),
1160             hosts=[HostSpec(h) for h in hosts],
1161             daemons=daemons,
1162         ).place()
1163     assert str(e.value) == expected
1164
1165
1166 class ActiveAssignmentTest(NamedTuple):
1167     service_type: str
1168     placement: PlacementSpec
1169     hosts: List[str]
1170     daemons: List[DaemonDescription]
1171     expected: List[List[str]]
1172     expected_add: List[List[str]]
1173     expected_remove: List[List[str]]
1174
1175
1176 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected,expected_add,expected_remove",
1177                          [
1178                              ActiveAssignmentTest(
1179                                  'mgr',
1180                                  PlacementSpec(count=2),
1181                                  'host1 host2 host3'.split(),
1182                                  [
1183                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1184                                      DaemonDescription('mgr', 'b', 'host2'),
1185                                      DaemonDescription('mgr', 'c', 'host3'),
1186                                  ],
1187                                  [['host1', 'host2'], ['host1', 'host3']],
1188                                  [[]],
1189                                  [['mgr.b'], ['mgr.c']]
1190                              ),
1191                              ActiveAssignmentTest(
1192                                  'mgr',
1193                                  PlacementSpec(count=2),
1194                                  'host1 host2 host3'.split(),
1195                                  [
1196                                      DaemonDescription('mgr', 'a', 'host1'),
1197                                      DaemonDescription('mgr', 'b', 'host2'),
1198                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1199                                  ],
1200                                  [['host1', 'host3'], ['host2', 'host3']],
1201                                  [[]],
1202                                  [['mgr.a'], ['mgr.b']]
1203                              ),
1204                              ActiveAssignmentTest(
1205                                  'mgr',
1206                                  PlacementSpec(count=1),
1207                                  'host1 host2 host3'.split(),
1208                                  [
1209                                      DaemonDescription('mgr', 'a', 'host1'),
1210                                      DaemonDescription('mgr', 'b', 'host2', is_active=True),
1211                                      DaemonDescription('mgr', 'c', 'host3'),
1212                                  ],
1213                                  [['host2']],
1214                                  [[]],
1215                                  [['mgr.a', 'mgr.c']]
1216                              ),
1217                              ActiveAssignmentTest(
1218                                  'mgr',
1219                                  PlacementSpec(count=1),
1220                                  'host1 host2 host3'.split(),
1221                                  [
1222                                      DaemonDescription('mgr', 'a', 'host1'),
1223                                      DaemonDescription('mgr', 'b', 'host2'),
1224                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1225                                  ],
1226                                  [['host3']],
1227                                  [[]],
1228                                  [['mgr.a', 'mgr.b']]
1229                              ),
1230                              ActiveAssignmentTest(
1231                                  'mgr',
1232                                  PlacementSpec(count=1),
1233                                  'host1 host2 host3'.split(),
1234                                  [
1235                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1236                                      DaemonDescription('mgr', 'b', 'host2'),
1237                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1238                                  ],
1239                                  [['host1'], ['host3']],
1240                                  [[]],
1241                                  [['mgr.a', 'mgr.b'], ['mgr.b', 'mgr.c']]
1242                              ),
1243                              ActiveAssignmentTest(
1244                                  'mgr',
1245                                  PlacementSpec(count=2),
1246                                  'host1 host2 host3'.split(),
1247                                  [
1248                                      DaemonDescription('mgr', 'a', 'host1'),
1249                                      DaemonDescription('mgr', 'b', 'host2', is_active=True),
1250                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1251                                  ],
1252                                  [['host2', 'host3']],
1253                                  [[]],
1254                                  [['mgr.a']]
1255                              ),
1256                              ActiveAssignmentTest(
1257                                  'mgr',
1258                                  PlacementSpec(count=1),
1259                                  'host1 host2 host3'.split(),
1260                                  [
1261                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1262                                      DaemonDescription('mgr', 'b', 'host2', is_active=True),
1263                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1264                                  ],
1265                                  [['host1'], ['host2'], ['host3']],
1266                                  [[]],
1267                                  [['mgr.a', 'mgr.b'], ['mgr.b', 'mgr.c'], ['mgr.a', 'mgr.c']]
1268                              ),
1269                              ActiveAssignmentTest(
1270                                  'mgr',
1271                                  PlacementSpec(count=1),
1272                                  'host1 host2 host3'.split(),
1273                                  [
1274                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1275                                      DaemonDescription('mgr', 'a2', 'host1'),
1276                                      DaemonDescription('mgr', 'b', 'host2'),
1277                                      DaemonDescription('mgr', 'c', 'host3'),
1278                                  ],
1279                                  [['host1']],
1280                                  [[]],
1281                                  [['mgr.a2', 'mgr.b', 'mgr.c']]
1282                              ),
1283                              ActiveAssignmentTest(
1284                                  'mgr',
1285                                  PlacementSpec(count=1),
1286                                  'host1 host2 host3'.split(),
1287                                  [
1288                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1289                                      DaemonDescription('mgr', 'a2', 'host1', is_active=True),
1290                                      DaemonDescription('mgr', 'b', 'host2'),
1291                                      DaemonDescription('mgr', 'c', 'host3'),
1292                                  ],
1293                                  [['host1']],
1294                                  [[]],
1295                                  [['mgr.a', 'mgr.b', 'mgr.c'], ['mgr.a2', 'mgr.b', 'mgr.c']]
1296                              ),
1297                              ActiveAssignmentTest(
1298                                  'mgr',
1299                                  PlacementSpec(count=2),
1300                                  'host1 host2 host3'.split(),
1301                                  [
1302                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1303                                      DaemonDescription('mgr', 'a2', 'host1'),
1304                                      DaemonDescription('mgr', 'b', 'host2'),
1305                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1306                                  ],
1307                                  [['host1', 'host3']],
1308                                  [[]],
1309                                  [['mgr.a2', 'mgr.b']]
1310                              ),
1311                              # Explicit placement should override preference for active daemon
1312                              ActiveAssignmentTest(
1313                                  'mgr',
1314                                  PlacementSpec(count=1, hosts=['host1']),
1315                                  'host1 host2 host3'.split(),
1316                                  [
1317                                      DaemonDescription('mgr', 'a', 'host1'),
1318                                      DaemonDescription('mgr', 'b', 'host2'),
1319                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1320                                  ],
1321                                  [['host1']],
1322                                  [[]],
1323                                  [['mgr.b', 'mgr.c']]
1324                              ),
1325
1326                          ])
1327 def test_active_assignment(service_type, placement, hosts, daemons, expected, expected_add, expected_remove):
1328
1329     spec = ServiceSpec(service_type=service_type,
1330                        service_id=None,
1331                        placement=placement)
1332
1333     hosts, to_add, to_remove = HostAssignment(
1334         spec=spec,
1335         hosts=[HostSpec(h) for h in hosts],
1336         daemons=daemons,
1337     ).place()
1338     assert sorted([h.hostname for h in hosts]) in expected
1339     assert sorted([h.hostname for h in to_add]) in expected_add
1340     assert sorted([h.name() for h in to_remove]) in expected_remove