ceph/src/pybind/mgr/cephadm/tests/test_scheduling.py

   1 # Disable autopep8 for this file:
   2
   3 # fmt: off
   4
   5 from typing import NamedTuple, List, Dict, Optional
   6 import pytest
   7
   8 from ceph.deployment.hostspec import HostSpec
   9 from ceph.deployment.service_spec import ServiceSpec, PlacementSpec, IngressSpec
  10 from ceph.deployment.hostspec import SpecValidationError
  11
  12 from cephadm.module import HostAssignment
  13 from cephadm.schedule import DaemonPlacement
  14 from orchestrator import DaemonDescription, OrchestratorValidationError, OrchestratorError
  15
  16
  17 def wrapper(func):
  18     # some odd thingy to revert the order or arguments
  19     def inner(*args):
  20         def inner2(expected):
  21             func(expected, *args)
  22         return inner2
  23     return inner
  24
  25
  26 @wrapper
  27 def none(expected):
  28     assert expected == []
  29
  30
  31 @wrapper
  32 def one_of(expected, *hosts):
  33     if not isinstance(expected, list):
  34         assert False, str(expected)
  35     assert len(expected) == 1, f'one_of failed len({expected}) != 1'
  36     assert expected[0] in hosts
  37
  38
  39 @wrapper
  40 def two_of(expected, *hosts):
  41     if not isinstance(expected, list):
  42         assert False, str(expected)
  43     assert len(expected) == 2, f'one_of failed len({expected}) != 2'
  44     matches = 0
  45     for h in hosts:
  46         matches += int(h in expected)
  47     if matches != 2:
  48         assert False, f'two of {hosts} not in {expected}'
  49
  50
  51 @wrapper
  52 def exactly(expected, *hosts):
  53     assert expected == list(hosts)
  54
  55
  56 @wrapper
  57 def error(expected, kind, match):
  58     assert isinstance(expected, kind), (str(expected), match)
  59     assert str(expected) == match, (str(expected), match)
  60
  61
  62 @wrapper
  63 def _or(expected, *inners):
  64     def catch(inner):
  65         try:
  66             inner(expected)
  67         except AssertionError as e:
  68             return e
  69     result = [catch(i) for i in inners]
  70     if None not in result:
  71         assert False, f"_or failed: {expected}"
  72
  73
  74 def _always_true(_):
  75     pass
  76
  77
  78 def k(s):
  79     return [e for e in s.split(' ') if e]
  80
  81
  82 def get_result(key, results):
  83     def match(one):
  84         for o, k in zip(one, key):
  85             if o != k and o != '*':
  86                 return False
  87         return True
  88     return [v for k, v in results if match(k)][0]
  89
  90
  91 def mk_spec_and_host(spec_section, hosts, explicit_key, explicit, count):
  92
  93     if spec_section == 'hosts':
  94         mk_spec = lambda: ServiceSpec('mgr', placement=PlacementSpec(  # noqa: E731
  95             hosts=explicit,
  96             count=count,
  97         ))
  98     elif spec_section == 'label':
  99         mk_spec = lambda: ServiceSpec('mgr', placement=PlacementSpec(  # noqa: E731
 100             label='mylabel',
 101             count=count,
 102         ))
 103     elif spec_section == 'host_pattern':
 104         pattern = {
 105             'e': 'notfound',
 106             '1': '1',
 107             '12': '[1-2]',
 108             '123': '*',
 109         }[explicit_key]
 110         mk_spec = lambda: ServiceSpec('mgr', placement=PlacementSpec(  # noqa: E731
 111             host_pattern=pattern,
 112             count=count,
 113         ))
 114     else:
 115         assert False
 116
 117     hosts = [
 118         HostSpec(h, labels=['mylabel']) if h in explicit else HostSpec(h)
 119         for h in hosts
 120     ]
 121
 122     return mk_spec, hosts
 123
 124
 125 def run_scheduler_test(results, mk_spec, hosts, daemons, key_elems):
 126     key = ' '.join('N' if e is None else str(e) for e in key_elems)
 127     try:
 128         assert_res = get_result(k(key), results)
 129     except IndexError:
 130         try:
 131             spec = mk_spec()
 132             host_res, to_add, to_remove = HostAssignment(
 133                 spec=spec,
 134                 hosts=hosts,
 135                 unreachable_hosts=[],
 136                 daemons=daemons,
 137             ).place()
 138             if isinstance(host_res, list):
 139                 e = ', '.join(repr(h.hostname) for h in host_res)
 140                 assert False, f'`(k("{key}"), exactly({e})),` not found'
 141             assert False, f'`(k("{key}"), ...),` not found'
 142         except OrchestratorError as e:
 143             assert False, f'`(k("{key}"), error({type(e).__name__}, {repr(str(e))})),` not found'
 144
 145     for _ in range(10):  # scheduler has a random component
 146         try:
 147             spec = mk_spec()
 148             host_res, to_add, to_remove = HostAssignment(
 149                 spec=spec,
 150                 hosts=hosts,
 151                 unreachable_hosts=[],
 152                 daemons=daemons
 153             ).place()
 154
 155             assert_res(sorted([h.hostname for h in host_res]))
 156         except Exception as e:
 157             assert_res(e)
 158
 159
 160 # * first match from the top wins
 161 # * where e=[], *=any
 162 #
 163 #       + list of known hosts available for scheduling (host_key)
 164 #       |   + hosts used for explict placement (explicit_key)
 165 #       |   |   + count
 166 #       |   |   | + section (host, label, pattern)
 167 #       |   |   | |     + expected result
 168 #       |   |   | |     |
 169 test_explicit_scheduler_results = [
 170     (k("*   *   0 *"), error(SpecValidationError, 'num/count must be >= 1')),
 171     (k("*   e   N l"), error(OrchestratorValidationError, 'Cannot place <ServiceSpec for service_name=mgr>: No matching hosts for label mylabel')),
 172     (k("*   e   N p"), error(OrchestratorValidationError, 'Cannot place <ServiceSpec for service_name=mgr>: No matching hosts')),
 173     (k("*   e   N h"), error(OrchestratorValidationError, 'placement spec is empty: no hosts, no label, no pattern, no count')),
 174     (k("*   e   * *"), none),
 175     (k("1   12  * h"), error(OrchestratorValidationError, "Cannot place <ServiceSpec for service_name=mgr> on 2: Unknown hosts")),
 176     (k("1   123 * h"), error(OrchestratorValidationError, "Cannot place <ServiceSpec for service_name=mgr> on 2, 3: Unknown hosts")),
 177     (k("1   *   * *"), exactly('1')),
 178     (k("12  1   * *"), exactly('1')),
 179     (k("12  12  1 *"), one_of('1', '2')),
 180     (k("12  12  * *"), exactly('1', '2')),
 181     (k("12  123 * h"), error(OrchestratorValidationError, "Cannot place <ServiceSpec for service_name=mgr> on 3: Unknown hosts")),
 182     (k("12  123 1 *"), one_of('1', '2', '3')),
 183     (k("12  123 * *"), two_of('1', '2', '3')),
 184     (k("123 1   * *"), exactly('1')),
 185     (k("123 12  1 *"), one_of('1', '2')),
 186     (k("123 12  * *"), exactly('1', '2')),
 187     (k("123 123 1 *"), one_of('1', '2', '3')),
 188     (k("123 123 2 *"), two_of('1', '2', '3')),
 189     (k("123 123 * *"), exactly('1', '2', '3')),
 190 ]
 191
 192
 193 @pytest.mark.parametrize("dp,n,result",
 194     [   # noqa: E128
 195         (
 196             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[80]),
 197             0,
 198             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[80]),
 199         ),
 200         (
 201             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[80]),
 202             2,
 203             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[82]),
 204         ),
 205         (
 206             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[80, 90]),
 207             2,
 208             DaemonPlacement(daemon_type='mgr', hostname='host1', ports=[82, 92]),
 209         ),
 210     ])
 211 def test_daemon_placement_renumber(dp, n, result):
 212     assert dp.renumber_ports(n) == result
 213
 214
 215 @pytest.mark.parametrize(
 216     'dp,dd,result',
 217     [
 218         (
 219             DaemonPlacement(daemon_type='mgr', hostname='host1'),
 220             DaemonDescription('mgr', 'a', 'host1'),
 221             True
 222         ),
 223         (
 224             DaemonPlacement(daemon_type='mgr', hostname='host1', name='a'),
 225             DaemonDescription('mgr', 'a', 'host1'),
 226             True
 227         ),
 228         (
 229             DaemonPlacement(daemon_type='mon', hostname='host1', name='a'),
 230             DaemonDescription('mgr', 'a', 'host1'),
 231             False
 232         ),
 233         (
 234             DaemonPlacement(daemon_type='mgr', hostname='host1', name='a'),
 235             DaemonDescription('mgr', 'b', 'host1'),
 236             False
 237         ),
 238     ])
 239 def test_daemon_placement_match(dp, dd, result):
 240     assert dp.matches_daemon(dd) == result
 241
 242
 243 @pytest.mark.parametrize("spec_section_key,spec_section",
 244     [   # noqa: E128
 245         ('h', 'hosts'),
 246         ('l', 'label'),
 247         ('p', 'host_pattern'),
 248     ])
 249 @pytest.mark.parametrize("count",
 250     [   # noqa: E128
 251         None,
 252         0,
 253         1,
 254         2,
 255         3,
 256     ])
 257 @pytest.mark.parametrize("explicit_key, explicit",
 258     [   # noqa: E128
 259         ('e', []),
 260         ('1', ['1']),
 261         ('12', ['1', '2']),
 262         ('123', ['1', '2', '3']),
 263     ])
 264 @pytest.mark.parametrize("host_key, hosts",
 265     [   # noqa: E128
 266         ('1', ['1']),
 267         ('12', ['1', '2']),
 268         ('123', ['1', '2', '3']),
 269     ])
 270 def test_explicit_scheduler(host_key, hosts,
 271                             explicit_key, explicit,
 272                             count,
 273                             spec_section_key, spec_section):
 274
 275     mk_spec, hosts = mk_spec_and_host(spec_section, hosts, explicit_key, explicit, count)
 276     run_scheduler_test(
 277         results=test_explicit_scheduler_results,
 278         mk_spec=mk_spec,
 279         hosts=hosts,
 280         daemons=[],
 281         key_elems=(host_key, explicit_key, count, spec_section_key)
 282     )
 283
 284
 285 # * first match from the top wins
 286 # * where e=[], *=any
 287 #
 288 #       + list of known hosts available for scheduling (host_key)
 289 #       |   + hosts used for explict placement (explicit_key)
 290 #       |   |   + count
 291 #       |   |   | + existing daemons
 292 #       |   |   | |     + section (host, label, pattern)
 293 #       |   |   | |     |   + expected result
 294 #       |   |   | |     |   |
 295 test_scheduler_daemons_results = [
 296     (k("*   1   * *   *"), exactly('1')),
 297     (k("1   123 * *   h"), error(OrchestratorValidationError, 'Cannot place <ServiceSpec for service_name=mgr> on 2, 3: Unknown hosts')),
 298     (k("1   123 * *   *"), exactly('1')),
 299     (k("12  123 * *   h"), error(OrchestratorValidationError, 'Cannot place <ServiceSpec for service_name=mgr> on 3: Unknown hosts')),
 300     (k("12  123 N *   *"), exactly('1', '2')),
 301     (k("12  123 1 *   *"), one_of('1', '2')),
 302     (k("12  123 2 *   *"), exactly('1', '2')),
 303     (k("12  123 3 *   *"), exactly('1', '2')),
 304     (k("123 123 N *   *"), exactly('1', '2', '3')),
 305     (k("123 123 1 e   *"), one_of('1', '2', '3')),
 306     (k("123 123 1 1   *"), exactly('1')),
 307     (k("123 123 1 3   *"), exactly('3')),
 308     (k("123 123 1 12  *"), one_of('1', '2')),
 309     (k("123 123 1 112 *"), one_of('1', '2')),
 310     (k("123 123 1 23  *"), one_of('2', '3')),
 311     (k("123 123 1 123 *"), one_of('1', '2', '3')),
 312     (k("123 123 2 e   *"), two_of('1', '2', '3')),
 313     (k("123 123 2 1   *"), _or(exactly('1', '2'), exactly('1', '3'))),
 314     (k("123 123 2 3   *"), _or(exactly('1', '3'), exactly('2', '3'))),
 315     (k("123 123 2 12  *"), exactly('1', '2')),
 316     (k("123 123 2 112 *"), exactly('1', '2')),
 317     (k("123 123 2 23  *"), exactly('2', '3')),
 318     (k("123 123 2 123 *"), two_of('1', '2', '3')),
 319     (k("123 123 3 *   *"), exactly('1', '2', '3')),
 320 ]
 321
 322
 323 @pytest.mark.parametrize("spec_section_key,spec_section",
 324     [   # noqa: E128
 325         ('h', 'hosts'),
 326         ('l', 'label'),
 327         ('p', 'host_pattern'),
 328     ])
 329 @pytest.mark.parametrize("daemons_key, daemons",
 330     [   # noqa: E128
 331         ('e', []),
 332         ('1', ['1']),
 333         ('3', ['3']),
 334         ('12', ['1', '2']),
 335         ('112', ['1', '1', '2']),  # deal with existing co-located daemons
 336         ('23', ['2', '3']),
 337         ('123', ['1', '2', '3']),
 338     ])
 339 @pytest.mark.parametrize("count",
 340     [   # noqa: E128
 341         None,
 342         1,
 343         2,
 344         3,
 345     ])
 346 @pytest.mark.parametrize("explicit_key, explicit",
 347     [   # noqa: E128
 348         ('1', ['1']),
 349         ('123', ['1', '2', '3']),
 350     ])
 351 @pytest.mark.parametrize("host_key, hosts",
 352     [   # noqa: E128
 353         ('1', ['1']),
 354         ('12', ['1', '2']),
 355         ('123', ['1', '2', '3']),
 356     ])
 357 def test_scheduler_daemons(host_key, hosts,
 358                            explicit_key, explicit,
 359                            count,
 360                            daemons_key, daemons,
 361                            spec_section_key, spec_section):
 362     mk_spec, hosts = mk_spec_and_host(spec_section, hosts, explicit_key, explicit, count)
 363     dds = [
 364         DaemonDescription('mgr', d, d)
 365         for d in daemons
 366     ]
 367     run_scheduler_test(
 368         results=test_scheduler_daemons_results,
 369         mk_spec=mk_spec,
 370         hosts=hosts,
 371         daemons=dds,
 372         key_elems=(host_key, explicit_key, count, daemons_key, spec_section_key)
 373     )
 374
 375
 376 # =========================
 377
 378
 379 class NodeAssignmentTest(NamedTuple):
 380     service_type: str
 381     placement: PlacementSpec
 382     hosts: List[str]
 383     daemons: List[DaemonDescription]
 384     rank_map: Optional[Dict[int, Dict[int, Optional[str]]]]
 385     post_rank_map: Optional[Dict[int, Dict[int, Optional[str]]]]
 386     expected: List[str]
 387     expected_add: List[str]
 388     expected_remove: List[DaemonDescription]
 389
 390
 391 @pytest.mark.parametrize("service_type,placement,hosts,daemons,rank_map,post_rank_map,expected,expected_add,expected_remove",
 392     [   # noqa: E128
 393         # just hosts
 394         NodeAssignmentTest(
 395             'mgr',
 396             PlacementSpec(hosts=['smithi060']),
 397             ['smithi060'],
 398             [],
 399             None, None,
 400             ['mgr:smithi060'], ['mgr:smithi060'], []
 401         ),
 402         # all_hosts
 403         NodeAssignmentTest(
 404             'mgr',
 405             PlacementSpec(host_pattern='*'),
 406             'host1 host2 host3'.split(),
 407             [
 408                 DaemonDescription('mgr', 'a', 'host1'),
 409                 DaemonDescription('mgr', 'b', 'host2'),
 410             ],
 411             None, None,
 412             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 413             ['mgr:host3'],
 414             []
 415         ),
 416         # all_hosts + count_per_host
 417         NodeAssignmentTest(
 418             'mds',
 419             PlacementSpec(host_pattern='*', count_per_host=2),
 420             'host1 host2 host3'.split(),
 421             [
 422                 DaemonDescription('mds', 'a', 'host1'),
 423                 DaemonDescription('mds', 'b', 'host2'),
 424             ],
 425             None, None,
 426             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 427             ['mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 428             []
 429         ),
 430         # count that is bigger than the amount of hosts. Truncate to len(hosts)
 431         # mgr should not be co-located to each other.
 432         NodeAssignmentTest(
 433             'mgr',
 434             PlacementSpec(count=4),
 435             'host1 host2 host3'.split(),
 436             [],
 437             None, None,
 438             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 439             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 440             []
 441         ),
 442         # count that is bigger than the amount of hosts; wrap around.
 443         NodeAssignmentTest(
 444             'mds',
 445             PlacementSpec(count=6),
 446             'host1 host2 host3'.split(),
 447             [],
 448             None, None,
 449             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 450             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 451             []
 452         ),
 453         # count + partial host list
 454         NodeAssignmentTest(
 455             'mgr',
 456             PlacementSpec(count=3, hosts=['host3']),
 457             'host1 host2 host3'.split(),
 458             [
 459                 DaemonDescription('mgr', 'a', 'host1'),
 460                 DaemonDescription('mgr', 'b', 'host2'),
 461             ],
 462             None, None,
 463             ['mgr:host3'],
 464             ['mgr:host3'],
 465             ['mgr.a', 'mgr.b']
 466         ),
 467         # count + partial host list (with colo)
 468         NodeAssignmentTest(
 469             'mds',
 470             PlacementSpec(count=3, hosts=['host3']),
 471             'host1 host2 host3'.split(),
 472             [
 473                 DaemonDescription('mds', 'a', 'host1'),
 474                 DaemonDescription('mds', 'b', 'host2'),
 475             ],
 476             None, None,
 477             ['mds:host3', 'mds:host3', 'mds:host3'],
 478             ['mds:host3', 'mds:host3', 'mds:host3'],
 479             ['mds.a', 'mds.b']
 480         ),
 481         # count 1 + partial host list
 482         NodeAssignmentTest(
 483             'mgr',
 484             PlacementSpec(count=1, hosts=['host3']),
 485             'host1 host2 host3'.split(),
 486             [
 487                 DaemonDescription('mgr', 'a', 'host1'),
 488                 DaemonDescription('mgr', 'b', 'host2'),
 489             ],
 490             None, None,
 491             ['mgr:host3'],
 492             ['mgr:host3'],
 493             ['mgr.a', 'mgr.b']
 494         ),
 495         # count + partial host list + existing
 496         NodeAssignmentTest(
 497             'mgr',
 498             PlacementSpec(count=2, hosts=['host3']),
 499             'host1 host2 host3'.split(),
 500             [
 501                 DaemonDescription('mgr', 'a', 'host1'),
 502             ],
 503             None, None,
 504             ['mgr:host3'],
 505             ['mgr:host3'],
 506             ['mgr.a']
 507         ),
 508         # count + partial host list + existing (deterministic)
 509         NodeAssignmentTest(
 510             'mgr',
 511             PlacementSpec(count=2, hosts=['host1']),
 512             'host1 host2'.split(),
 513             [
 514                 DaemonDescription('mgr', 'a', 'host1'),
 515             ],
 516             None, None,
 517             ['mgr:host1'],
 518             [],
 519             []
 520         ),
 521         # count + partial host list + existing (deterministic)
 522         NodeAssignmentTest(
 523             'mgr',
 524             PlacementSpec(count=2, hosts=['host1']),
 525             'host1 host2'.split(),
 526             [
 527                 DaemonDescription('mgr', 'a', 'host2'),
 528             ],
 529             None, None,
 530             ['mgr:host1'],
 531             ['mgr:host1'],
 532             ['mgr.a']
 533         ),
 534         # label only
 535         NodeAssignmentTest(
 536             'mgr',
 537             PlacementSpec(label='foo'),
 538             'host1 host2 host3'.split(),
 539             [],
 540             None, None,
 541             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 542             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 543             []
 544         ),
 545         # label + count (truncate to host list)
 546         NodeAssignmentTest(
 547             'mgr',
 548             PlacementSpec(count=4, label='foo'),
 549             'host1 host2 host3'.split(),
 550             [],
 551             None, None,
 552             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 553             ['mgr:host1', 'mgr:host2', 'mgr:host3'],
 554             []
 555         ),
 556         # label + count (with colo)
 557         NodeAssignmentTest(
 558             'mds',
 559             PlacementSpec(count=6, label='foo'),
 560             'host1 host2 host3'.split(),
 561             [],
 562             None, None,
 563             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 564             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3'],
 565             []
 566         ),
 567         # label only + count_per_hst
 568         NodeAssignmentTest(
 569             'mds',
 570             PlacementSpec(label='foo', count_per_host=3),
 571             'host1 host2 host3'.split(),
 572             [],
 573             None, None,
 574             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3',
 575              'mds:host1', 'mds:host2', 'mds:host3'],
 576             ['mds:host1', 'mds:host2', 'mds:host3', 'mds:host1', 'mds:host2', 'mds:host3',
 577              'mds:host1', 'mds:host2', 'mds:host3'],
 578             []
 579         ),
 580         # host_pattern
 581         NodeAssignmentTest(
 582             'mgr',
 583             PlacementSpec(host_pattern='mgr*'),
 584             'mgrhost1 mgrhost2 datahost'.split(),
 585             [],
 586             None, None,
 587             ['mgr:mgrhost1', 'mgr:mgrhost2'],
 588             ['mgr:mgrhost1', 'mgr:mgrhost2'],
 589             []
 590         ),
 591         # host_pattern + count_per_host
 592         NodeAssignmentTest(
 593             'mds',
 594             PlacementSpec(host_pattern='mds*', count_per_host=3),
 595             'mdshost1 mdshost2 datahost'.split(),
 596             [],
 597             None, None,
 598             ['mds:mdshost1', 'mds:mdshost2', 'mds:mdshost1', 'mds:mdshost2', 'mds:mdshost1', 'mds:mdshost2'],
 599             ['mds:mdshost1', 'mds:mdshost2', 'mds:mdshost1', 'mds:mdshost2', 'mds:mdshost1', 'mds:mdshost2'],
 600             []
 601         ),
 602         # label + count_per_host + ports
 603         NodeAssignmentTest(
 604             'rgw',
 605             PlacementSpec(count=6, label='foo'),
 606             'host1 host2 host3'.split(),
 607             [],
 608             None, None,
 609             ['rgw:host1(*:80)', 'rgw:host2(*:80)', 'rgw:host3(*:80)',
 610              'rgw:host1(*:81)', 'rgw:host2(*:81)', 'rgw:host3(*:81)'],
 611             ['rgw:host1(*:80)', 'rgw:host2(*:80)', 'rgw:host3(*:80)',
 612              'rgw:host1(*:81)', 'rgw:host2(*:81)', 'rgw:host3(*:81)'],
 613             []
 614         ),
 615         # label + count_per_host + ports (+ xisting)
 616         NodeAssignmentTest(
 617             'rgw',
 618             PlacementSpec(count=6, label='foo'),
 619             'host1 host2 host3'.split(),
 620             [
 621                 DaemonDescription('rgw', 'a', 'host1', ports=[81]),
 622                 DaemonDescription('rgw', 'b', 'host2', ports=[80]),
 623                 DaemonDescription('rgw', 'c', 'host1', ports=[82]),
 624             ],
 625             None, None,
 626             ['rgw:host1(*:80)', 'rgw:host2(*:80)', 'rgw:host3(*:80)',
 627              'rgw:host1(*:81)', 'rgw:host2(*:81)', 'rgw:host3(*:81)'],
 628             ['rgw:host1(*:80)', 'rgw:host3(*:80)',
 629              'rgw:host2(*:81)', 'rgw:host3(*:81)'],
 630             ['rgw.c']
 631         ),
 632         # cephadm.py teuth case
 633         NodeAssignmentTest(
 634             'mgr',
 635             PlacementSpec(count=3, hosts=['host1=y', 'host2=x']),
 636             'host1 host2'.split(),
 637             [
 638                 DaemonDescription('mgr', 'y', 'host1'),
 639                 DaemonDescription('mgr', 'x', 'host2'),
 640             ],
 641             None, None,
 642             ['mgr:host1(name=y)', 'mgr:host2(name=x)'],
 643             [], []
 644         ),
 645
 646         # note: host -> rank mapping is psuedo-random based on svc name, so these
 647         # host/rank pairs may seem random but they match the nfs.mynfs seed used by
 648         # the test.
 649
 650         # ranked, fresh
 651         NodeAssignmentTest(
 652             'nfs',
 653             PlacementSpec(count=3),
 654             'host1 host2 host3'.split(),
 655             [],
 656             {},
 657             {0: {0: None}, 1: {0: None}, 2: {0: None}},
 658             ['nfs:host1(rank=0.0)', 'nfs:host2(rank=1.0)', 'nfs:host3(rank=2.0)'],
 659             ['nfs:host1(rank=0.0)', 'nfs:host2(rank=1.0)', 'nfs:host3(rank=2.0)'],
 660             []
 661         ),
 662         # 21: ranked, exist
 663         NodeAssignmentTest(
 664             'nfs',
 665             PlacementSpec(count=3),
 666             'host1 host2 host3'.split(),
 667             [
 668                 DaemonDescription('nfs', '0.1', 'host1', rank=0, rank_generation=1),
 669             ],
 670             {0: {1: '0.1'}},
 671             {0: {1: '0.1'}, 1: {0: None}, 2: {0: None}},
 672             ['nfs:host1(rank=0.1)', 'nfs:host2(rank=1.0)', 'nfs:host3(rank=2.0)'],
 673             ['nfs:host2(rank=1.0)', 'nfs:host3(rank=2.0)'],
 674             []
 675         ),
 676         # ranked, exist, different ranks
 677         NodeAssignmentTest(
 678             'nfs',
 679             PlacementSpec(count=3),
 680             'host1 host2 host3'.split(),
 681             [
 682                 DaemonDescription('nfs', '0.1', 'host1', rank=0, rank_generation=1),
 683                 DaemonDescription('nfs', '1.1', 'host2', rank=1, rank_generation=1),
 684             ],
 685             {0: {1: '0.1'}, 1: {1: '1.1'}},
 686             {0: {1: '0.1'}, 1: {1: '1.1'}, 2: {0: None}},
 687             ['nfs:host1(rank=0.1)', 'nfs:host2(rank=1.1)', 'nfs:host3(rank=2.0)'],
 688             ['nfs:host3(rank=2.0)'],
 689             []
 690         ),
 691         # ranked, exist, different ranks (2)
 692         NodeAssignmentTest(
 693             'nfs',
 694             PlacementSpec(count=3),
 695             'host1 host2 host3'.split(),
 696             [
 697                 DaemonDescription('nfs', '0.1', 'host1', rank=0, rank_generation=1),
 698                 DaemonDescription('nfs', '1.1', 'host3', rank=1, rank_generation=1),
 699             ],
 700             {0: {1: '0.1'}, 1: {1: '1.1'}},
 701             {0: {1: '0.1'}, 1: {1: '1.1'}, 2: {0: None}},
 702             ['nfs:host1(rank=0.1)', 'nfs:host3(rank=1.1)', 'nfs:host2(rank=2.0)'],
 703             ['nfs:host2(rank=2.0)'],
 704             []
 705         ),
 706         # ranked, exist, extra ranks
 707         NodeAssignmentTest(
 708             'nfs',
 709             PlacementSpec(count=3),
 710             'host1 host2 host3'.split(),
 711             [
 712                 DaemonDescription('nfs', '0.5', 'host1', rank=0, rank_generation=5),
 713                 DaemonDescription('nfs', '1.5', 'host2', rank=1, rank_generation=5),
 714                 DaemonDescription('nfs', '4.5', 'host2', rank=4, rank_generation=5),
 715             ],
 716             {0: {5: '0.5'}, 1: {5: '1.5'}},
 717             {0: {5: '0.5'}, 1: {5: '1.5'}, 2: {0: None}},
 718             ['nfs:host1(rank=0.5)', 'nfs:host2(rank=1.5)', 'nfs:host3(rank=2.0)'],
 719             ['nfs:host3(rank=2.0)'],
 720             ['nfs.4.5']
 721         ),
 722         # 25: ranked, exist, extra ranks (scale down: kill off high rank)
 723         NodeAssignmentTest(
 724             'nfs',
 725             PlacementSpec(count=2),
 726             'host3 host2 host1'.split(),
 727             [
 728                 DaemonDescription('nfs', '0.5', 'host1', rank=0, rank_generation=5),
 729                 DaemonDescription('nfs', '1.5', 'host2', rank=1, rank_generation=5),
 730                 DaemonDescription('nfs', '2.5', 'host3', rank=2, rank_generation=5),
 731             ],
 732             {0: {5: '0.5'}, 1: {5: '1.5'}, 2: {5: '2.5'}},
 733             {0: {5: '0.5'}, 1: {5: '1.5'}, 2: {5: '2.5'}},
 734             ['nfs:host1(rank=0.5)', 'nfs:host2(rank=1.5)'],
 735             [],
 736             ['nfs.2.5']
 737         ),
 738         # ranked, exist, extra ranks (scale down hosts)
 739         NodeAssignmentTest(
 740             'nfs',
 741             PlacementSpec(count=2),
 742             'host1 host3'.split(),
 743             [
 744                 DaemonDescription('nfs', '0.5', 'host1', rank=0, rank_generation=5),
 745                 DaemonDescription('nfs', '1.5', 'host2', rank=1, rank_generation=5),
 746                 DaemonDescription('nfs', '2.5', 'host3', rank=4, rank_generation=5),
 747             ],
 748             {0: {5: '0.5'}, 1: {5: '1.5'}, 2: {5: '2.5'}},
 749             {0: {5: '0.5'}, 1: {5: '1.5', 6: None}, 2: {5: '2.5'}},
 750             ['nfs:host1(rank=0.5)', 'nfs:host3(rank=1.6)'],
 751             ['nfs:host3(rank=1.6)'],
 752             ['nfs.2.5', 'nfs.1.5']
 753         ),
 754         # ranked, exist, duplicate rank
 755         NodeAssignmentTest(
 756             'nfs',
 757             PlacementSpec(count=3),
 758             'host1 host2 host3'.split(),
 759             [
 760                 DaemonDescription('nfs', '0.0', 'host1', rank=0, rank_generation=0),
 761                 DaemonDescription('nfs', '1.1', 'host2', rank=1, rank_generation=1),
 762                 DaemonDescription('nfs', '1.2', 'host3', rank=1, rank_generation=2),
 763             ],
 764             {0: {0: '0.0'}, 1: {2: '1.2'}},
 765             {0: {0: '0.0'}, 1: {2: '1.2'}, 2: {0: None}},
 766             ['nfs:host1(rank=0.0)', 'nfs:host3(rank=1.2)', 'nfs:host2(rank=2.0)'],
 767             ['nfs:host2(rank=2.0)'],
 768             ['nfs.1.1']
 769         ),
 770         # 28: ranked, all gens stale (failure during update cycle)
 771         NodeAssignmentTest(
 772             'nfs',
 773             PlacementSpec(count=2),
 774             'host1 host2 host3'.split(),
 775             [
 776                 DaemonDescription('nfs', '0.2', 'host1', rank=0, rank_generation=2),
 777                 DaemonDescription('nfs', '1.2', 'host2', rank=1, rank_generation=2),
 778             ],
 779             {0: {2: '0.2'}, 1: {2: '1.2', 3: '1.3'}},
 780             {0: {2: '0.2'}, 1: {2: '1.2', 3: '1.3', 4: None}},
 781             ['nfs:host1(rank=0.2)', 'nfs:host2(rank=1.4)'],
 782             ['nfs:host2(rank=1.4)'],
 783             ['nfs.1.2']
 784         ),
 785         # ranked, not enough hosts
 786         NodeAssignmentTest(
 787             'nfs',
 788             PlacementSpec(count=4),
 789             'host1 host2 host3'.split(),
 790             [
 791                 DaemonDescription('nfs', '0.2', 'host1', rank=0, rank_generation=2),
 792                 DaemonDescription('nfs', '1.2', 'host2', rank=1, rank_generation=2),
 793             ],
 794             {0: {2: '0.2'}, 1: {2: '1.2'}},
 795             {0: {2: '0.2'}, 1: {2: '1.2'}, 2: {0: None}},
 796             ['nfs:host1(rank=0.2)', 'nfs:host2(rank=1.2)', 'nfs:host3(rank=2.0)'],
 797             ['nfs:host3(rank=2.0)'],
 798             []
 799         ),
 800         # ranked, scale down
 801         NodeAssignmentTest(
 802             'nfs',
 803             PlacementSpec(hosts=['host2']),
 804             'host1 host2'.split(),
 805             [
 806                 DaemonDescription('nfs', '0.2', 'host1', rank=0, rank_generation=2),
 807                 DaemonDescription('nfs', '1.2', 'host2', rank=1, rank_generation=2),
 808                 DaemonDescription('nfs', '2.2', 'host3', rank=2, rank_generation=2),
 809             ],
 810             {0: {2: '0.2'}, 1: {2: '1.2'}, 2: {2: '2.2'}},
 811             {0: {2: '0.2', 3: None}, 1: {2: '1.2'}, 2: {2: '2.2'}},
 812             ['nfs:host2(rank=0.3)'],
 813             ['nfs:host2(rank=0.3)'],
 814             ['nfs.0.2', 'nfs.1.2', 'nfs.2.2']
 815         ),
 816
 817     ])
 818 def test_node_assignment(service_type, placement, hosts, daemons, rank_map, post_rank_map,
 819                          expected, expected_add, expected_remove):
 820     spec = None
 821     service_id = None
 822     allow_colo = False
 823     if service_type == 'rgw':
 824         service_id = 'realm.zone'
 825         allow_colo = True
 826     elif service_type == 'mds':
 827         service_id = 'myfs'
 828         allow_colo = True
 829     elif service_type == 'nfs':
 830         service_id = 'mynfs'
 831         spec = ServiceSpec(service_type=service_type,
 832                            service_id=service_id,
 833                            placement=placement)
 834
 835     if not spec:
 836         spec = ServiceSpec(service_type=service_type,
 837                            service_id=service_id,
 838                            placement=placement)
 839
 840     all_slots, to_add, to_remove = HostAssignment(
 841         spec=spec,
 842         hosts=[HostSpec(h, labels=['foo']) for h in hosts],
 843         unreachable_hosts=[],
 844         daemons=daemons,
 845         allow_colo=allow_colo,
 846         rank_map=rank_map,
 847     ).place()
 848
 849     assert rank_map == post_rank_map
 850
 851     got = [str(p) for p in all_slots]
 852     num_wildcard = 0
 853     for i in expected:
 854         if i == '*':
 855             num_wildcard += 1
 856         else:
 857             assert i in got
 858             got.remove(i)
 859     assert num_wildcard == len(got)
 860
 861     got = [str(p) for p in to_add]
 862     num_wildcard = 0
 863     for i in expected_add:
 864         if i == '*':
 865             num_wildcard += 1
 866         else:
 867             assert i in got
 868             got.remove(i)
 869     assert num_wildcard == len(got)
 870
 871     assert sorted([d.name() for d in to_remove]) == sorted(expected_remove)
 872
 873
 874 class NodeAssignmentTest2(NamedTuple):
 875     service_type: str
 876     placement: PlacementSpec
 877     hosts: List[str]
 878     daemons: List[DaemonDescription]
 879     expected_len: int
 880     in_set: List[str]
 881
 882
 883 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected_len,in_set",
 884     [   # noqa: E128
 885         # just count
 886         NodeAssignmentTest2(
 887             'mgr',
 888             PlacementSpec(count=1),
 889             'host1 host2 host3'.split(),
 890             [],
 891             1,
 892             ['host1', 'host2', 'host3'],
 893         ),
 894
 895         # hosts + (smaller) count
 896         NodeAssignmentTest2(
 897             'mgr',
 898             PlacementSpec(count=1, hosts='host1 host2'.split()),
 899             'host1 host2'.split(),
 900             [],
 901             1,
 902             ['host1', 'host2'],
 903         ),
 904         # hosts + (smaller) count, existing
 905         NodeAssignmentTest2(
 906             'mgr',
 907             PlacementSpec(count=1, hosts='host1 host2 host3'.split()),
 908             'host1 host2 host3'.split(),
 909             [DaemonDescription('mgr', 'mgr.a', 'host1')],
 910             1,
 911             ['host1', 'host2', 'host3'],
 912         ),
 913         # hosts + (smaller) count, (more) existing
 914         NodeAssignmentTest2(
 915             'mgr',
 916             PlacementSpec(count=1, hosts='host1 host2 host3'.split()),
 917             'host1 host2 host3'.split(),
 918             [
 919                 DaemonDescription('mgr', 'a', 'host1'),
 920                 DaemonDescription('mgr', 'b', 'host2'),
 921             ],
 922             1,
 923             ['host1', 'host2']
 924         ),
 925         # count + partial host list
 926         NodeAssignmentTest2(
 927             'mgr',
 928             PlacementSpec(count=2, hosts=['host3']),
 929             'host1 host2 host3'.split(),
 930             [],
 931             1,
 932             ['host1', 'host2', 'host3']
 933         ),
 934         # label + count
 935         NodeAssignmentTest2(
 936             'mgr',
 937             PlacementSpec(count=1, label='foo'),
 938             'host1 host2 host3'.split(),
 939             [],
 940             1,
 941             ['host1', 'host2', 'host3']
 942         ),
 943     ])
 944 def test_node_assignment2(service_type, placement, hosts,
 945                           daemons, expected_len, in_set):
 946     hosts, to_add, to_remove = HostAssignment(
 947         spec=ServiceSpec(service_type, placement=placement),
 948         hosts=[HostSpec(h, labels=['foo']) for h in hosts],
 949         unreachable_hosts=[],
 950         daemons=daemons,
 951     ).place()
 952     assert len(hosts) == expected_len
 953     for h in [h.hostname for h in hosts]:
 954         assert h in in_set
 955
 956
 957 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected_len,must_have",
 958     [   # noqa: E128
 959         # hosts + (smaller) count, (more) existing
 960         NodeAssignmentTest2(
 961             'mgr',
 962             PlacementSpec(count=3, hosts='host3'.split()),
 963             'host1 host2 host3'.split(),
 964             [],
 965             1,
 966             ['host3']
 967         ),
 968         # count + partial host list
 969         NodeAssignmentTest2(
 970             'mgr',
 971             PlacementSpec(count=2, hosts=['host3']),
 972             'host1 host2 host3'.split(),
 973             [],
 974             1,
 975             ['host3']
 976         ),
 977     ])
 978 def test_node_assignment3(service_type, placement, hosts,
 979                           daemons, expected_len, must_have):
 980     hosts, to_add, to_remove = HostAssignment(
 981         spec=ServiceSpec(service_type, placement=placement),
 982         hosts=[HostSpec(h) for h in hosts],
 983         unreachable_hosts=[],
 984         daemons=daemons,
 985     ).place()
 986     assert len(hosts) == expected_len
 987     for h in must_have:
 988         assert h in [h.hostname for h in hosts]
 989
 990
 991 class NodeAssignmentTest4(NamedTuple):
 992     spec: ServiceSpec
 993     networks: Dict[str, Dict[str, Dict[str, List[str]]]]
 994     daemons: List[DaemonDescription]
 995     expected: List[str]
 996     expected_add: List[str]
 997     expected_remove: List[DaemonDescription]
 998
 999
1000 @pytest.mark.parametrize("spec,networks,daemons,expected,expected_add,expected_remove",
1001     [   # noqa: E128
1002         NodeAssignmentTest4(
1003             ServiceSpec(
1004                 service_type='rgw',
1005                 service_id='foo',
1006                 placement=PlacementSpec(count=6, label='foo'),
1007                 networks=['10.0.0.0/8'],
1008             ),
1009             {
1010                 'host1': {'10.0.0.0/8': {'eth0': ['10.0.0.1']}},
1011                 'host2': {'10.0.0.0/8': {'eth0': ['10.0.0.2']}},
1012                 'host3': {'192.168.0.0/16': {'eth0': ['192.168.0.1']}},
1013             },
1014             [],
1015             ['rgw:host1(10.0.0.1:80)', 'rgw:host2(10.0.0.2:80)',
1016              'rgw:host1(10.0.0.1:81)', 'rgw:host2(10.0.0.2:81)',
1017              'rgw:host1(10.0.0.1:82)', 'rgw:host2(10.0.0.2:82)'],
1018             ['rgw:host1(10.0.0.1:80)', 'rgw:host2(10.0.0.2:80)',
1019              'rgw:host1(10.0.0.1:81)', 'rgw:host2(10.0.0.2:81)',
1020              'rgw:host1(10.0.0.1:82)', 'rgw:host2(10.0.0.2:82)'],
1021             []
1022         ),
1023         NodeAssignmentTest4(
1024             IngressSpec(
1025                 service_type='ingress',
1026                 service_id='rgw.foo',
1027                 frontend_port=443,
1028                 monitor_port=8888,
1029                 virtual_ip='10.0.0.20/8',
1030                 backend_service='rgw.foo',
1031                 placement=PlacementSpec(label='foo'),
1032                 networks=['10.0.0.0/8'],
1033             ),
1034             {
1035                 'host1': {'10.0.0.0/8': {'eth0': ['10.0.0.1']}},
1036                 'host2': {'10.0.0.0/8': {'eth1': ['10.0.0.2']}},
1037                 'host3': {'192.168.0.0/16': {'eth2': ['192.168.0.1']}},
1038             },
1039             [],
1040             ['haproxy:host1(10.0.0.1:443,8888)', 'haproxy:host2(10.0.0.2:443,8888)',
1041              'keepalived:host1', 'keepalived:host2'],
1042             ['haproxy:host1(10.0.0.1:443,8888)', 'haproxy:host2(10.0.0.2:443,8888)',
1043              'keepalived:host1', 'keepalived:host2'],
1044             []
1045         ),
1046         NodeAssignmentTest4(
1047             IngressSpec(
1048                 service_type='ingress',
1049                 service_id='rgw.foo',
1050                 frontend_port=443,
1051                 monitor_port=8888,
1052                 virtual_ip='10.0.0.20/8',
1053                 backend_service='rgw.foo',
1054                 placement=PlacementSpec(label='foo'),
1055                 networks=['10.0.0.0/8'],
1056             ),
1057             {
1058                 'host1': {'10.0.0.0/8': {'eth0': ['10.0.0.1']}},
1059                 'host2': {'10.0.0.0/8': {'eth1': ['10.0.0.2']}},
1060                 'host3': {'192.168.0.0/16': {'eth2': ['192.168.0.1']}},
1061             },
1062             [
1063                 DaemonDescription('haproxy', 'a', 'host1', ip='10.0.0.1',
1064                                   ports=[443, 8888]),
1065                 DaemonDescription('keepalived', 'b', 'host2'),
1066                 DaemonDescription('keepalived', 'c', 'host3'),
1067             ],
1068             ['haproxy:host1(10.0.0.1:443,8888)', 'haproxy:host2(10.0.0.2:443,8888)',
1069              'keepalived:host1', 'keepalived:host2'],
1070             ['haproxy:host2(10.0.0.2:443,8888)',
1071              'keepalived:host1'],
1072             ['keepalived.c']
1073         ),
1074     ])
1075 def test_node_assignment4(spec, networks, daemons,
1076                           expected, expected_add, expected_remove):
1077     all_slots, to_add, to_remove = HostAssignment(
1078         spec=spec,
1079         hosts=[HostSpec(h, labels=['foo']) for h in networks.keys()],
1080         unreachable_hosts=[],
1081         daemons=daemons,
1082         allow_colo=True,
1083         networks=networks,
1084         primary_daemon_type='haproxy' if spec.service_type == 'ingress' else spec.service_type,
1085         per_host_daemon_type='keepalived' if spec.service_type == 'ingress' else None,
1086     ).place()
1087
1088     got = [str(p) for p in all_slots]
1089     num_wildcard = 0
1090     for i in expected:
1091         if i == '*':
1092             num_wildcard += 1
1093         else:
1094             assert i in got
1095             got.remove(i)
1096     assert num_wildcard == len(got)
1097
1098     got = [str(p) for p in to_add]
1099     num_wildcard = 0
1100     for i in expected_add:
1101         if i == '*':
1102             num_wildcard += 1
1103         else:
1104             assert i in got
1105             got.remove(i)
1106     assert num_wildcard == len(got)
1107
1108     assert sorted([d.name() for d in to_remove]) == sorted(expected_remove)
1109
1110
1111 @pytest.mark.parametrize("placement",
1112     [   # noqa: E128
1113         ('1 *'),
1114         ('* label:foo'),
1115         ('* host1 host2'),
1116         ('hostname12hostname12hostname12hostname12hostname12hostname12hostname12'),  # > 63 chars
1117     ])
1118 def test_bad_placements(placement):
1119     try:
1120         PlacementSpec.from_string(placement.split(' '))
1121         assert False
1122     except SpecValidationError:
1123         pass
1124
1125
1126 class NodeAssignmentTestBadSpec(NamedTuple):
1127     service_type: str
1128     placement: PlacementSpec
1129     hosts: List[str]
1130     daemons: List[DaemonDescription]
1131     expected: str
1132
1133
1134 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected",
1135     [   # noqa: E128
1136         # unknown host
1137         NodeAssignmentTestBadSpec(
1138             'mgr',
1139             PlacementSpec(hosts=['unknownhost']),
1140             ['knownhost'],
1141             [],
1142             "Cannot place <ServiceSpec for service_name=mgr> on unknownhost: Unknown hosts"
1143         ),
1144         # unknown host pattern
1145         NodeAssignmentTestBadSpec(
1146             'mgr',
1147             PlacementSpec(host_pattern='unknownhost'),
1148             ['knownhost'],
1149             [],
1150             "Cannot place <ServiceSpec for service_name=mgr>: No matching hosts"
1151         ),
1152         # unknown label
1153         NodeAssignmentTestBadSpec(
1154             'mgr',
1155             PlacementSpec(label='unknownlabel'),
1156             [],
1157             [],
1158             "Cannot place <ServiceSpec for service_name=mgr>: No matching hosts for label unknownlabel"
1159         ),
1160     ])
1161 def test_bad_specs(service_type, placement, hosts, daemons, expected):
1162     with pytest.raises(OrchestratorValidationError) as e:
1163         hosts, to_add, to_remove = HostAssignment(
1164             spec=ServiceSpec(service_type, placement=placement),
1165             hosts=[HostSpec(h) for h in hosts],
1166             unreachable_hosts=[],
1167             daemons=daemons,
1168         ).place()
1169     assert str(e.value) == expected
1170
1171
1172 class ActiveAssignmentTest(NamedTuple):
1173     service_type: str
1174     placement: PlacementSpec
1175     hosts: List[str]
1176     daemons: List[DaemonDescription]
1177     expected: List[List[str]]
1178     expected_add: List[List[str]]
1179     expected_remove: List[List[str]]
1180
1181
1182 @pytest.mark.parametrize("service_type,placement,hosts,daemons,expected,expected_add,expected_remove",
1183                          [
1184                              ActiveAssignmentTest(
1185                                  'mgr',
1186                                  PlacementSpec(count=2),
1187                                  'host1 host2 host3'.split(),
1188                                  [
1189                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1190                                      DaemonDescription('mgr', 'b', 'host2'),
1191                                      DaemonDescription('mgr', 'c', 'host3'),
1192                                  ],
1193                                  [['host1', 'host2'], ['host1', 'host3']],
1194                                  [[]],
1195                                  [['mgr.b'], ['mgr.c']]
1196                              ),
1197                              ActiveAssignmentTest(
1198                                  'mgr',
1199                                  PlacementSpec(count=2),
1200                                  'host1 host2 host3'.split(),
1201                                  [
1202                                      DaemonDescription('mgr', 'a', 'host1'),
1203                                      DaemonDescription('mgr', 'b', 'host2'),
1204                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1205                                  ],
1206                                  [['host1', 'host3'], ['host2', 'host3']],
1207                                  [[]],
1208                                  [['mgr.a'], ['mgr.b']]
1209                              ),
1210                              ActiveAssignmentTest(
1211                                  'mgr',
1212                                  PlacementSpec(count=1),
1213                                  'host1 host2 host3'.split(),
1214                                  [
1215                                      DaemonDescription('mgr', 'a', 'host1'),
1216                                      DaemonDescription('mgr', 'b', 'host2', is_active=True),
1217                                      DaemonDescription('mgr', 'c', 'host3'),
1218                                  ],
1219                                  [['host2']],
1220                                  [[]],
1221                                  [['mgr.a', 'mgr.c']]
1222                              ),
1223                              ActiveAssignmentTest(
1224                                  'mgr',
1225                                  PlacementSpec(count=1),
1226                                  'host1 host2 host3'.split(),
1227                                  [
1228                                      DaemonDescription('mgr', 'a', 'host1'),
1229                                      DaemonDescription('mgr', 'b', 'host2'),
1230                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1231                                  ],
1232                                  [['host3']],
1233                                  [[]],
1234                                  [['mgr.a', 'mgr.b']]
1235                              ),
1236                              ActiveAssignmentTest(
1237                                  'mgr',
1238                                  PlacementSpec(count=1),
1239                                  'host1 host2 host3'.split(),
1240                                  [
1241                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1242                                      DaemonDescription('mgr', 'b', 'host2'),
1243                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1244                                  ],
1245                                  [['host1'], ['host3']],
1246                                  [[]],
1247                                  [['mgr.a', 'mgr.b'], ['mgr.b', 'mgr.c']]
1248                              ),
1249                              ActiveAssignmentTest(
1250                                  'mgr',
1251                                  PlacementSpec(count=2),
1252                                  'host1 host2 host3'.split(),
1253                                  [
1254                                      DaemonDescription('mgr', 'a', 'host1'),
1255                                      DaemonDescription('mgr', 'b', 'host2', is_active=True),
1256                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1257                                  ],
1258                                  [['host2', 'host3']],
1259                                  [[]],
1260                                  [['mgr.a']]
1261                              ),
1262                              ActiveAssignmentTest(
1263                                  'mgr',
1264                                  PlacementSpec(count=1),
1265                                  'host1 host2 host3'.split(),
1266                                  [
1267                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1268                                      DaemonDescription('mgr', 'b', 'host2', is_active=True),
1269                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1270                                  ],
1271                                  [['host1'], ['host2'], ['host3']],
1272                                  [[]],
1273                                  [['mgr.a', 'mgr.b'], ['mgr.b', 'mgr.c'], ['mgr.a', 'mgr.c']]
1274                              ),
1275                              ActiveAssignmentTest(
1276                                  'mgr',
1277                                  PlacementSpec(count=1),
1278                                  'host1 host2 host3'.split(),
1279                                  [
1280                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1281                                      DaemonDescription('mgr', 'a2', 'host1'),
1282                                      DaemonDescription('mgr', 'b', 'host2'),
1283                                      DaemonDescription('mgr', 'c', 'host3'),
1284                                  ],
1285                                  [['host1']],
1286                                  [[]],
1287                                  [['mgr.a2', 'mgr.b', 'mgr.c']]
1288                              ),
1289                              ActiveAssignmentTest(
1290                                  'mgr',
1291                                  PlacementSpec(count=1),
1292                                  'host1 host2 host3'.split(),
1293                                  [
1294                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1295                                      DaemonDescription('mgr', 'a2', 'host1', is_active=True),
1296                                      DaemonDescription('mgr', 'b', 'host2'),
1297                                      DaemonDescription('mgr', 'c', 'host3'),
1298                                  ],
1299                                  [['host1']],
1300                                  [[]],
1301                                  [['mgr.a', 'mgr.b', 'mgr.c'], ['mgr.a2', 'mgr.b', 'mgr.c']]
1302                              ),
1303                              ActiveAssignmentTest(
1304                                  'mgr',
1305                                  PlacementSpec(count=2),
1306                                  'host1 host2 host3'.split(),
1307                                  [
1308                                      DaemonDescription('mgr', 'a', 'host1', is_active=True),
1309                                      DaemonDescription('mgr', 'a2', 'host1'),
1310                                      DaemonDescription('mgr', 'b', 'host2'),
1311                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1312                                  ],
1313                                  [['host1', 'host3']],
1314                                  [[]],
1315                                  [['mgr.a2', 'mgr.b']]
1316                              ),
1317                              # Explicit placement should override preference for active daemon
1318                              ActiveAssignmentTest(
1319                                  'mgr',
1320                                  PlacementSpec(count=1, hosts=['host1']),
1321                                  'host1 host2 host3'.split(),
1322                                  [
1323                                      DaemonDescription('mgr', 'a', 'host1'),
1324                                      DaemonDescription('mgr', 'b', 'host2'),
1325                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1326                                  ],
1327                                  [['host1']],
1328                                  [[]],
1329                                  [['mgr.b', 'mgr.c']]
1330                              ),
1331
1332                          ])
1333 def test_active_assignment(service_type, placement, hosts, daemons, expected, expected_add, expected_remove):
1334
1335     spec = ServiceSpec(service_type=service_type,
1336                        service_id=None,
1337                        placement=placement)
1338
1339     hosts, to_add, to_remove = HostAssignment(
1340         spec=spec,
1341         hosts=[HostSpec(h) for h in hosts],
1342         unreachable_hosts=[],
1343         daemons=daemons,
1344     ).place()
1345     assert sorted([h.hostname for h in hosts]) in expected
1346     assert sorted([h.hostname for h in to_add]) in expected_add
1347     assert sorted([h.name() for h in to_remove]) in expected_remove
1348
1349
1350 class UnreachableHostsTest(NamedTuple):
1351     service_type: str
1352     placement: PlacementSpec
1353     hosts: List[str]
1354     unreachables_hosts: List[str]
1355     daemons: List[DaemonDescription]
1356     expected_add: List[List[str]]
1357     expected_remove: List[List[str]]
1358
1359
1360 @pytest.mark.parametrize("service_type,placement,hosts,unreachable_hosts,daemons,expected_add,expected_remove",
1361                          [
1362                              UnreachableHostsTest(
1363                                  'mgr',
1364                                  PlacementSpec(count=3),
1365                                  'host1 host2 host3'.split(),
1366                                  ['host2'],
1367                                  [],
1368                                  [['host1', 'host3']],
1369                                  [[]],
1370                              ),
1371                              UnreachableHostsTest(
1372                                  'mgr',
1373                                  PlacementSpec(hosts=['host3']),
1374                                  'host1 host2 host3'.split(),
1375                                  ['host1'],
1376                                  [
1377                                      DaemonDescription('mgr', 'a', 'host1'),
1378                                      DaemonDescription('mgr', 'b', 'host2'),
1379                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1380                                  ],
1381                                  [[]],
1382                                  [['mgr.b']],
1383                              ),
1384                              UnreachableHostsTest(
1385                                  'mgr',
1386                                  PlacementSpec(count=3),
1387                                  'host1 host2 host3 host4'.split(),
1388                                  ['host1'],
1389                                  [
1390                                      DaemonDescription('mgr', 'a', 'host1'),
1391                                      DaemonDescription('mgr', 'b', 'host2'),
1392                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1393                                  ],
1394                                  [[]],
1395                                  [[]],
1396                              ),
1397                              UnreachableHostsTest(
1398                                  'mgr',
1399                                  PlacementSpec(count=1),
1400                                  'host1 host2 host3 host4'.split(),
1401                                  'host1 host3'.split(),
1402                                  [
1403                                      DaemonDescription('mgr', 'a', 'host1'),
1404                                      DaemonDescription('mgr', 'b', 'host2'),
1405                                      DaemonDescription('mgr', 'c', 'host3', is_active=True),
1406                                  ],
1407                                  [[]],
1408                                  [['mgr.b']],
1409                              ),
1410                              UnreachableHostsTest(
1411                                  'mgr',
1412                                  PlacementSpec(count=3),
1413                                  'host1 host2 host3 host4'.split(),
1414                                  ['host2'],
1415                                  [],
1416                                  [['host1', 'host3', 'host4']],
1417                                  [[]],
1418                              ),
1419                              UnreachableHostsTest(
1420                                  'mgr',
1421                                  PlacementSpec(count=3),
1422                                  'host1 host2 host3 host4'.split(),
1423                                  'host1 host4'.split(),
1424                                  [],
1425                                  [['host2', 'host3']],
1426                                  [[]],
1427                              ),
1428
1429                          ])
1430 def test_unreachable_host(service_type, placement, hosts, unreachable_hosts, daemons, expected_add, expected_remove):
1431
1432     spec = ServiceSpec(service_type=service_type,
1433                        service_id=None,
1434                        placement=placement)
1435
1436     hosts, to_add, to_remove = HostAssignment(
1437         spec=spec,
1438         hosts=[HostSpec(h) for h in hosts],
1439         unreachable_hosts=[HostSpec(h) for h in unreachable_hosts],
1440         daemons=daemons,
1441     ).place()
1442     assert sorted([h.hostname for h in to_add]) in expected_add
1443     assert sorted([h.name() for h in to_remove]) in expected_remove
1444
1445
1446 class RescheduleFromOfflineTest(NamedTuple):
1447     service_type: str
1448     placement: PlacementSpec
1449     hosts: List[str]
1450     maintenance_hosts: List[str]
1451     offline_hosts: List[str]
1452     daemons: List[DaemonDescription]
1453     expected_add: List[List[str]]
1454     expected_remove: List[List[str]]
1455
1456
1457 @pytest.mark.parametrize("service_type,placement,hosts,maintenance_hosts,offline_hosts,daemons,expected_add,expected_remove",
1458                          [
1459                              RescheduleFromOfflineTest(
1460                                  'nfs',
1461                                  PlacementSpec(count=2),
1462                                  'host1 host2 host3'.split(),
1463                                  [],
1464                                  ['host2'],
1465                                  [
1466                                      DaemonDescription('nfs', 'a', 'host1'),
1467                                      DaemonDescription('nfs', 'b', 'host2'),
1468                                  ],
1469                                  [['host3']],
1470                                  [[]],
1471                              ),
1472                              RescheduleFromOfflineTest(
1473                                  'nfs',
1474                                  PlacementSpec(count=2),
1475                                  'host1 host2 host3'.split(),
1476                                  ['host2'],
1477                                  [],
1478                                  [
1479                                      DaemonDescription('nfs', 'a', 'host1'),
1480                                      DaemonDescription('nfs', 'b', 'host2'),
1481                                  ],
1482                                  [[]],
1483                                  [[]],
1484                              ),
1485                              RescheduleFromOfflineTest(
1486                                  'mon',
1487                                  PlacementSpec(count=2),
1488                                  'host1 host2 host3'.split(),
1489                                  [],
1490                                  ['host2'],
1491                                  [
1492                                      DaemonDescription('mon', 'a', 'host1'),
1493                                      DaemonDescription('mon', 'b', 'host2'),
1494                                  ],
1495                                  [[]],
1496                                  [[]],
1497                              ),
1498                          ])
1499 def test_remove_from_offline(service_type, placement, hosts, maintenance_hosts, offline_hosts, daemons, expected_add, expected_remove):
1500
1501     spec = ServiceSpec(service_type=service_type,
1502                        service_id='test',
1503                        placement=placement)
1504
1505     host_specs = [HostSpec(h) for h in hosts]
1506     for h in host_specs:
1507         if h.hostname in offline_hosts:
1508             h.status = 'offline'
1509         if h.hostname in maintenance_hosts:
1510             h.status = 'maintenance'
1511
1512     hosts, to_add, to_remove = HostAssignment(
1513         spec=spec,
1514         hosts=host_specs,
1515         unreachable_hosts=[h for h in host_specs if h.status],
1516         daemons=daemons,
1517     ).place()
1518     assert sorted([h.hostname for h in to_add]) in expected_add
1519     assert sorted([h.name() for h in to_remove]) in expected_remove