ceph/src/python-common/ceph/tests/test_service_spec.py

   1 # flake8: noqa
   2 import json
   3 import re
   4
   5 import yaml
   6
   7 import pytest
   8
   9 from ceph.deployment.service_spec import (
  10     AlertManagerSpec,
  11     ArgumentSpec,
  12     CustomContainerSpec,
  13     GrafanaSpec,
  14     HostPlacementSpec,
  15     IscsiServiceSpec,
  16     NFSServiceSpec,
  17     PlacementSpec,
  18     PrometheusSpec,
  19     RGWSpec,
  20     ServiceSpec,
  21 )
  22 from ceph.deployment.drive_group import DriveGroupSpec
  23 from ceph.deployment.hostspec import SpecValidationError
  24
  25
  26 @pytest.mark.parametrize("test_input,expected, require_network",
  27                          [("myhost", ('myhost', '', ''), False),
  28                           ("myhost=sname", ('myhost', '', 'sname'), False),
  29                           ("myhost:10.1.1.10", ('myhost', '10.1.1.10', ''), True),
  30                           ("myhost:10.1.1.10=sname", ('myhost', '10.1.1.10', 'sname'), True),
  31                           ("myhost:10.1.1.0/32", ('myhost', '10.1.1.0/32', ''), True),
  32                           ("myhost:10.1.1.0/32=sname", ('myhost', '10.1.1.0/32', 'sname'), True),
  33                           ("myhost:[v1:10.1.1.10:6789]", ('myhost', '[v1:10.1.1.10:6789]', ''), True),
  34                           ("myhost:[v1:10.1.1.10:6789]=sname", ('myhost', '[v1:10.1.1.10:6789]', 'sname'), True),
  35                           ("myhost:[v1:10.1.1.10:6789,v2:10.1.1.11:3000]", ('myhost', '[v1:10.1.1.10:6789,v2:10.1.1.11:3000]', ''), True),
  36                           ("myhost:[v1:10.1.1.10:6789,v2:10.1.1.11:3000]=sname", ('myhost', '[v1:10.1.1.10:6789,v2:10.1.1.11:3000]', 'sname'), True),
  37                           ])
  38 def test_parse_host_placement_specs(test_input, expected, require_network):
  39     ret = HostPlacementSpec.parse(test_input, require_network=require_network)
  40     assert ret == expected
  41     assert str(ret) == test_input
  42
  43     ps = PlacementSpec.from_string(test_input)
  44     assert ps.pretty_str() == test_input
  45     assert ps == PlacementSpec.from_string(ps.pretty_str())
  46
  47     # Testing the old verbose way of generating json. Don't remove:
  48     assert ret == HostPlacementSpec.from_json({
  49             'hostname': ret.hostname,
  50             'network': ret.network,
  51             'name': ret.name
  52         })
  53
  54     assert ret == HostPlacementSpec.from_json(ret.to_json())
  55
  56
  57 @pytest.mark.parametrize(
  58     "spec, raise_exception, msg",
  59     [
  60         (GrafanaSpec(protocol=''), True, '^Invalid protocol'),
  61         (GrafanaSpec(protocol='invalid'), True, '^Invalid protocol'),
  62         (GrafanaSpec(protocol='-http'), True, '^Invalid protocol'),
  63         (GrafanaSpec(protocol='-https'), True, '^Invalid protocol'),
  64         (GrafanaSpec(protocol='http'), False, ''),
  65         (GrafanaSpec(protocol='https'), False, ''),
  66         (GrafanaSpec(anonymous_access=False), True, '^Either initial'),  # we require inital_admin_password if anonymous_access is False
  67         (GrafanaSpec(anonymous_access=False, initial_admin_password='test'), False, ''),
  68     ])
  69 def test_apply_grafana(spec: GrafanaSpec, raise_exception: bool, msg: str):
  70     if  raise_exception:
  71         with pytest.raises(SpecValidationError, match=msg):
  72             spec.validate()
  73     else:
  74         spec.validate()
  75
  76 @pytest.mark.parametrize(
  77     "spec, raise_exception, msg",
  78     [
  79         # Valid retention_time values (valid units: 'y', 'w', 'd', 'h', 'm', 's')
  80         (PrometheusSpec(retention_time='1y'), False, ''),
  81         (PrometheusSpec(retention_time=' 10w '), False, ''),
  82         (PrometheusSpec(retention_time=' 1348d'), False, ''),
  83         (PrometheusSpec(retention_time='2000h '), False, ''),
  84         (PrometheusSpec(retention_time='173847m'), False, ''),
  85         (PrometheusSpec(retention_time='200s'), False, ''),
  86         (PrometheusSpec(retention_time='  '), False, ''),  # default value will be used
  87         # Invalid retention_time values
  88         (PrometheusSpec(retention_time='100k'), True, '^Invalid retention time'),     # invalid unit
  89         (PrometheusSpec(retention_time='10'), True, '^Invalid retention time'),       # no unit
  90         (PrometheusSpec(retention_time='100.00y'), True, '^Invalid retention time'),  # invalid value and valid unit
  91         (PrometheusSpec(retention_time='100.00k'), True, '^Invalid retention time'),  # invalid value and invalid unit
  92         (PrometheusSpec(retention_time='---'), True, '^Invalid retention time'),      # invalid value
  93
  94         # Valid retention_size values (valid units: 'B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')
  95         (PrometheusSpec(retention_size='123456789B'), False, ''),
  96         (PrometheusSpec(retention_size=' 200KB'), False, ''),
  97         (PrometheusSpec(retention_size='99999MB '), False, ''),
  98         (PrometheusSpec(retention_size=' 10GB '), False, ''),
  99         (PrometheusSpec(retention_size='100TB'), False, ''),
 100         (PrometheusSpec(retention_size='500PB'), False, ''),
 101         (PrometheusSpec(retention_size='200EB'), False, ''),
 102         (PrometheusSpec(retention_size='  '), False, ''),  # default value will be used
 103
 104         # Invalid retention_size values
 105         (PrometheusSpec(retention_size='100b'), True, '^Invalid retention size'),      # invalid unit (case sensitive)
 106         (PrometheusSpec(retention_size='333kb'), True, '^Invalid retention size'),     # invalid unit (case sensitive)
 107         (PrometheusSpec(retention_size='2000'), True, '^Invalid retention size'),      # no unit
 108         (PrometheusSpec(retention_size='200.00PB'), True, '^Invalid retention size'),  # invalid value and valid unit
 109         (PrometheusSpec(retention_size='400.B'), True, '^Invalid retention size'),     # invalid value and invalid unit
 110         (PrometheusSpec(retention_size='10.000s'), True, '^Invalid retention size'),   # invalid value and invalid unit
 111         (PrometheusSpec(retention_size='...'), True, '^Invalid retention size'),       # invalid value
 112
 113         # valid retention_size and valid retention_time
 114         (PrometheusSpec(retention_time='1y', retention_size='100GB'), False, ''),
 115         # invalid retention_time and valid retention_size
 116         (PrometheusSpec(retention_time='1j', retention_size='100GB'), True, '^Invalid retention time'),
 117         # valid retention_time and invalid retention_size
 118         (PrometheusSpec(retention_time='1y', retention_size='100gb'), True, '^Invalid retention size'),
 119         # valid retention_time and invalid retention_size
 120         (PrometheusSpec(retention_time='1y', retention_size='100gb'), True, '^Invalid retention size'),
 121         # invalid retention_time and invalid retention_size
 122         (PrometheusSpec(retention_time='1i', retention_size='100gb'), True, '^Invalid retention time'),
 123     ])
 124 def test_apply_prometheus(spec: PrometheusSpec, raise_exception: bool, msg: str):
 125     if raise_exception:
 126         with pytest.raises(SpecValidationError, match=msg):
 127                 spec.validate()
 128     else:
 129         spec.validate()
 130
 131 @pytest.mark.parametrize(
 132     "test_input,expected",
 133     [
 134         ('', "PlacementSpec()"),
 135         ("count:2", "PlacementSpec(count=2)"),
 136         ("3", "PlacementSpec(count=3)"),
 137         ("host1 host2", "PlacementSpec(hosts=[HostPlacementSpec(hostname='host1', network='', name=''), HostPlacementSpec(hostname='host2', network='', name='')])"),
 138         ("host1;host2", "PlacementSpec(hosts=[HostPlacementSpec(hostname='host1', network='', name=''), HostPlacementSpec(hostname='host2', network='', name='')])"),
 139         ("host1,host2", "PlacementSpec(hosts=[HostPlacementSpec(hostname='host1', network='', name=''), HostPlacementSpec(hostname='host2', network='', name='')])"),
 140         ("host1 host2=b", "PlacementSpec(hosts=[HostPlacementSpec(hostname='host1', network='', name=''), HostPlacementSpec(hostname='host2', network='', name='b')])"),
 141         ("host1=a host2=b", "PlacementSpec(hosts=[HostPlacementSpec(hostname='host1', network='', name='a'), HostPlacementSpec(hostname='host2', network='', name='b')])"),
 142         ("host1:1.2.3.4=a host2:1.2.3.5=b", "PlacementSpec(hosts=[HostPlacementSpec(hostname='host1', network='1.2.3.4', name='a'), HostPlacementSpec(hostname='host2', network='1.2.3.5', name='b')])"),
 143         ("myhost:[v1:10.1.1.10:6789]", "PlacementSpec(hosts=[HostPlacementSpec(hostname='myhost', network='[v1:10.1.1.10:6789]', name='')])"),
 144         ('2 host1 host2', "PlacementSpec(count=2, hosts=[HostPlacementSpec(hostname='host1', network='', name=''), HostPlacementSpec(hostname='host2', network='', name='')])"),
 145         ('label:foo', "PlacementSpec(label='foo')"),
 146         ('3 label:foo', "PlacementSpec(count=3, label='foo')"),
 147         ('*', "PlacementSpec(host_pattern='*')"),
 148         ('3 data[1-3]', "PlacementSpec(count=3, host_pattern='data[1-3]')"),
 149         ('3 data?', "PlacementSpec(count=3, host_pattern='data?')"),
 150         ('3 data*', "PlacementSpec(count=3, host_pattern='data*')"),
 151         ("count-per-host:4 label:foo", "PlacementSpec(count_per_host=4, label='foo')"),
 152     ])
 153 def test_parse_placement_specs(test_input, expected):
 154     ret = PlacementSpec.from_string(test_input)
 155     assert str(ret) == expected
 156     assert PlacementSpec.from_string(ret.pretty_str()) == ret, f'"{ret.pretty_str()}" != "{test_input}"'
 157
 158 @pytest.mark.parametrize(
 159     "test_input",
 160     [
 161         ("host=a host*"),
 162         ("host=a label:wrong"),
 163         ("host? host*"),
 164         ('host=a count-per-host:0'),
 165         ('host=a count-per-host:-10'),
 166         ('count:2 count-per-host:1'),
 167         ('host1=a host2=b count-per-host:2'),
 168         ('host1:10/8 count-per-host:2'),
 169         ('count-per-host:2'),
 170     ]
 171 )
 172 def test_parse_placement_specs_raises(test_input):
 173     with pytest.raises(SpecValidationError):
 174         PlacementSpec.from_string(test_input)
 175
 176 @pytest.mark.parametrize("test_input",
 177                          # wrong subnet
 178                          [("myhost:1.1.1.1/24"),
 179                           # wrong ip format
 180                           ("myhost:1"),
 181                           ])
 182 def test_parse_host_placement_specs_raises_wrong_format(test_input):
 183     with pytest.raises(ValueError):
 184         HostPlacementSpec.parse(test_input)
 185
 186
 187 @pytest.mark.parametrize(
 188     "p,hosts,size",
 189     [
 190         (
 191             PlacementSpec(count=3),
 192             ['host1', 'host2', 'host3', 'host4', 'host5'],
 193             3
 194         ),
 195         (
 196             PlacementSpec(host_pattern='*'),
 197             ['host1', 'host2', 'host3', 'host4', 'host5'],
 198             5
 199         ),
 200         (
 201             PlacementSpec(count_per_host=2, host_pattern='*'),
 202             ['host1', 'host2', 'host3', 'host4', 'host5'],
 203             10
 204         ),
 205         (
 206             PlacementSpec(host_pattern='foo*'),
 207             ['foo1', 'foo2', 'bar1', 'bar2'],
 208             2
 209         ),
 210         (
 211             PlacementSpec(count_per_host=2, host_pattern='foo*'),
 212             ['foo1', 'foo2', 'bar1', 'bar2'],
 213             4
 214         ),
 215     ])
 216 def test_placement_target_size(p, hosts, size):
 217     assert p.get_target_count(
 218         [HostPlacementSpec(n, '', '') for n in hosts]
 219     ) == size
 220
 221
 222 def _get_dict_spec(s_type, s_id):
 223     dict_spec = {
 224         "service_id": s_id,
 225         "service_type": s_type,
 226         "placement":
 227             dict(hosts=["host1:1.1.1.1"])
 228     }
 229     if s_type == 'nfs':
 230         pass
 231     elif s_type == 'iscsi':
 232         dict_spec['pool'] = 'pool'
 233         dict_spec['api_user'] = 'api_user'
 234         dict_spec['api_password'] = 'api_password'
 235     elif s_type == 'osd':
 236         dict_spec['spec'] = {
 237             'data_devices': {
 238                 'all': True
 239             }
 240         }
 241     elif s_type == 'rgw':
 242         dict_spec['rgw_realm'] = 'realm'
 243         dict_spec['rgw_zone'] = 'zone'
 244
 245     return dict_spec
 246
 247
 248 @pytest.mark.parametrize(
 249     "s_type,o_spec,s_id",
 250     [
 251         ("mgr", ServiceSpec, 'test'),
 252         ("mon", ServiceSpec, 'test'),
 253         ("mds", ServiceSpec, 'test'),
 254         ("rgw", RGWSpec, 'realm.zone'),
 255         ("nfs", NFSServiceSpec, 'test'),
 256         ("iscsi", IscsiServiceSpec, 'test'),
 257         ("osd", DriveGroupSpec, 'test'),
 258     ])
 259 def test_servicespec_map_test(s_type, o_spec, s_id):
 260     spec = ServiceSpec.from_json(_get_dict_spec(s_type, s_id))
 261     assert isinstance(spec, o_spec)
 262     assert isinstance(spec.placement, PlacementSpec)
 263     assert isinstance(spec.placement.hosts[0], HostPlacementSpec)
 264     assert spec.placement.hosts[0].hostname == 'host1'
 265     assert spec.placement.hosts[0].network == '1.1.1.1'
 266     assert spec.placement.hosts[0].name == ''
 267     assert spec.validate() is None
 268     ServiceSpec.from_json(spec.to_json())
 269
 270
 271 @pytest.mark.parametrize(
 272     "realm, zone, frontend_type, raise_exception, msg",
 273     [
 274         ('realm', 'zone1', 'beast', False, ''),
 275         ('realm', 'zone2', 'civetweb', False, ''),
 276         ('realm', None, 'beast', True, 'Cannot add RGW: Realm specified but no zone specified'),
 277         (None, 'zone1', 'beast', True, 'Cannot add RGW: Zone specified but no realm specified'),
 278         ('realm', 'zone', 'invalid-beast', True, '^Invalid rgw_frontend_type value'),
 279         ('realm', 'zone', 'invalid-civetweb', True, '^Invalid rgw_frontend_type value'),
 280     ])
 281 def test_rgw_servicespec_parse(realm, zone, frontend_type, raise_exception, msg):
 282     spec = RGWSpec(service_id='foo',
 283                    rgw_realm=realm,
 284                    rgw_zone=zone,
 285                    rgw_frontend_type=frontend_type)
 286     if raise_exception:
 287         with pytest.raises(SpecValidationError, match=msg):
 288             spec.validate()
 289     else:
 290         spec.validate()
 291
 292 def test_osd_unmanaged():
 293     osd_spec = {"placement": {"host_pattern": "*"},
 294                 "service_id": "all-available-devices",
 295                 "service_name": "osd.all-available-devices",
 296                 "service_type": "osd",
 297                 "spec": {"data_devices": {"all": True}, "filter_logic": "AND", "objectstore": "bluestore"},
 298                 "unmanaged": True}
 299
 300     dg_spec = ServiceSpec.from_json(osd_spec)
 301     assert dg_spec.unmanaged == True
 302
 303
 304 @pytest.mark.parametrize("y",
 305 """service_type: crash
 306 service_name: crash
 307 placement:
 308   host_pattern: '*'
 309 ---
 310 service_type: crash
 311 service_name: crash
 312 placement:
 313   host_pattern: '*'
 314 unmanaged: true
 315 ---
 316 service_type: rgw
 317 service_id: default-rgw-realm.eu-central-1.1
 318 service_name: rgw.default-rgw-realm.eu-central-1.1
 319 placement:
 320   hosts:
 321   - ceph-001
 322 networks:
 323 - 10.0.0.0/8
 324 - 192.168.0.0/16
 325 spec:
 326   rgw_frontend_type: civetweb
 327   rgw_realm: default-rgw-realm
 328   rgw_zone: eu-central-1
 329 ---
 330 service_type: osd
 331 service_id: osd_spec_default
 332 service_name: osd.osd_spec_default
 333 placement:
 334   host_pattern: '*'
 335 spec:
 336   data_devices:
 337     model: MC-55-44-XZ
 338   db_devices:
 339     model: SSD-123-foo
 340   filter_logic: AND
 341   objectstore: bluestore
 342   wal_devices:
 343     model: NVME-QQQQ-987
 344 ---
 345 service_type: alertmanager
 346 service_name: alertmanager
 347 spec:
 348   port: 1234
 349   user_data:
 350     default_webhook_urls:
 351     - foo
 352 ---
 353 service_type: grafana
 354 service_name: grafana
 355 spec:
 356   anonymous_access: true
 357   port: 1234
 358   protocol: https
 359 ---
 360 service_type: grafana
 361 service_name: grafana
 362 spec:
 363   anonymous_access: true
 364   initial_admin_password: secure
 365   port: 1234
 366   protocol: https
 367 ---
 368 service_type: ingress
 369 service_id: rgw.foo
 370 service_name: ingress.rgw.foo
 371 placement:
 372   hosts:
 373   - host1
 374   - host2
 375   - host3
 376 spec:
 377   backend_service: rgw.foo
 378   first_virtual_router_id: 50
 379   frontend_port: 8080
 380   monitor_port: 8081
 381   virtual_ip: 192.168.20.1/24
 382 ---
 383 service_type: nfs
 384 service_id: mynfs
 385 service_name: nfs.mynfs
 386 spec:
 387   port: 1234
 388 ---
 389 service_type: iscsi
 390 service_id: iscsi
 391 service_name: iscsi.iscsi
 392 networks:
 393 - ::0/8
 394 spec:
 395   api_password: admin
 396   api_port: 5000
 397   api_user: admin
 398   pool: pool
 399   trusted_ip_list:
 400   - ::1
 401   - ::2
 402 ---
 403 service_type: container
 404 service_id: hello-world
 405 service_name: container.hello-world
 406 spec:
 407   args:
 408   - --foo
 409   bind_mounts:
 410   - - type=bind
 411     - source=lib/modules
 412     - destination=/lib/modules
 413     - ro=true
 414   dirs:
 415   - foo
 416   - bar
 417   entrypoint: /usr/bin/bash
 418   envs:
 419   - FOO=0815
 420   files:
 421     bar.conf:
 422     - foo
 423     - bar
 424     foo.conf: 'foo
 425
 426       bar'
 427   gid: 2000
 428   image: docker.io/library/hello-world:latest
 429   ports:
 430   - 8080
 431   - 8443
 432   uid: 1000
 433   volume_mounts:
 434     foo: /foo
 435 ---
 436 service_type: snmp-gateway
 437 service_name: snmp-gateway
 438 placement:
 439   count: 1
 440 spec:
 441   credentials:
 442     snmp_community: public
 443   snmp_destination: 192.168.1.42:162
 444   snmp_version: V2c
 445 ---
 446 service_type: snmp-gateway
 447 service_name: snmp-gateway
 448 placement:
 449   count: 1
 450 spec:
 451   auth_protocol: MD5
 452   credentials:
 453     snmp_v3_auth_password: mypassword
 454     snmp_v3_auth_username: myuser
 455   engine_id: 8000C53F00000000
 456   port: 9464
 457   snmp_destination: 192.168.1.42:162
 458   snmp_version: V3
 459 ---
 460 service_type: snmp-gateway
 461 service_name: snmp-gateway
 462 placement:
 463   count: 1
 464 spec:
 465   credentials:
 466     snmp_v3_auth_password: mypassword
 467     snmp_v3_auth_username: myuser
 468     snmp_v3_priv_password: mysecret
 469   engine_id: 8000C53F00000000
 470   privacy_protocol: AES
 471   snmp_destination: 192.168.1.42:162
 472   snmp_version: V3
 473 """.split('---\n'))
 474 def test_yaml(y):
 475     data = yaml.safe_load(y)
 476     object = ServiceSpec.from_json(data)
 477
 478     assert yaml.dump(object) == y
 479     assert yaml.dump(ServiceSpec.from_json(object.to_json())) == y
 480
 481
 482 def test_alertmanager_spec_1():
 483     spec = AlertManagerSpec()
 484     assert spec.service_type == 'alertmanager'
 485     assert isinstance(spec.user_data, dict)
 486     assert len(spec.user_data.keys()) == 0
 487     assert spec.get_port_start() == [9093, 9094]
 488
 489
 490 def test_alertmanager_spec_2():
 491     spec = AlertManagerSpec(user_data={'default_webhook_urls': ['foo']})
 492     assert isinstance(spec.user_data, dict)
 493     assert 'default_webhook_urls' in spec.user_data.keys()
 494
 495
 496
 497 def test_repr():
 498     val = """ServiceSpec.from_json(yaml.safe_load('''service_type: crash
 499 service_name: crash
 500 placement:
 501   count: 42
 502 '''))"""
 503     obj = eval(val)
 504     assert obj.service_type == 'crash'
 505     assert val == repr(obj)
 506
 507 @pytest.mark.parametrize("spec1, spec2, eq",
 508                          [
 509                              (
 510                                      ServiceSpec(
 511                                          service_type='mon'
 512                                      ),
 513                                      ServiceSpec(
 514                                          service_type='mon'
 515                                      ),
 516                                      True
 517                              ),
 518                              (
 519                                      ServiceSpec(
 520                                          service_type='mon'
 521                                      ),
 522                                      ServiceSpec(
 523                                          service_type='mon',
 524                                          service_id='foo'
 525                                      ),
 526                                      True
 527                              ),
 528                              # Add service_type='mgr'
 529                              (
 530                                      ServiceSpec(
 531                                          service_type='osd'
 532                                      ),
 533                                      ServiceSpec(
 534                                          service_type='osd',
 535                                      ),
 536                                      True
 537                              ),
 538                              (
 539                                      ServiceSpec(
 540                                          service_type='osd'
 541                                      ),
 542                                      DriveGroupSpec(),
 543                                      True
 544                              ),
 545                              (
 546                                      ServiceSpec(
 547                                          service_type='osd'
 548                                      ),
 549                                      ServiceSpec(
 550                                          service_type='osd',
 551                                          service_id='foo',
 552                                      ),
 553                                      False
 554                              ),
 555                              (
 556                                      ServiceSpec(
 557                                          service_type='rgw',
 558                                          service_id='foo',
 559                                      ),
 560                                      RGWSpec(service_id='foo'),
 561                                      True
 562                              ),
 563                          ])
 564 def test_spec_hash_eq(spec1: ServiceSpec,
 565                       spec2: ServiceSpec,
 566                       eq: bool):
 567
 568     assert (spec1 == spec2) is eq
 569
 570 @pytest.mark.parametrize(
 571     "s_type,s_id,s_name",
 572     [
 573         ('mgr', 's_id', 'mgr'),
 574         ('mon', 's_id', 'mon'),
 575         ('mds', 's_id', 'mds.s_id'),
 576         ('rgw', 's_id', 'rgw.s_id'),
 577         ('nfs', 's_id', 'nfs.s_id'),
 578         ('iscsi', 's_id', 'iscsi.s_id'),
 579         ('osd', 's_id', 'osd.s_id'),
 580     ])
 581 def test_service_name(s_type, s_id, s_name):
 582     spec = ServiceSpec.from_json(_get_dict_spec(s_type, s_id))
 583     spec.validate()
 584     assert spec.service_name() == s_name
 585
 586 @pytest.mark.parametrize(
 587     's_type,s_id',
 588     [
 589         ('mds', 's:id'), # MDS service_id cannot contain an invalid char ':'
 590         ('mds', '1abc'), # MDS service_id cannot start with a numeric digit
 591         ('mds', ''),     # MDS service_id cannot be empty
 592         ('rgw', '*s_id'),
 593         ('nfs', 's/id'),
 594         ('iscsi', 's@id'),
 595         ('osd', 's;id'),
 596     ])
 597
 598 def test_service_id_raises_invalid_char(s_type, s_id):
 599     with pytest.raises(SpecValidationError):
 600         spec = ServiceSpec.from_json(_get_dict_spec(s_type, s_id))
 601         spec.validate()
 602
 603 def test_custom_container_spec():
 604     spec = CustomContainerSpec(service_id='hello-world',
 605                                image='docker.io/library/hello-world:latest',
 606                                entrypoint='/usr/bin/bash',
 607                                uid=1000,
 608                                gid=2000,
 609                                volume_mounts={'foo': '/foo'},
 610                                args=['--foo'],
 611                                envs=['FOO=0815'],
 612                                bind_mounts=[
 613                                    [
 614                                        'type=bind',
 615                                        'source=lib/modules',
 616                                        'destination=/lib/modules',
 617                                        'ro=true'
 618                                    ]
 619                                ],
 620                                ports=[8080, 8443],
 621                                dirs=['foo', 'bar'],
 622                                files={
 623                                    'foo.conf': 'foo\nbar',
 624                                    'bar.conf': ['foo', 'bar']
 625                                })
 626     assert spec.service_type == 'container'
 627     assert spec.entrypoint == '/usr/bin/bash'
 628     assert spec.uid == 1000
 629     assert spec.gid == 2000
 630     assert spec.volume_mounts == {'foo': '/foo'}
 631     assert spec.args == ['--foo']
 632     assert spec.envs == ['FOO=0815']
 633     assert spec.bind_mounts == [
 634         [
 635             'type=bind',
 636             'source=lib/modules',
 637             'destination=/lib/modules',
 638             'ro=true'
 639         ]
 640     ]
 641     assert spec.ports == [8080, 8443]
 642     assert spec.dirs == ['foo', 'bar']
 643     assert spec.files == {
 644         'foo.conf': 'foo\nbar',
 645         'bar.conf': ['foo', 'bar']
 646     }
 647
 648
 649 def test_custom_container_spec_config_json():
 650     spec = CustomContainerSpec(service_id='foo', image='foo', dirs=None)
 651     config_json = spec.config_json()
 652     for key in ['entrypoint', 'uid', 'gid', 'bind_mounts', 'dirs']:
 653         assert key not in config_json
 654
 655
 656 def test_ingress_spec():
 657     yaml_str = """service_type: ingress
 658 service_id: rgw.foo
 659 placement:
 660   hosts:
 661     - host1
 662     - host2
 663     - host3
 664 spec:
 665   virtual_ip: 192.168.20.1/24
 666   backend_service: rgw.foo
 667   frontend_port: 8080
 668   monitor_port: 8081
 669 """
 670     yaml_file = yaml.safe_load(yaml_str)
 671     spec = ServiceSpec.from_json(yaml_file)
 672     assert spec.service_type == "ingress"
 673     assert spec.service_id == "rgw.foo"
 674     assert spec.virtual_ip == "192.168.20.1/24"
 675     assert spec.frontend_port == 8080
 676     assert spec.monitor_port == 8081
 677
 678
 679 @pytest.mark.parametrize("y, error_match", [
 680     ("""
 681 service_type: rgw
 682 service_id: foo
 683 placement:
 684   count_per_host: "twelve"
 685 """, "count-per-host must be a numeric value",),
 686     ("""
 687 service_type: rgw
 688 service_id: foo
 689 placement:
 690   count_per_host: "2"
 691 """, "count-per-host must be an integer value",),
 692     ("""
 693 service_type: rgw
 694 service_id: foo
 695 placement:
 696   count_per_host: 7.36
 697 """, "count-per-host must be an integer value",),
 698     ("""
 699 service_type: rgw
 700 service_id: foo
 701 placement:
 702   count: "fifteen"
 703 """, "num/count must be a numeric value",),
 704     ("""
 705 service_type: rgw
 706 service_id: foo
 707 placement:
 708   count: "4"
 709 """, "num/count must be an integer value",),
 710     ("""
 711 service_type: rgw
 712 service_id: foo
 713 placement:
 714   count: 7.36
 715 """, "num/count must be an integer value",),
 716     ("""
 717 service_type: rgw
 718 service_id: foo
 719 placement:
 720   count: 0
 721 """, "num/count must be >= 1",),
 722     ("""
 723 service_type: rgw
 724 service_id: foo
 725 placement:
 726   count_per_host: 0
 727 """, "count-per-host must be >= 1",),
 728     ("""
 729 service_type: snmp-gateway
 730 service_name: snmp-gateway
 731 placement:
 732   count: 1
 733 spec:
 734   credentials:
 735     snmp_v3_auth_password: mypassword
 736     snmp_v3_auth_username: myuser
 737     snmp_v3_priv_password: mysecret
 738   port: 9464
 739   engine_id: 8000c53f0000000000
 740   privacy_protocol: WEIRD
 741   snmp_destination: 192.168.122.1:162
 742   auth_protocol: BIZARRE
 743   snmp_version: V3
 744 """, "auth_protocol unsupported. Must be one of MD5, SHA"),
 745     ("""
 746 ---
 747 service_type: snmp-gateway
 748 service_name: snmp-gateway
 749 placement:
 750   count: 1
 751 spec:
 752   credentials:
 753     snmp_community: public
 754   snmp_destination: 192.168.1.42:162
 755   snmp_version: V4
 756 """, 'snmp_version unsupported. Must be one of V2c, V3'),
 757         ("""
 758 ---
 759 service_type: snmp-gateway
 760 service_name: snmp-gateway
 761 placement:
 762   count: 1
 763 spec:
 764   credentials:
 765     snmp_community: public
 766   port: 9464
 767   snmp_destination: 192.168.1.42:162
 768 """, re.escape('Missing SNMP version (snmp_version)')),
 769         ("""
 770 ---
 771 service_type: snmp-gateway
 772 service_name: snmp-gateway
 773 placement:
 774   count: 1
 775 spec:
 776   credentials:
 777     snmp_v3_auth_username: myuser
 778     snmp_v3_auth_password: mypassword
 779   port: 9464
 780   auth_protocol: wah
 781   snmp_destination: 192.168.1.42:162
 782   snmp_version: V3
 783 """, 'auth_protocol unsupported. Must be one of MD5, SHA'),
 784         ("""
 785 ---
 786 service_type: snmp-gateway
 787 service_name: snmp-gateway
 788 placement:
 789   count: 1
 790 spec:
 791   credentials:
 792     snmp_v3_auth_username: myuser
 793     snmp_v3_auth_password: mypassword
 794     snmp_v3_priv_password: mysecret
 795   port: 9464
 796   auth_protocol: SHA
 797   privacy_protocol: weewah
 798   snmp_destination: 192.168.1.42:162
 799   snmp_version: V3
 800 """, 'privacy_protocol unsupported. Must be one of DES, AES'),
 801         ("""
 802 ---
 803 service_type: snmp-gateway
 804 service_name: snmp-gateway
 805 placement:
 806   count: 1
 807 spec:
 808   credentials:
 809     snmp_v3_auth_username: myuser
 810     snmp_v3_auth_password: mypassword
 811     snmp_v3_priv_password: mysecret
 812   port: 9464
 813   auth_protocol: SHA
 814   privacy_protocol: AES
 815   snmp_destination: 192.168.1.42:162
 816   snmp_version: V3
 817 """, 'Must provide an engine_id for SNMP V3 notifications'),
 818         ("""
 819 ---
 820 service_type: snmp-gateway
 821 service_name: snmp-gateway
 822 placement:
 823   count: 1
 824 spec:
 825   credentials:
 826     snmp_community: public
 827   port: 9464
 828   snmp_destination: 192.168.1.42
 829   snmp_version: V2c
 830 """, re.escape('SNMP destination (snmp_destination) type (IPv4) is invalid. Must be either: IPv4:Port, Name:Port')),
 831         ("""
 832 ---
 833 service_type: snmp-gateway
 834 service_name: snmp-gateway
 835 placement:
 836   count: 1
 837 spec:
 838   credentials:
 839     snmp_v3_auth_username: myuser
 840     snmp_v3_auth_password: mypassword
 841     snmp_v3_priv_password: mysecret
 842   port: 9464
 843   auth_protocol: SHA
 844   privacy_protocol: AES
 845   engine_id: bogus
 846   snmp_destination: 192.168.1.42:162
 847   snmp_version: V3
 848 """, 'engine_id must be a string containing 10-64 hex characters. Its length must be divisible by 2'),
 849         ("""
 850 ---
 851 service_type: snmp-gateway
 852 service_name: snmp-gateway
 853 placement:
 854   count: 1
 855 spec:
 856   credentials:
 857     snmp_v3_auth_username: myuser
 858     snmp_v3_auth_password: mypassword
 859   port: 9464
 860   auth_protocol: SHA
 861   engine_id: 8000C53F0000000000
 862   snmp_version: V3
 863 """, re.escape('SNMP destination (snmp_destination) must be provided')),
 864         ("""
 865 ---
 866 service_type: snmp-gateway
 867 service_name: snmp-gateway
 868 placement:
 869   count: 1
 870 spec:
 871   credentials:
 872     snmp_v3_auth_username: myuser
 873     snmp_v3_auth_password: mypassword
 874     snmp_v3_priv_password: mysecret
 875   port: 9464
 876   auth_protocol: SHA
 877   privacy_protocol: AES
 878   engine_id: 8000C53F0000000000
 879   snmp_destination: my.imaginary.snmp-host
 880   snmp_version: V3
 881 """, re.escape('SNMP destination (snmp_destination) is invalid: DNS lookup failed')),
 882         ("""
 883 ---
 884 service_type: snmp-gateway
 885 service_name: snmp-gateway
 886 placement:
 887   count: 1
 888 spec:
 889   credentials:
 890     snmp_v3_auth_username: myuser
 891     snmp_v3_auth_password: mypassword
 892     snmp_v3_priv_password: mysecret
 893   port: 9464
 894   auth_protocol: SHA
 895   privacy_protocol: AES
 896   engine_id: 8000C53F0000000000
 897   snmp_destination: 10.79.32.10:fred
 898   snmp_version: V3
 899 """, re.escape('SNMP destination (snmp_destination) is invalid: Port must be numeric')),
 900         ("""
 901 ---
 902 service_type: snmp-gateway
 903 service_name: snmp-gateway
 904 placement:
 905   count: 1
 906 spec:
 907   credentials:
 908     snmp_v3_auth_username: myuser
 909     snmp_v3_auth_password: mypassword
 910     snmp_v3_priv_password: mysecret
 911   port: 9464
 912   auth_protocol: SHA
 913   privacy_protocol: AES
 914   engine_id: 8000C53
 915   snmp_destination: 10.79.32.10:162
 916   snmp_version: V3
 917 """, 'engine_id must be a string containing 10-64 hex characters. Its length must be divisible by 2'),
 918         ("""
 919 ---
 920 service_type: snmp-gateway
 921 service_name: snmp-gateway
 922 placement:
 923   count: 1
 924 spec:
 925   credentials:
 926     snmp_v3_auth_username: myuser
 927     snmp_v3_auth_password: mypassword
 928     snmp_v3_priv_password: mysecret
 929   port: 9464
 930   auth_protocol: SHA
 931   privacy_protocol: AES
 932   engine_id: 8000C53DOH!
 933   snmp_destination: 10.79.32.10:162
 934   snmp_version: V3
 935 """, 'engine_id must be a string containing 10-64 hex characters. Its length must be divisible by 2'),
 936         ("""
 937 ---
 938 service_type: snmp-gateway
 939 service_name: snmp-gateway
 940 placement:
 941   count: 1
 942 spec:
 943   credentials:
 944     snmp_v3_auth_username: myuser
 945     snmp_v3_auth_password: mypassword
 946     snmp_v3_priv_password: mysecret
 947   port: 9464
 948   auth_protocol: SHA
 949   privacy_protocol: AES
 950   engine_id: 8000C53FCA7344403DC611EC9B985254002537A6C53FCA7344403DC6112537A60
 951   snmp_destination: 10.79.32.10:162
 952   snmp_version: V3
 953 """, 'engine_id must be a string containing 10-64 hex characters. Its length must be divisible by 2'),
 954         ("""
 955 ---
 956 service_type: snmp-gateway
 957 service_name: snmp-gateway
 958 placement:
 959   count: 1
 960 spec:
 961   credentials:
 962     snmp_v3_auth_username: myuser
 963     snmp_v3_auth_password: mypassword
 964     snmp_v3_priv_password: mysecret
 965   port: 9464
 966   auth_protocol: SHA
 967   privacy_protocol: AES
 968   engine_id: 8000C53F00000
 969   snmp_destination: 10.79.32.10:162
 970   snmp_version: V3
 971 """, 'engine_id must be a string containing 10-64 hex characters. Its length must be divisible by 2'),
 972     ])
 973 def test_service_spec_validation_error(y, error_match):
 974     data = yaml.safe_load(y)
 975     with pytest.raises(SpecValidationError) as err:
 976         specObj = ServiceSpec.from_json(data)
 977     assert err.match(error_match)
 978
 979
 980 @pytest.mark.parametrize("y, ec_args, ee_args, ec_final_args, ee_final_args", [
 981     pytest.param("""
 982 service_type: container
 983 service_id: hello-world
 984 service_name: container.hello-world
 985 spec:
 986   args:
 987   - --foo
 988   bind_mounts:
 989   - - type=bind
 990     - source=lib/modules
 991     - destination=/lib/modules
 992     - ro=true
 993   dirs:
 994   - foo
 995   - bar
 996   entrypoint: /usr/bin/bash
 997   envs:
 998   - FOO=0815
 999   files:
1000     bar.conf:
1001     - foo
1002     - bar
1003     foo.conf: 'foo
1004
1005       bar'
1006   gid: 2000
1007   image: docker.io/library/hello-world:latest
1008   ports:
1009   - 8080
1010   - 8443
1011   uid: 1000
1012   volume_mounts:
1013     foo: /foo
1014 """,
1015     None,
1016     None,
1017     None,
1018     None,
1019     id="no_extra_args"),
1020     pytest.param("""
1021 service_type: container
1022 service_id: hello-world
1023 service_name: container.hello-world
1024 spec:
1025   args:
1026   - --foo
1027   extra_entrypoint_args:
1028   - "--lasers=blue"
1029   - "--enable-confetti"
1030   bind_mounts:
1031   - - type=bind
1032     - source=lib/modules
1033     - destination=/lib/modules
1034     - ro=true
1035   dirs:
1036   - foo
1037   - bar
1038   entrypoint: /usr/bin/bash
1039   envs:
1040   - FOO=0815
1041   files:
1042     bar.conf:
1043     - foo
1044     - bar
1045     foo.conf: 'foo
1046
1047       bar'
1048   gid: 2000
1049   image: docker.io/library/hello-world:latest
1050   ports:
1051   - 8080
1052   - 8443
1053   uid: 1000
1054   volume_mounts:
1055     foo: /foo
1056 """,
1057     None,
1058     ["--lasers=blue", "--enable-confetti"],
1059     None,
1060     ["--lasers=blue", "--enable-confetti"],
1061     id="only_extra_entrypoint_args_spec"),
1062     pytest.param("""
1063 service_type: container
1064 service_id: hello-world
1065 service_name: container.hello-world
1066 spec:
1067   args:
1068   - --foo
1069   bind_mounts:
1070   - - type=bind
1071     - source=lib/modules
1072     - destination=/lib/modules
1073     - ro=true
1074   dirs:
1075   - foo
1076   - bar
1077   entrypoint: /usr/bin/bash
1078   envs:
1079   - FOO=0815
1080   files:
1081     bar.conf:
1082     - foo
1083     - bar
1084     foo.conf: 'foo
1085
1086       bar'
1087   gid: 2000
1088   image: docker.io/library/hello-world:latest
1089   ports:
1090   - 8080
1091   - 8443
1092   uid: 1000
1093   volume_mounts:
1094     foo: /foo
1095 extra_entrypoint_args:
1096 - "--lasers blue"
1097 - "--enable-confetti"
1098 """,
1099     None,
1100     ["--lasers blue", "--enable-confetti"],
1101     None,
1102     ["--lasers", "blue", "--enable-confetti"],
1103     id="only_extra_entrypoint_args_toplevel"),
1104     pytest.param("""
1105 service_type: nfs
1106 service_id: mynfs
1107 service_name: nfs.mynfs
1108 spec:
1109   port: 1234
1110   extra_entrypoint_args:
1111   - "--lasers=blue"
1112   - "--title=Custom NFS Options"
1113   extra_container_args:
1114   - "--cap-add=CAP_NET_BIND_SERVICE"
1115   - "--oom-score-adj=12"
1116 """,
1117     ["--cap-add=CAP_NET_BIND_SERVICE", "--oom-score-adj=12"],
1118     ["--lasers=blue", "--title=Custom NFS Options"],
1119     ["--cap-add=CAP_NET_BIND_SERVICE", "--oom-score-adj=12"],
1120     ["--lasers=blue", "--title=Custom", "NFS", "Options"],
1121     id="both_kinds_nfs"),
1122     pytest.param("""
1123 service_type: container
1124 service_id: hello-world
1125 service_name: container.hello-world
1126 spec:
1127   args:
1128   - --foo
1129   bind_mounts:
1130   - - type=bind
1131     - source=lib/modules
1132     - destination=/lib/modules
1133     - ro=true
1134   dirs:
1135   - foo
1136   - bar
1137   entrypoint: /usr/bin/bash
1138   envs:
1139   - FOO=0815
1140   files:
1141     bar.conf:
1142     - foo
1143     - bar
1144     foo.conf: 'foo
1145
1146       bar'
1147   gid: 2000
1148   image: docker.io/library/hello-world:latest
1149   ports:
1150   - 8080
1151   - 8443
1152   uid: 1000
1153   volume_mounts:
1154     foo: /foo
1155 extra_entrypoint_args:
1156 - argument: "--lasers=blue"
1157   split: true
1158 - argument: "--enable-confetti"
1159 """,
1160     None,
1161     [
1162         {"argument": "--lasers=blue", "split": True},
1163         {"argument": "--enable-confetti", "split": False},
1164     ],
1165     None,
1166     [
1167         "--lasers=blue",
1168         "--enable-confetti",
1169     ],
1170     id="only_extra_entrypoint_args_obj_toplevel"),
1171     pytest.param("""
1172 service_type: container
1173 service_id: hello-world
1174 service_name: container.hello-world
1175 spec:
1176   args:
1177   - --foo
1178   bind_mounts:
1179   - - type=bind
1180     - source=lib/modules
1181     - destination=/lib/modules
1182     - ro=true
1183   dirs:
1184   - foo
1185   - bar
1186   entrypoint: /usr/bin/bash
1187   envs:
1188   - FOO=0815
1189   files:
1190     bar.conf:
1191     - foo
1192     - bar
1193     foo.conf: 'foo
1194
1195       bar'
1196   gid: 2000
1197   image: docker.io/library/hello-world:latest
1198   ports:
1199   - 8080
1200   - 8443
1201   uid: 1000
1202   volume_mounts:
1203     foo: /foo
1204   extra_entrypoint_args:
1205   - argument: "--lasers=blue"
1206     split: true
1207   - argument: "--enable-confetti"
1208 """,
1209     None,
1210     [
1211         {"argument": "--lasers=blue", "split": True},
1212         {"argument": "--enable-confetti", "split": False},
1213     ],
1214     None,
1215     [
1216         "--lasers=blue",
1217         "--enable-confetti",
1218     ],
1219     id="only_extra_entrypoint_args_obj_indented"),
1220     pytest.param("""
1221 service_type: nfs
1222 service_id: mynfs
1223 service_name: nfs.mynfs
1224 spec:
1225   port: 1234
1226 extra_entrypoint_args:
1227 - argument: "--lasers=blue"
1228 - argument: "--title=Custom NFS Options"
1229 extra_container_args:
1230 - argument: "--cap-add=CAP_NET_BIND_SERVICE"
1231 - argument: "--oom-score-adj=12"
1232 """,
1233     [
1234         {"argument": "--cap-add=CAP_NET_BIND_SERVICE", "split": False},
1235         {"argument": "--oom-score-adj=12", "split": False},
1236     ],
1237     [
1238         {"argument": "--lasers=blue", "split": False},
1239         {"argument": "--title=Custom NFS Options", "split": False},
1240     ],
1241     [
1242         "--cap-add=CAP_NET_BIND_SERVICE",
1243         "--oom-score-adj=12",
1244     ],
1245     [
1246         "--lasers=blue",
1247         "--title=Custom NFS Options",
1248     ],
1249     id="both_kinds_obj_nfs"),
1250 ])
1251 def test_extra_args_handling(y, ec_args, ee_args, ec_final_args, ee_final_args):
1252     data = yaml.safe_load(y)
1253     spec_obj = ServiceSpec.from_json(data)
1254
1255     assert ArgumentSpec.map_json(spec_obj.extra_container_args) == ec_args
1256     assert ArgumentSpec.map_json(spec_obj.extra_entrypoint_args) == ee_args
1257     if ec_final_args is None:
1258         assert spec_obj.extra_container_args is None
1259     else:
1260         ec_res = []
1261         for args in spec_obj.extra_container_args:
1262             ec_res.extend(args.to_args())
1263         assert ec_res == ec_final_args
1264     if ee_final_args is None:
1265         assert spec_obj.extra_entrypoint_args is None
1266     else:
1267         ee_res = []
1268         for args in spec_obj.extra_entrypoint_args:
1269             ee_res.extend(args.to_args())
1270         assert ee_res == ee_final_args