ceph/src/python-common/ceph/deployment/service_spec.py

   1 import fnmatch
   2 import os
   3 import re
   4 import enum
   5 from collections import OrderedDict
   6 from contextlib import contextmanager
   7 from functools import wraps
   8 from ipaddress import ip_network, ip_address
   9 from typing import Optional, Dict, Any, List, Union, Callable, Iterable, Type, TypeVar, cast, \
  10     NamedTuple, Mapping, Iterator
  11
  12 import yaml
  13
  14 from ceph.deployment.hostspec import HostSpec, SpecValidationError, assert_valid_host
  15 from ceph.deployment.utils import unwrap_ipv6, valid_addr
  16 from ceph.utils import is_hex
  17
  18 ServiceSpecT = TypeVar('ServiceSpecT', bound='ServiceSpec')
  19 FuncT = TypeVar('FuncT', bound=Callable)
  20
  21
  22 def handle_type_error(method: FuncT) -> FuncT:
  23     @wraps(method)
  24     def inner(cls: Any, *args: Any, **kwargs: Any) -> Any:
  25         try:
  26             return method(cls, *args, **kwargs)
  27         except (TypeError, AttributeError) as e:
  28             error_msg = '{}: {}'.format(cls.__name__, e)
  29             raise SpecValidationError(error_msg)
  30     return cast(FuncT, inner)
  31
  32
  33 class HostPlacementSpec(NamedTuple):
  34     hostname: str
  35     network: str
  36     name: str
  37
  38     def __str__(self) -> str:
  39         res = ''
  40         res += self.hostname
  41         if self.network:
  42             res += ':' + self.network
  43         if self.name:
  44             res += '=' + self.name
  45         return res
  46
  47     @classmethod
  48     @handle_type_error
  49     def from_json(cls, data: Union[dict, str]) -> 'HostPlacementSpec':
  50         if isinstance(data, str):
  51             return cls.parse(data)
  52         return cls(**data)
  53
  54     def to_json(self) -> str:
  55         return str(self)
  56
  57     @classmethod
  58     def parse(cls, host, require_network=True):
  59         # type: (str, bool) -> HostPlacementSpec
  60         """
  61         Split host into host, network, and (optional) daemon name parts.  The network
  62         part can be an IP, CIDR, or ceph addrvec like '[v2:1.2.3.4:3300,v1:1.2.3.4:6789]'.
  63         e.g.,
  64           "myhost"
  65           "myhost=name"
  66           "myhost:1.2.3.4"
  67           "myhost:1.2.3.4=name"
  68           "myhost:1.2.3.0/24"
  69           "myhost:1.2.3.0/24=name"
  70           "myhost:[v2:1.2.3.4:3000]=name"
  71           "myhost:[v2:1.2.3.4:3000,v1:1.2.3.4:6789]=name"
  72         """
  73         # Matches from start to : or = or until end of string
  74         host_re = r'^(.*?)(:|=|$)'
  75         # Matches from : to = or until end of string
  76         ip_re = r':(.*?)(=|$)'
  77         # Matches from = to end of string
  78         name_re = r'=(.*?)$'
  79
  80         # assign defaults
  81         host_spec = cls('', '', '')
  82
  83         match_host = re.search(host_re, host)
  84         if match_host:
  85             host_spec = host_spec._replace(hostname=match_host.group(1))
  86
  87         name_match = re.search(name_re, host)
  88         if name_match:
  89             host_spec = host_spec._replace(name=name_match.group(1))
  90
  91         ip_match = re.search(ip_re, host)
  92         if ip_match:
  93             host_spec = host_spec._replace(network=ip_match.group(1))
  94
  95         if not require_network:
  96             return host_spec
  97
  98         networks = list()  # type: List[str]
  99         network = host_spec.network
 100         # in case we have [v2:1.2.3.4:3000,v1:1.2.3.4:6478]
 101         if ',' in network:
 102             networks = [x for x in network.split(',')]
 103         else:
 104             if network != '':
 105                 networks.append(network)
 106
 107         for network in networks:
 108             # only if we have versioned network configs
 109             if network.startswith('v') or network.startswith('[v'):
 110                 # if this is ipv6 we can't just simply split on ':' so do
 111                 # a split once and rsplit once to leave us with just ipv6 addr
 112                 network = network.split(':', 1)[1]
 113                 network = network.rsplit(':', 1)[0]
 114             try:
 115                 # if subnets are defined, also verify the validity
 116                 if '/' in network:
 117                     ip_network(network)
 118                 else:
 119                     ip_address(unwrap_ipv6(network))
 120             except ValueError as e:
 121                 # logging?
 122                 raise e
 123         host_spec.validate()
 124         return host_spec
 125
 126     def validate(self) -> None:
 127         assert_valid_host(self.hostname)
 128
 129
 130 class PlacementSpec(object):
 131     """
 132     For APIs that need to specify a host subset
 133     """
 134
 135     def __init__(self,
 136                  label=None,  # type: Optional[str]
 137                  hosts=None,  # type: Union[List[str],List[HostPlacementSpec], None]
 138                  count=None,  # type: Optional[int]
 139                  count_per_host=None,  # type: Optional[int]
 140                  host_pattern=None,  # type: Optional[str]
 141                  ):
 142         # type: (...) -> None
 143         self.label = label
 144         self.hosts = []  # type: List[HostPlacementSpec]
 145
 146         if hosts:
 147             self.set_hosts(hosts)
 148
 149         self.count = count  # type: Optional[int]
 150         self.count_per_host = count_per_host   # type: Optional[int]
 151
 152         #: fnmatch patterns to select hosts. Can also be a single host.
 153         self.host_pattern = host_pattern  # type: Optional[str]
 154
 155         self.validate()
 156
 157     def is_empty(self) -> bool:
 158         return (
 159             self.label is None
 160             and not self.hosts
 161             and not self.host_pattern
 162             and self.count is None
 163             and self.count_per_host is None
 164         )
 165
 166     def __eq__(self, other: Any) -> bool:
 167         if isinstance(other, PlacementSpec):
 168             return self.label == other.label \
 169                    and self.hosts == other.hosts \
 170                    and self.count == other.count \
 171                    and self.host_pattern == other.host_pattern \
 172                    and self.count_per_host == other.count_per_host
 173         return NotImplemented
 174
 175     def set_hosts(self, hosts: Union[List[str], List[HostPlacementSpec]]) -> None:
 176         # To backpopulate the .hosts attribute when using labels or count
 177         # in the orchestrator backend.
 178         if all([isinstance(host, HostPlacementSpec) for host in hosts]):
 179             self.hosts = hosts  # type: ignore
 180         else:
 181             self.hosts = [HostPlacementSpec.parse(x, require_network=False)  # type: ignore
 182                           for x in hosts if x]
 183
 184     # deprecated
 185     def filter_matching_hosts(self, _get_hosts_func: Callable) -> List[str]:
 186         return self.filter_matching_hostspecs(_get_hosts_func(as_hostspec=True))
 187
 188     def filter_matching_hostspecs(self, hostspecs: Iterable[HostSpec]) -> List[str]:
 189         if self.hosts:
 190             all_hosts = [hs.hostname for hs in hostspecs]
 191             return [h.hostname for h in self.hosts if h.hostname in all_hosts]
 192         if self.label:
 193             return [hs.hostname for hs in hostspecs if self.label in hs.labels]
 194         all_hosts = [hs.hostname for hs in hostspecs]
 195         if self.host_pattern:
 196             return fnmatch.filter(all_hosts, self.host_pattern)
 197         return all_hosts
 198
 199     def get_target_count(self, hostspecs: Iterable[HostSpec]) -> int:
 200         if self.count:
 201             return self.count
 202         return len(self.filter_matching_hostspecs(hostspecs)) * (self.count_per_host or 1)
 203
 204     def pretty_str(self) -> str:
 205         """
 206         >>> #doctest: +SKIP
 207         ... ps = PlacementSpec(...)  # For all placement specs:
 208         ... PlacementSpec.from_string(ps.pretty_str()) == ps
 209         """
 210         kv = []
 211         if self.hosts:
 212             kv.append(';'.join([str(h) for h in self.hosts]))
 213         if self.count:
 214             kv.append('count:%d' % self.count)
 215         if self.count_per_host:
 216             kv.append('count-per-host:%d' % self.count_per_host)
 217         if self.label:
 218             kv.append('label:%s' % self.label)
 219         if self.host_pattern:
 220             kv.append(self.host_pattern)
 221         return ';'.join(kv)
 222
 223     def __repr__(self) -> str:
 224         kv = []
 225         if self.count:
 226             kv.append('count=%d' % self.count)
 227         if self.count_per_host:
 228             kv.append('count_per_host=%d' % self.count_per_host)
 229         if self.label:
 230             kv.append('label=%s' % repr(self.label))
 231         if self.hosts:
 232             kv.append('hosts={!r}'.format(self.hosts))
 233         if self.host_pattern:
 234             kv.append('host_pattern={!r}'.format(self.host_pattern))
 235         return "PlacementSpec(%s)" % ', '.join(kv)
 236
 237     @classmethod
 238     @handle_type_error
 239     def from_json(cls, data: dict) -> 'PlacementSpec':
 240         c = data.copy()
 241         hosts = c.get('hosts', [])
 242         if hosts:
 243             c['hosts'] = []
 244             for host in hosts:
 245                 c['hosts'].append(HostPlacementSpec.from_json(host))
 246         _cls = cls(**c)
 247         _cls.validate()
 248         return _cls
 249
 250     def to_json(self) -> dict:
 251         r: Dict[str, Any] = {}
 252         if self.label:
 253             r['label'] = self.label
 254         if self.hosts:
 255             r['hosts'] = [host.to_json() for host in self.hosts]
 256         if self.count:
 257             r['count'] = self.count
 258         if self.count_per_host:
 259             r['count_per_host'] = self.count_per_host
 260         if self.host_pattern:
 261             r['host_pattern'] = self.host_pattern
 262         return r
 263
 264     def validate(self) -> None:
 265         if self.hosts and self.label:
 266             # TODO: a less generic Exception
 267             raise SpecValidationError('Host and label are mutually exclusive')
 268         if self.count is not None:
 269             try:
 270                 intval = int(self.count)
 271             except (ValueError, TypeError):
 272                 raise SpecValidationError("num/count must be a numeric value")
 273             if self.count != intval:
 274                 raise SpecValidationError("num/count must be an integer value")
 275             if self.count < 1:
 276                 raise SpecValidationError("num/count must be >= 1")
 277         if self.count_per_host is not None:
 278             try:
 279                 intval = int(self.count_per_host)
 280             except (ValueError, TypeError):
 281                 raise SpecValidationError("count-per-host must be a numeric value")
 282             if self.count_per_host != intval:
 283                 raise SpecValidationError("count-per-host must be an integer value")
 284             if self.count_per_host < 1:
 285                 raise SpecValidationError("count-per-host must be >= 1")
 286         if self.count_per_host is not None and not (
 287                 self.label
 288                 or self.hosts
 289                 or self.host_pattern
 290         ):
 291             raise SpecValidationError(
 292                 "count-per-host must be combined with label or hosts or host_pattern"
 293             )
 294         if self.count is not None and self.count_per_host is not None:
 295             raise SpecValidationError("cannot combine count and count-per-host")
 296         if (
 297                 self.count_per_host is not None
 298                 and self.hosts
 299                 and any([hs.network or hs.name for hs in self.hosts])
 300         ):
 301             raise SpecValidationError(
 302                 "count-per-host cannot be combined explicit placement with names or networks"
 303             )
 304         if self.host_pattern:
 305             if not isinstance(self.host_pattern, str):
 306                 raise SpecValidationError('host_pattern must be of type string')
 307             if self.hosts:
 308                 raise SpecValidationError('cannot combine host patterns and hosts')
 309
 310         for h in self.hosts:
 311             h.validate()
 312
 313     @classmethod
 314     def from_string(cls, arg):
 315         # type: (Optional[str]) -> PlacementSpec
 316         """
 317         A single integer is parsed as a count:
 318
 319         >>> PlacementSpec.from_string('3')
 320         PlacementSpec(count=3)
 321
 322         A list of names is parsed as host specifications:
 323
 324         >>> PlacementSpec.from_string('host1 host2')
 325         PlacementSpec(hosts=[HostPlacementSpec(hostname='host1', network='', name=''), HostPlacemen\
 326 tSpec(hostname='host2', network='', name='')])
 327
 328         You can also prefix the hosts with a count as follows:
 329
 330         >>> PlacementSpec.from_string('2 host1 host2')
 331         PlacementSpec(count=2, hosts=[HostPlacementSpec(hostname='host1', network='', name=''), Hos\
 332 tPlacementSpec(hostname='host2', network='', name='')])
 333
 334         You can specify labels using `label:<label>`
 335
 336         >>> PlacementSpec.from_string('label:mon')
 337         PlacementSpec(label='mon')
 338
 339         Labels also support a count:
 340
 341         >>> PlacementSpec.from_string('3 label:mon')
 342         PlacementSpec(count=3, label='mon')
 343
 344         fnmatch is also supported:
 345
 346         >>> PlacementSpec.from_string('data[1-3]')
 347         PlacementSpec(host_pattern='data[1-3]')
 348
 349         >>> PlacementSpec.from_string(None)
 350         PlacementSpec()
 351         """
 352         if arg is None or not arg:
 353             strings = []
 354         elif isinstance(arg, str):
 355             if ' ' in arg:
 356                 strings = arg.split(' ')
 357             elif ';' in arg:
 358                 strings = arg.split(';')
 359             elif ',' in arg and '[' not in arg:
 360                 # FIXME: this isn't quite right.  we want to avoid breaking
 361                 # a list of mons with addrvecs... so we're basically allowing
 362                 # , most of the time, except when addrvecs are used.  maybe
 363                 # ok?
 364                 strings = arg.split(',')
 365             else:
 366                 strings = [arg]
 367         else:
 368             raise SpecValidationError('invalid placement %s' % arg)
 369
 370         count = None
 371         count_per_host = None
 372         if strings:
 373             try:
 374                 count = int(strings[0])
 375                 strings = strings[1:]
 376             except ValueError:
 377                 pass
 378         for s in strings:
 379             if s.startswith('count:'):
 380                 try:
 381                     count = int(s[len('count:'):])
 382                     strings.remove(s)
 383                     break
 384                 except ValueError:
 385                     pass
 386         for s in strings:
 387             if s.startswith('count-per-host:'):
 388                 try:
 389                     count_per_host = int(s[len('count-per-host:'):])
 390                     strings.remove(s)
 391                     break
 392                 except ValueError:
 393                     pass
 394
 395         advanced_hostspecs = [h for h in strings if
 396                               (':' in h or '=' in h or not any(c in '[]?*:=' for c in h)) and
 397                               'label:' not in h]
 398         for a_h in advanced_hostspecs:
 399             strings.remove(a_h)
 400
 401         labels = [x for x in strings if 'label:' in x]
 402         if len(labels) > 1:
 403             raise SpecValidationError('more than one label provided: {}'.format(labels))
 404         for l in labels:
 405             strings.remove(l)
 406         label = labels[0][6:] if labels else None
 407
 408         host_patterns = strings
 409         if len(host_patterns) > 1:
 410             raise SpecValidationError(
 411                 'more than one host pattern provided: {}'.format(host_patterns))
 412
 413         ps = PlacementSpec(count=count,
 414                            count_per_host=count_per_host,
 415                            hosts=advanced_hostspecs,
 416                            label=label,
 417                            host_pattern=host_patterns[0] if host_patterns else None)
 418         return ps
 419
 420
 421 _service_spec_from_json_validate = True
 422
 423
 424 class CustomConfig:
 425     """
 426     Class to specify custom config files to be mounted in daemon's container
 427     """
 428
 429     _fields = ['content', 'mount_path']
 430
 431     def __init__(self, content: str, mount_path: str) -> None:
 432         self.content: str = content
 433         self.mount_path: str = mount_path
 434         self.validate()
 435
 436     def to_json(self) -> Dict[str, Any]:
 437         return {
 438             'content': self.content,
 439             'mount_path': self.mount_path,
 440         }
 441
 442     @classmethod
 443     def from_json(cls, data: Dict[str, Any]) -> "CustomConfig":
 444         for k in cls._fields:
 445             if k not in data:
 446                 raise SpecValidationError(f'CustomConfig must have "{k}" field')
 447         for k in data.keys():
 448             if k not in cls._fields:
 449                 raise SpecValidationError(f'CustomConfig got unknown field "{k}"')
 450         return cls(**data)
 451
 452     @property
 453     def filename(self) -> str:
 454         return os.path.basename(self.mount_path)
 455
 456     def __eq__(self, other: Any) -> bool:
 457         if isinstance(other, CustomConfig):
 458             return (
 459                 self.content == other.content
 460                 and self.mount_path == other.mount_path
 461             )
 462         return NotImplemented
 463
 464     def __repr__(self) -> str:
 465         return f'CustomConfig({self.mount_path})'
 466
 467     def validate(self) -> None:
 468         if not isinstance(self.content, str):
 469             raise SpecValidationError(
 470                     f'CustomConfig content must be a string. Got {type(self.content)}')
 471         if not isinstance(self.mount_path, str):
 472             raise SpecValidationError(
 473                     f'CustomConfig content must be a string. Got {type(self.mount_path)}')
 474
 475
 476 @contextmanager
 477 def service_spec_allow_invalid_from_json() -> Iterator[None]:
 478     """
 479     I know this is evil, but unfortunately `ceph orch ls`
 480     may return invalid OSD specs for OSDs not associated to
 481     and specs. If you have a better idea, please!
 482     """
 483     global _service_spec_from_json_validate
 484     _service_spec_from_json_validate = False
 485     yield
 486     _service_spec_from_json_validate = True
 487
 488
 489 class ArgumentSpec:
 490     """The ArgumentSpec type represents an argument that can be
 491     passed to an underyling subsystem, like a container engine or
 492     another command line tool.
 493
 494     The ArgumentSpec aims to be backwards compatible with the previous
 495     form of argument, a single string. The string was always assumed
 496     to be indentended to be split on spaces. For example:
 497     `--cpus 8` becomes `["--cpus", "8"]`. This type is converted from
 498     either a string or an json/yaml object. In the object form you
 499     can choose if the string part should be split so an argument like
 500     `--migrate-from=//192.168.5.22/My Documents` can be expressed.
 501     """
 502     _fields = ['argument', 'split']
 503
 504     class OriginalType(enum.Enum):
 505         OBJECT = 0
 506         STRING = 1
 507
 508     def __init__(
 509         self,
 510         argument: str,
 511         split: bool = False,
 512         *,
 513         origin: OriginalType = OriginalType.OBJECT,
 514     ) -> None:
 515         self.argument = argument
 516         self.split = bool(split)
 517         # origin helps with round-tripping between inputs that
 518         # are simple strings or objects (dicts)
 519         self._origin = origin
 520         self.validate()
 521
 522     def to_json(self) -> Union[str, Dict[str, Any]]:
 523         """Return a json-safe represenation of the ArgumentSpec."""
 524         if self._origin == self.OriginalType.STRING:
 525             return self.argument
 526         return {
 527             'argument': self.argument,
 528             'split': self.split,
 529         }
 530
 531     def to_args(self) -> List[str]:
 532         """Convert this ArgumentSpec into a list of arguments suitable for
 533         adding to an argv-style command line.
 534         """
 535         if not self.split:
 536             return [self.argument]
 537         return [part for part in self.argument.split(" ") if part]
 538
 539     def __eq__(self, other: Any) -> bool:
 540         if isinstance(other, ArgumentSpec):
 541             return (
 542                 self.argument == other.argument
 543                 and self.split == other.split
 544             )
 545         if isinstance(other, object):
 546             # This is a workaround for silly ceph mgr object/type identity
 547             # mismatches due to multiple python interpreters in use.
 548             try:
 549                 argument = getattr(other, 'argument')
 550                 split = getattr(other, 'split')
 551                 return (self.argument == argument and self.split == split)
 552             except AttributeError:
 553                 pass
 554         return NotImplemented
 555
 556     def __repr__(self) -> str:
 557         return f'ArgumentSpec({self.argument!r}, {self.split!r})'
 558
 559     def validate(self) -> None:
 560         if not isinstance(self.argument, str):
 561             raise SpecValidationError(
 562                     f'ArgumentSpec argument must be a string. Got {type(self.argument)}')
 563         if not isinstance(self.split, bool):
 564             raise SpecValidationError(
 565                     f'ArgumentSpec split must be a boolean. Got {type(self.split)}')
 566
 567     @classmethod
 568     def from_json(cls, data: Union[str, Dict[str, Any]]) -> "ArgumentSpec":
 569         """Convert a json-object (dict) to an ArgumentSpec."""
 570         if isinstance(data, str):
 571             return cls(data, split=True, origin=cls.OriginalType.STRING)
 572         if 'argument' not in data:
 573             raise SpecValidationError(f'ArgumentSpec must have an "argument" field')
 574         for k in data.keys():
 575             if k not in cls._fields:
 576                 raise SpecValidationError(f'ArgumentSpec got an unknown field {k!r}')
 577         return cls(**data)
 578
 579     @staticmethod
 580     def map_json(
 581         values: Optional["ArgumentList"]
 582     ) -> Optional[List[Union[str, Dict[str, Any]]]]:
 583         """Given a list of ArgumentSpec objects return a json-safe
 584         representation.of them."""
 585         if values is None:
 586             return None
 587         return [v.to_json() for v in values]
 588
 589     @classmethod
 590     def from_general_args(cls, data: "GeneralArgList") -> "ArgumentList":
 591         """Convert a list of strs, dicts, or existing ArgumentSpec objects
 592         to a list of only ArgumentSpec objects.
 593         """
 594         out: ArgumentList = []
 595         for item in data:
 596             if isinstance(item, (str, dict)):
 597                 out.append(cls.from_json(item))
 598             elif isinstance(item, cls):
 599                 out.append(item)
 600             elif hasattr(item, 'to_json'):
 601                 # This is a workaround for silly ceph mgr object/type identity
 602                 # mismatches due to multiple python interpreters in use.
 603                 # It should be safe because we already have to be able to
 604                 # round-trip between json/yaml.
 605                 out.append(cls.from_json(item.to_json()))
 606             else:
 607                 raise SpecValidationError(f"Unknown type for argument: {type(item)}")
 608         return out
 609
 610
 611 ArgumentList = List[ArgumentSpec]
 612 GeneralArgList = List[Union[str, Dict[str, Any], "ArgumentSpec"]]
 613
 614
 615 class ServiceSpec(object):
 616     """
 617     Details of service creation.
 618
 619     Request to the orchestrator for a cluster of daemons
 620     such as MDS, RGW, iscsi gateway, nvmeof gateway, MONs, MGRs, Prometheus
 621
 622     This structure is supposed to be enough information to
 623     start the services.
 624     """
 625     KNOWN_SERVICE_TYPES = 'alertmanager crash grafana iscsi nvmeof loki promtail mds mgr mon nfs ' \
 626                           'node-exporter osd prometheus rbd-mirror rgw agent ceph-exporter ' \
 627                           'container ingress cephfs-mirror snmp-gateway jaeger-tracing ' \
 628                           'elasticsearch jaeger-agent jaeger-collector jaeger-query'.split()
 629     REQUIRES_SERVICE_ID = 'iscsi nvmeof mds nfs rgw container ingress '.split()
 630     MANAGED_CONFIG_OPTIONS = [
 631         'mds_join_fs',
 632     ]
 633
 634     @classmethod
 635     def _cls(cls: Type[ServiceSpecT], service_type: str) -> Type[ServiceSpecT]:
 636         from ceph.deployment.drive_group import DriveGroupSpec
 637
 638         ret = {
 639             'mon': MONSpec,
 640             'rgw': RGWSpec,
 641             'nfs': NFSServiceSpec,
 642             'osd': DriveGroupSpec,
 643             'mds': MDSSpec,
 644             'iscsi': IscsiServiceSpec,
 645             'nvmeof': NvmeofServiceSpec,
 646             'alertmanager': AlertManagerSpec,
 647             'ingress': IngressSpec,
 648             'container': CustomContainerSpec,
 649             'grafana': GrafanaSpec,
 650             'node-exporter': MonitoringSpec,
 651             'ceph-exporter': CephExporterSpec,
 652             'prometheus': PrometheusSpec,
 653             'loki': MonitoringSpec,
 654             'promtail': MonitoringSpec,
 655             'snmp-gateway': SNMPGatewaySpec,
 656             'elasticsearch': TracingSpec,
 657             'jaeger-agent': TracingSpec,
 658             'jaeger-collector': TracingSpec,
 659             'jaeger-query': TracingSpec,
 660             'jaeger-tracing': TracingSpec,
 661         }.get(service_type, cls)
 662         if ret == ServiceSpec and not service_type:
 663             raise SpecValidationError('Spec needs a "service_type" key.')
 664         return ret
 665
 666     def __new__(cls: Type[ServiceSpecT], *args: Any, **kwargs: Any) -> ServiceSpecT:
 667         """
 668         Some Python foo to make sure, we don't have an object
 669         like `ServiceSpec('rgw')` of type `ServiceSpec`. Now we have:
 670
 671         >>> type(ServiceSpec('rgw')) == type(RGWSpec('rgw'))
 672         True
 673
 674         """
 675         if cls != ServiceSpec:
 676             return object.__new__(cls)
 677         service_type = kwargs.get('service_type', args[0] if args else None)
 678         sub_cls: Any = cls._cls(service_type)
 679         return object.__new__(sub_cls)
 680
 681     def __init__(self,
 682                  service_type: str,
 683                  service_id: Optional[str] = None,
 684                  placement: Optional[PlacementSpec] = None,
 685                  count: Optional[int] = None,
 686                  config: Optional[Dict[str, str]] = None,
 687                  unmanaged: bool = False,
 688                  preview_only: bool = False,
 689                  networks: Optional[List[str]] = None,
 690                  extra_container_args: Optional[GeneralArgList] = None,
 691                  extra_entrypoint_args: Optional[GeneralArgList] = None,
 692                  custom_configs: Optional[List[CustomConfig]] = None,
 693                  ):
 694
 695         #: See :ref:`orchestrator-cli-placement-spec`.
 696         self.placement = PlacementSpec() if placement is None else placement  # type: PlacementSpec
 697
 698         assert service_type in ServiceSpec.KNOWN_SERVICE_TYPES, service_type
 699         #: The type of the service. Needs to be either a Ceph
 700         #: service (``mon``, ``crash``, ``mds``, ``mgr``, ``osd`` or
 701         #: ``rbd-mirror``), a gateway (``nfs`` or ``rgw``), part of the
 702         #: monitoring stack (``alertmanager``, ``grafana``, ``node-exporter`` or
 703         #: ``prometheus``) or (``container``) for custom containers.
 704         self.service_type = service_type
 705
 706         #: The name of the service. Required for ``iscsi``, ``nvmeof``, ``mds``, ``nfs``, ``osd``,
 707         #: ``rgw``, ``container``, ``ingress``
 708         self.service_id = None
 709
 710         if self.service_type in self.REQUIRES_SERVICE_ID or self.service_type == 'osd':
 711             self.service_id = service_id
 712
 713         #: If set to ``true``, the orchestrator will not deploy nor remove
 714         #: any daemon associated with this service. Placement and all other properties
 715         #: will be ignored. This is useful, if you do not want this service to be
 716         #: managed temporarily. For cephadm, See :ref:`cephadm-spec-unmanaged`
 717         self.unmanaged = unmanaged
 718         self.preview_only = preview_only
 719
 720         #: A list of network identities instructing the daemons to only bind
 721         #: on the particular networks in that list. In case the cluster is distributed
 722         #: across multiple networks, you can add multiple networks. See
 723         #: :ref:`cephadm-monitoring-networks-ports`,
 724         #: :ref:`cephadm-rgw-networks` and :ref:`cephadm-mgr-networks`.
 725         self.networks: List[str] = networks or []
 726
 727         self.config: Optional[Dict[str, str]] = None
 728         if config:
 729             self.config = {k.replace(' ', '_'): v for k, v in config.items()}
 730
 731         self.extra_container_args: Optional[ArgumentList] = None
 732         self.extra_entrypoint_args: Optional[ArgumentList] = None
 733         if extra_container_args:
 734             self.extra_container_args = ArgumentSpec.from_general_args(
 735                 extra_container_args)
 736         if extra_entrypoint_args:
 737             self.extra_entrypoint_args = ArgumentSpec.from_general_args(
 738                 extra_entrypoint_args)
 739         self.custom_configs: Optional[List[CustomConfig]] = custom_configs
 740
 741     @classmethod
 742     @handle_type_error
 743     def from_json(cls: Type[ServiceSpecT], json_spec: Dict) -> ServiceSpecT:
 744         """
 745         Initialize 'ServiceSpec' object data from a json structure
 746
 747         There are two valid styles for service specs:
 748
 749         the "old" style:
 750
 751         .. code:: yaml
 752
 753             service_type: nfs
 754             service_id: foo
 755             pool: mypool
 756             namespace: myns
 757
 758         and the "new" style:
 759
 760         .. code:: yaml
 761
 762             service_type: nfs
 763             service_id: foo
 764             config:
 765               some_option: the_value
 766             networks: [10.10.0.0/16]
 767             spec:
 768               pool: mypool
 769               namespace: myns
 770
 771         In https://tracker.ceph.com/issues/45321 we decided that we'd like to
 772         prefer the new style as it is more readable and provides a better
 773         understanding of what fields are special for a give service type.
 774
 775         Note, we'll need to stay compatible with both versions for the
 776         the next two major releases (octopus, pacific).
 777
 778         :param json_spec: A valid dict with ServiceSpec
 779
 780         :meta private:
 781         """
 782         if not isinstance(json_spec, dict):
 783             raise SpecValidationError(
 784                 f'Service Spec is not an (JSON or YAML) object. got "{str(json_spec)}"')
 785
 786         json_spec = cls.normalize_json(json_spec)
 787
 788         c = json_spec.copy()
 789
 790         # kludge to make `from_json` compatible to `Orchestrator.describe_service`
 791         # Open question: Remove `service_id` form to_json?
 792         if c.get('service_name', ''):
 793             service_type_id = c['service_name'].split('.', 1)
 794
 795             if not c.get('service_type', ''):
 796                 c['service_type'] = service_type_id[0]
 797             if not c.get('service_id', '') and len(service_type_id) > 1:
 798                 c['service_id'] = service_type_id[1]
 799             del c['service_name']
 800
 801         service_type = c.get('service_type', '')
 802         _cls = cls._cls(service_type)
 803
 804         if 'status' in c:
 805             del c['status']  # kludge to make us compatible to `ServiceDescription.to_json()`
 806
 807         return _cls._from_json_impl(c)  # type: ignore
 808
 809     @staticmethod
 810     def normalize_json(json_spec: dict) -> dict:
 811         networks = json_spec.get('networks')
 812         if networks is None:
 813             return json_spec
 814         if isinstance(networks, list):
 815             return json_spec
 816         if not isinstance(networks, str):
 817             raise SpecValidationError(f'Networks ({networks}) must be a string or list of strings')
 818         json_spec['networks'] = [networks]
 819         return json_spec
 820
 821     @classmethod
 822     def _from_json_impl(cls: Type[ServiceSpecT], json_spec: dict) -> ServiceSpecT:
 823         args = {}  # type: Dict[str, Any]
 824         for k, v in json_spec.items():
 825             if k == 'placement':
 826                 v = PlacementSpec.from_json(v)
 827             if k == 'custom_configs':
 828                 v = [CustomConfig.from_json(c) for c in v]
 829             if k == 'spec':
 830                 args.update(v)
 831                 continue
 832             args.update({k: v})
 833         _cls = cls(**args)
 834         if _service_spec_from_json_validate:
 835             _cls.validate()
 836         return _cls
 837
 838     def service_name(self) -> str:
 839         n = self.service_type
 840         if self.service_id:
 841             n += '.' + self.service_id
 842         return n
 843
 844     def get_port_start(self) -> List[int]:
 845         # If defined, we will allocate and number ports starting at this
 846         # point.
 847         return []
 848
 849     def get_virtual_ip(self) -> Optional[str]:
 850         return None
 851
 852     def to_json(self):
 853         # type: () -> OrderedDict[str, Any]
 854         ret: OrderedDict[str, Any] = OrderedDict()
 855         ret['service_type'] = self.service_type
 856         if self.service_id:
 857             ret['service_id'] = self.service_id
 858         ret['service_name'] = self.service_name()
 859         if self.placement.to_json():
 860             ret['placement'] = self.placement.to_json()
 861         if self.unmanaged:
 862             ret['unmanaged'] = self.unmanaged
 863         if self.networks:
 864             ret['networks'] = self.networks
 865         if self.extra_container_args:
 866             ret['extra_container_args'] = ArgumentSpec.map_json(
 867                 self.extra_container_args
 868             )
 869         if self.extra_entrypoint_args:
 870             ret['extra_entrypoint_args'] = ArgumentSpec.map_json(
 871                 self.extra_entrypoint_args
 872             )
 873         if self.custom_configs:
 874             ret['custom_configs'] = [c.to_json() for c in self.custom_configs]
 875
 876         c = {}
 877         for key, val in sorted(self.__dict__.items(), key=lambda tpl: tpl[0]):
 878             if key in ret:
 879                 continue
 880             if hasattr(val, 'to_json'):
 881                 val = val.to_json()
 882             if val:
 883                 c[key] = val
 884         if c:
 885             ret['spec'] = c
 886         return ret
 887
 888     def validate(self) -> None:
 889         if not self.service_type:
 890             raise SpecValidationError('Cannot add Service: type required')
 891
 892         if self.service_type != 'osd':
 893             if self.service_type in self.REQUIRES_SERVICE_ID and not self.service_id:
 894                 raise SpecValidationError('Cannot add Service: id required')
 895             if self.service_type not in self.REQUIRES_SERVICE_ID and self.service_id:
 896                 raise SpecValidationError(
 897                         f'Service of type \'{self.service_type}\' should not contain a service id')
 898
 899         if self.service_id:
 900             if not re.match('^[a-zA-Z0-9_.-]+$', str(self.service_id)):
 901                 raise SpecValidationError('Service id contains invalid characters, '
 902                                           'only [a-zA-Z0-9_.-] allowed')
 903
 904         if self.placement is not None:
 905             self.placement.validate()
 906         if self.config:
 907             for k, v in self.config.items():
 908                 if k in self.MANAGED_CONFIG_OPTIONS:
 909                     raise SpecValidationError(
 910                         f'Cannot set config option {k} in spec: it is managed by cephadm'
 911                     )
 912         for network in self.networks or []:
 913             try:
 914                 ip_network(network)
 915             except ValueError as e:
 916                 raise SpecValidationError(
 917                     f'Cannot parse network {network}: {e}'
 918                 )
 919
 920     def __repr__(self) -> str:
 921         y = yaml.dump(cast(dict, self), default_flow_style=False)
 922         return f"{self.__class__.__name__}.from_json(yaml.safe_load('''{y}'''))"
 923
 924     def __eq__(self, other: Any) -> bool:
 925         return (self.__class__ == other.__class__
 926                 and
 927                 self.__dict__ == other.__dict__)
 928
 929     def one_line_str(self) -> str:
 930         return '<{} for service_name={}>'.format(self.__class__.__name__, self.service_name())
 931
 932     @staticmethod
 933     def yaml_representer(dumper: 'yaml.SafeDumper', data: 'ServiceSpec') -> Any:
 934         return dumper.represent_dict(cast(Mapping, data.to_json().items()))
 935
 936
 937 yaml.add_representer(ServiceSpec, ServiceSpec.yaml_representer)
 938
 939
 940 class NFSServiceSpec(ServiceSpec):
 941     def __init__(self,
 942                  service_type: str = 'nfs',
 943                  service_id: Optional[str] = None,
 944                  placement: Optional[PlacementSpec] = None,
 945                  unmanaged: bool = False,
 946                  preview_only: bool = False,
 947                  config: Optional[Dict[str, str]] = None,
 948                  networks: Optional[List[str]] = None,
 949                  port: Optional[int] = None,
 950                  virtual_ip: Optional[str] = None,
 951                  extra_container_args: Optional[GeneralArgList] = None,
 952                  extra_entrypoint_args: Optional[GeneralArgList] = None,
 953                  enable_haproxy_protocol: bool = False,
 954                  custom_configs: Optional[List[CustomConfig]] = None,
 955                  ):
 956         assert service_type == 'nfs'
 957         super(NFSServiceSpec, self).__init__(
 958             'nfs', service_id=service_id,
 959             placement=placement, unmanaged=unmanaged, preview_only=preview_only,
 960             config=config, networks=networks, extra_container_args=extra_container_args,
 961             extra_entrypoint_args=extra_entrypoint_args, custom_configs=custom_configs)
 962
 963         self.port = port
 964         self.virtual_ip = virtual_ip
 965         self.enable_haproxy_protocol = enable_haproxy_protocol
 966
 967     def get_port_start(self) -> List[int]:
 968         if self.port:
 969             return [self.port]
 970         return []
 971
 972     def rados_config_name(self):
 973         # type: () -> str
 974         return 'conf-' + self.service_name()
 975
 976
 977 yaml.add_representer(NFSServiceSpec, ServiceSpec.yaml_representer)
 978
 979
 980 class RGWSpec(ServiceSpec):
 981     """
 982     Settings to configure a (multisite) Ceph RGW
 983
 984     .. code-block:: yaml
 985
 986         service_type: rgw
 987         service_id: myrealm.myzone
 988         spec:
 989             rgw_realm: myrealm
 990             rgw_zonegroup: myzonegroup
 991             rgw_zone: myzone
 992             ssl: true
 993             rgw_frontend_port: 1234
 994             rgw_frontend_type: beast
 995             rgw_frontend_ssl_certificate: ...
 996
 997     See also: :ref:`orchestrator-cli-service-spec`
 998     """
 999
1000     MANAGED_CONFIG_OPTIONS = ServiceSpec.MANAGED_CONFIG_OPTIONS + [
1001         'rgw_zone',
1002         'rgw_realm',
1003         'rgw_zonegroup',
1004         'rgw_frontends',
1005     ]
1006
1007     def __init__(self,
1008                  service_type: str = 'rgw',
1009                  service_id: Optional[str] = None,
1010                  placement: Optional[PlacementSpec] = None,
1011                  rgw_realm: Optional[str] = None,
1012                  rgw_zonegroup: Optional[str] = None,
1013                  rgw_zone: Optional[str] = None,
1014                  rgw_frontend_port: Optional[int] = None,
1015                  rgw_frontend_ssl_certificate: Optional[List[str]] = None,
1016                  rgw_frontend_type: Optional[str] = None,
1017                  rgw_frontend_extra_args: Optional[List[str]] = None,
1018                  unmanaged: bool = False,
1019                  ssl: bool = False,
1020                  preview_only: bool = False,
1021                  config: Optional[Dict[str, str]] = None,
1022                  networks: Optional[List[str]] = None,
1023                  subcluster: Optional[str] = None,  # legacy, only for from_json on upgrade
1024                  extra_container_args: Optional[GeneralArgList] = None,
1025                  extra_entrypoint_args: Optional[GeneralArgList] = None,
1026                  custom_configs: Optional[List[CustomConfig]] = None,
1027                  rgw_realm_token: Optional[str] = None,
1028                  update_endpoints: Optional[bool] = False,
1029                  zone_endpoints: Optional[str] = None  # commad separated endpoints list
1030                  ):
1031         assert service_type == 'rgw', service_type
1032
1033         # for backward compatibility with octopus spec files,
1034         if not service_id and (rgw_realm and rgw_zone):
1035             service_id = rgw_realm + '.' + rgw_zone
1036
1037         super(RGWSpec, self).__init__(
1038             'rgw', service_id=service_id,
1039             placement=placement, unmanaged=unmanaged,
1040             preview_only=preview_only, config=config, networks=networks,
1041             extra_container_args=extra_container_args, extra_entrypoint_args=extra_entrypoint_args,
1042             custom_configs=custom_configs)
1043
1044         #: The RGW realm associated with this service. Needs to be manually created
1045         #: if the spec is being applied directly to cephdam. In case of rgw module
1046         #: the realm is created automatically.
1047         self.rgw_realm: Optional[str] = rgw_realm
1048         #: The RGW zonegroup associated with this service. Needs to be manually created
1049         #: if the spec is being applied directly to cephdam. In case of rgw module
1050         #: the zonegroup is created automatically.
1051         self.rgw_zonegroup: Optional[str] = rgw_zonegroup
1052         #: The RGW zone associated with this service. Needs to be manually created
1053         #: if the spec is being applied directly to cephdam. In case of rgw module
1054         #: the zone is created automatically.
1055         self.rgw_zone: Optional[str] = rgw_zone
1056         #: Port of the RGW daemons
1057         self.rgw_frontend_port: Optional[int] = rgw_frontend_port
1058         #: List of SSL certificates
1059         self.rgw_frontend_ssl_certificate: Optional[List[str]] = rgw_frontend_ssl_certificate
1060         #: civetweb or beast (default: beast). See :ref:`rgw_frontends`
1061         self.rgw_frontend_type: Optional[str] = rgw_frontend_type
1062         #: List of extra arguments for rgw_frontend in the form opt=value. See :ref:`rgw_frontends`
1063         self.rgw_frontend_extra_args: Optional[List[str]] = rgw_frontend_extra_args
1064         #: enable SSL
1065         self.ssl = ssl
1066         self.rgw_realm_token = rgw_realm_token
1067         self.update_endpoints = update_endpoints
1068         self.zone_endpoints = zone_endpoints
1069
1070     def get_port_start(self) -> List[int]:
1071         return [self.get_port()]
1072
1073     def get_port(self) -> int:
1074         if self.rgw_frontend_port:
1075             return self.rgw_frontend_port
1076         if self.ssl:
1077             return 443
1078         else:
1079             return 80
1080
1081     def validate(self) -> None:
1082         super(RGWSpec, self).validate()
1083
1084         if self.rgw_realm and not self.rgw_zone:
1085             raise SpecValidationError(
1086                     'Cannot add RGW: Realm specified but no zone specified')
1087         if self.rgw_zone and not self.rgw_realm:
1088             raise SpecValidationError('Cannot add RGW: Zone specified but no realm specified')
1089
1090         if self.rgw_frontend_type is not None:
1091             if self.rgw_frontend_type not in ['beast', 'civetweb']:
1092                 raise SpecValidationError(
1093                     'Invalid rgw_frontend_type value. Valid values are: beast, civetweb.\n'
1094                     'Additional rgw type parameters can be passed using rgw_frontend_extra_args.'
1095                 )
1096
1097
1098 yaml.add_representer(RGWSpec, ServiceSpec.yaml_representer)
1099
1100
1101 class NvmeofServiceSpec(ServiceSpec):
1102     def __init__(self,
1103                  service_type: str = 'nvmeof',
1104                  service_id: Optional[str] = None,
1105                  name: Optional[str] = None,
1106                  group: Optional[str] = None,
1107                  port: Optional[int] = None,
1108                  pool: Optional[str] = None,
1109                  enable_auth: bool = False,
1110                  server_key: Optional[str] = None,
1111                  server_cert: Optional[str] = None,
1112                  client_key: Optional[str] = None,
1113                  client_cert: Optional[str] = None,
1114                  spdk_path: Optional[str] = None,
1115                  tgt_path: Optional[str] = None,
1116                  timeout: Optional[int] = 60,
1117                  conn_retries: Optional[int] = 10,
1118                  transports: Optional[str] = 'tcp',
1119                  transport_tcp_options: Optional[Dict[str, int]] =
1120                  {"in_capsule_data_size": 8192, "max_io_qpairs_per_ctrlr": 7},
1121                  tgt_cmd_extra_args: Optional[str] = None,
1122                  placement: Optional[PlacementSpec] = None,
1123                  unmanaged: bool = False,
1124                  preview_only: bool = False,
1125                  config: Optional[Dict[str, str]] = None,
1126                  networks: Optional[List[str]] = None,
1127                  extra_container_args: Optional[GeneralArgList] = None,
1128                  extra_entrypoint_args: Optional[GeneralArgList] = None,
1129                  custom_configs: Optional[List[CustomConfig]] = None,
1130                  ):
1131         assert service_type == 'nvmeof'
1132         super(NvmeofServiceSpec, self).__init__('nvmeof', service_id=service_id,
1133                                                 placement=placement, unmanaged=unmanaged,
1134                                                 preview_only=preview_only,
1135                                                 config=config, networks=networks,
1136                                                 extra_container_args=extra_container_args,
1137                                                 extra_entrypoint_args=extra_entrypoint_args,
1138                                                 custom_configs=custom_configs)
1139
1140         #: RADOS pool where ceph-nvmeof config data is stored.
1141         self.pool = pool
1142         #: ``port`` port of the nvmeof gateway
1143         self.port = port or 5500
1144         #: ``name`` name of the nvmeof gateway
1145         self.name = name
1146         #: ``group`` name of the nvmeof gateway
1147         self.group = group
1148         #: ``enable_auth`` enables user authentication on nvmeof gateway
1149         self.enable_auth = enable_auth
1150         #: ``server_key`` gateway server key
1151         self.server_key = server_key or './server.key'
1152         #: ``server_cert`` gateway server certificate
1153         self.server_cert = server_cert or './server.crt'
1154         #: ``client_key`` client key
1155         self.client_key = client_key or './client.key'
1156         #: ``client_cert`` client certificate
1157         self.client_cert = client_cert or './client.crt'
1158         #: ``spdk_path`` path to SPDK
1159         self.spdk_path = spdk_path or '/usr/local/bin/nvmf_tgt'
1160         #: ``tgt_path`` nvmeof target path
1161         self.tgt_path = tgt_path or '/usr/local/bin/nvmf_tgt'
1162         #: ``timeout`` ceph connectivity timeout
1163         self.timeout = timeout
1164         #: ``conn_retries`` ceph connection retries number
1165         self.conn_retries = conn_retries
1166         #: ``transports`` tcp
1167         self.transports = transports
1168         #: List of extra arguments for transports in the form opt=value
1169         self.transport_tcp_options: Optional[Dict[str, int]] = transport_tcp_options
1170         #: ``tgt_cmd_extra_args`` extra arguments for the nvmf_tgt process
1171         self.tgt_cmd_extra_args = tgt_cmd_extra_args
1172
1173     def get_port_start(self) -> List[int]:
1174         return [5500, 4420, 8009]
1175
1176     def validate(self) -> None:
1177         #  TODO: what other parameters should be validated as part of this function?
1178         super(NvmeofServiceSpec, self).validate()
1179
1180         if not self.pool:
1181             raise SpecValidationError('Cannot add NVMEOF: No Pool specified')
1182
1183         if self.enable_auth:
1184             if not any([self.server_key, self.server_cert, self.client_key, self.client_cert]):
1185                 raise SpecValidationError(
1186                     'enable_auth is true but client/server certificates are missing')
1187
1188         if self.transports not in ['tcp']:
1189             raise SpecValidationError('Invalid transport. Valid values are tcp')
1190
1191
1192 yaml.add_representer(NvmeofServiceSpec, ServiceSpec.yaml_representer)
1193
1194
1195 class IscsiServiceSpec(ServiceSpec):
1196     def __init__(self,
1197                  service_type: str = 'iscsi',
1198                  service_id: Optional[str] = None,
1199                  pool: Optional[str] = None,
1200                  trusted_ip_list: Optional[str] = None,
1201                  api_port: Optional[int] = 5000,
1202                  api_user: Optional[str] = 'admin',
1203                  api_password: Optional[str] = 'admin',
1204                  api_secure: Optional[bool] = None,
1205                  ssl_cert: Optional[str] = None,
1206                  ssl_key: Optional[str] = None,
1207                  placement: Optional[PlacementSpec] = None,
1208                  unmanaged: bool = False,
1209                  preview_only: bool = False,
1210                  config: Optional[Dict[str, str]] = None,
1211                  networks: Optional[List[str]] = None,
1212                  extra_container_args: Optional[GeneralArgList] = None,
1213                  extra_entrypoint_args: Optional[GeneralArgList] = None,
1214                  custom_configs: Optional[List[CustomConfig]] = None,
1215                  ):
1216         assert service_type == 'iscsi'
1217         super(IscsiServiceSpec, self).__init__('iscsi', service_id=service_id,
1218                                                placement=placement, unmanaged=unmanaged,
1219                                                preview_only=preview_only,
1220                                                config=config, networks=networks,
1221                                                extra_container_args=extra_container_args,
1222                                                extra_entrypoint_args=extra_entrypoint_args,
1223                                                custom_configs=custom_configs)
1224
1225         #: RADOS pool where ceph-iscsi config data is stored.
1226         self.pool = pool
1227         #: list of trusted IP addresses
1228         self.trusted_ip_list = trusted_ip_list
1229         #: ``api_port`` as defined in the ``iscsi-gateway.cfg``
1230         self.api_port = api_port
1231         #: ``api_user`` as defined in the ``iscsi-gateway.cfg``
1232         self.api_user = api_user
1233         #: ``api_password`` as defined in the ``iscsi-gateway.cfg``
1234         self.api_password = api_password
1235         #: ``api_secure`` as defined in the ``iscsi-gateway.cfg``
1236         self.api_secure = api_secure
1237         #: SSL certificate
1238         self.ssl_cert = ssl_cert
1239         #: SSL private key
1240         self.ssl_key = ssl_key
1241
1242         if not self.api_secure and self.ssl_cert and self.ssl_key:
1243             self.api_secure = True
1244
1245     def get_port_start(self) -> List[int]:
1246         return [self.api_port or 5000]
1247
1248     def validate(self) -> None:
1249         super(IscsiServiceSpec, self).validate()
1250
1251         if not self.pool:
1252             raise SpecValidationError(
1253                 'Cannot add ISCSI: No Pool specified')
1254
1255         # Do not need to check for api_user and api_password as they
1256         # now default to 'admin' when setting up the gateway url. Older
1257         # iSCSI specs from before this change should be fine as they will
1258         # have been required to have an api_user and api_password set and
1259         # will be unaffected by the new default value.
1260
1261
1262 yaml.add_representer(IscsiServiceSpec, ServiceSpec.yaml_representer)
1263
1264
1265 class IngressSpec(ServiceSpec):
1266     def __init__(self,
1267                  service_type: str = 'ingress',
1268                  service_id: Optional[str] = None,
1269                  config: Optional[Dict[str, str]] = None,
1270                  networks: Optional[List[str]] = None,
1271                  placement: Optional[PlacementSpec] = None,
1272                  backend_service: Optional[str] = None,
1273                  frontend_port: Optional[int] = None,
1274                  ssl_cert: Optional[str] = None,
1275                  ssl_key: Optional[str] = None,
1276                  ssl_dh_param: Optional[str] = None,
1277                  ssl_ciphers: Optional[List[str]] = None,
1278                  ssl_options: Optional[List[str]] = None,
1279                  monitor_port: Optional[int] = None,
1280                  monitor_user: Optional[str] = None,
1281                  monitor_password: Optional[str] = None,
1282                  enable_stats: Optional[bool] = None,
1283                  keepalived_password: Optional[str] = None,
1284                  virtual_ip: Optional[str] = None,
1285                  virtual_ips_list: Optional[List[str]] = None,
1286                  virtual_interface_networks: Optional[List[str]] = [],
1287                  use_keepalived_multicast: Optional[bool] = False,
1288                  vrrp_interface_network: Optional[str] = None,
1289                  first_virtual_router_id: Optional[int] = 50,
1290                  unmanaged: bool = False,
1291                  ssl: bool = False,
1292                  keepalive_only: bool = False,
1293                  extra_container_args: Optional[GeneralArgList] = None,
1294                  extra_entrypoint_args: Optional[GeneralArgList] = None,
1295                  enable_haproxy_protocol: bool = False,
1296                  custom_configs: Optional[List[CustomConfig]] = None,
1297                  ):
1298         assert service_type == 'ingress'
1299
1300         super(IngressSpec, self).__init__(
1301             'ingress', service_id=service_id,
1302             placement=placement, config=config,
1303             networks=networks,
1304             extra_container_args=extra_container_args,
1305             extra_entrypoint_args=extra_entrypoint_args,
1306             custom_configs=custom_configs
1307         )
1308         self.backend_service = backend_service
1309         self.frontend_port = frontend_port
1310         self.ssl_cert = ssl_cert
1311         self.ssl_key = ssl_key
1312         self.ssl_dh_param = ssl_dh_param
1313         self.ssl_ciphers = ssl_ciphers
1314         self.ssl_options = ssl_options
1315         self.monitor_port = monitor_port
1316         self.monitor_user = monitor_user
1317         self.monitor_password = monitor_password
1318         self.keepalived_password = keepalived_password
1319         self.virtual_ip = virtual_ip
1320         self.virtual_ips_list = virtual_ips_list
1321         self.virtual_interface_networks = virtual_interface_networks or []
1322         self.use_keepalived_multicast = use_keepalived_multicast
1323         self.vrrp_interface_network = vrrp_interface_network
1324         self.first_virtual_router_id = first_virtual_router_id
1325         self.unmanaged = unmanaged
1326         self.ssl = ssl
1327         self.keepalive_only = keepalive_only
1328         self.enable_haproxy_protocol = enable_haproxy_protocol
1329
1330     def get_port_start(self) -> List[int]:
1331         ports = []
1332         if self.frontend_port is not None:
1333             ports.append(cast(int, self.frontend_port))
1334         if self.monitor_port is not None:
1335             ports.append(cast(int, self.monitor_port))
1336         return ports
1337
1338     def get_virtual_ip(self) -> Optional[str]:
1339         return self.virtual_ip
1340
1341     def validate(self) -> None:
1342         super(IngressSpec, self).validate()
1343
1344         if not self.backend_service:
1345             raise SpecValidationError(
1346                 'Cannot add ingress: No backend_service specified')
1347         if not self.keepalive_only and not self.frontend_port:
1348             raise SpecValidationError(
1349                 'Cannot add ingress: No frontend_port specified')
1350         if not self.monitor_port:
1351             raise SpecValidationError(
1352                 'Cannot add ingress: No monitor_port specified')
1353         if not self.virtual_ip and not self.virtual_ips_list:
1354             raise SpecValidationError(
1355                 'Cannot add ingress: No virtual_ip provided')
1356         if self.virtual_ip is not None and self.virtual_ips_list is not None:
1357             raise SpecValidationError(
1358                 'Cannot add ingress: Single and multiple virtual IPs specified')
1359
1360
1361 yaml.add_representer(IngressSpec, ServiceSpec.yaml_representer)
1362
1363
1364 class CustomContainerSpec(ServiceSpec):
1365     def __init__(self,
1366                  service_type: str = 'container',
1367                  service_id: Optional[str] = None,
1368                  config: Optional[Dict[str, str]] = None,
1369                  networks: Optional[List[str]] = None,
1370                  placement: Optional[PlacementSpec] = None,
1371                  unmanaged: bool = False,
1372                  preview_only: bool = False,
1373                  image: Optional[str] = None,
1374                  entrypoint: Optional[str] = None,
1375                  extra_entrypoint_args: Optional[GeneralArgList] = None,
1376                  uid: Optional[int] = None,
1377                  gid: Optional[int] = None,
1378                  volume_mounts: Optional[Dict[str, str]] = {},
1379                  # args are for the container runtime, not entrypoint
1380                  args: Optional[GeneralArgList] = [],
1381                  envs: Optional[List[str]] = [],
1382                  privileged: Optional[bool] = False,
1383                  bind_mounts: Optional[List[List[str]]] = None,
1384                  ports: Optional[List[int]] = [],
1385                  dirs: Optional[List[str]] = [],
1386                  files: Optional[Dict[str, Any]] = {},
1387                  ):
1388         assert service_type == 'container'
1389         assert service_id is not None
1390         assert image is not None
1391
1392         super(CustomContainerSpec, self).__init__(
1393             service_type, service_id,
1394             placement=placement, unmanaged=unmanaged,
1395             preview_only=preview_only, config=config,
1396             networks=networks, extra_entrypoint_args=extra_entrypoint_args)
1397
1398         self.image = image
1399         self.entrypoint = entrypoint
1400         self.uid = uid
1401         self.gid = gid
1402         self.volume_mounts = volume_mounts
1403         self.args = args
1404         self.envs = envs
1405         self.privileged = privileged
1406         self.bind_mounts = bind_mounts
1407         self.ports = ports
1408         self.dirs = dirs
1409         self.files = files
1410
1411     def config_json(self) -> Dict[str, Any]:
1412         """
1413         Helper function to get the value of the `--config-json` cephadm
1414         command line option. It will contain all specification properties
1415         that haven't a `None` value. Such properties will get default
1416         values in cephadm.
1417         :return: Returns a dictionary containing all specification
1418             properties.
1419         """
1420         config_json = {}
1421         for prop in ['image', 'entrypoint', 'uid', 'gid', 'args',
1422                      'envs', 'volume_mounts', 'privileged',
1423                      'bind_mounts', 'ports', 'dirs', 'files']:
1424             value = getattr(self, prop)
1425             if value is not None:
1426                 config_json[prop] = value
1427         return config_json
1428
1429
1430 yaml.add_representer(CustomContainerSpec, ServiceSpec.yaml_representer)
1431
1432
1433 class MonitoringSpec(ServiceSpec):
1434     def __init__(self,
1435                  service_type: str,
1436                  service_id: Optional[str] = None,
1437                  config: Optional[Dict[str, str]] = None,
1438                  networks: Optional[List[str]] = None,
1439                  placement: Optional[PlacementSpec] = None,
1440                  unmanaged: bool = False,
1441                  preview_only: bool = False,
1442                  port: Optional[int] = None,
1443                  extra_container_args: Optional[GeneralArgList] = None,
1444                  extra_entrypoint_args: Optional[GeneralArgList] = None,
1445                  custom_configs: Optional[List[CustomConfig]] = None,
1446                  ):
1447         assert service_type in ['grafana', 'node-exporter', 'prometheus', 'alertmanager',
1448                                 'loki', 'promtail']
1449
1450         super(MonitoringSpec, self).__init__(
1451             service_type, service_id,
1452             placement=placement, unmanaged=unmanaged,
1453             preview_only=preview_only, config=config,
1454             networks=networks, extra_container_args=extra_container_args,
1455             extra_entrypoint_args=extra_entrypoint_args,
1456             custom_configs=custom_configs)
1457
1458         self.service_type = service_type
1459         self.port = port
1460
1461     def get_port_start(self) -> List[int]:
1462         return [self.get_port()]
1463
1464     def get_port(self) -> int:
1465         if self.port:
1466             return self.port
1467         else:
1468             return {'prometheus': 9095,
1469                     'node-exporter': 9100,
1470                     'alertmanager': 9093,
1471                     'grafana': 3000,
1472                     'loki': 3100,
1473                     'promtail': 9080}[self.service_type]
1474
1475
1476 yaml.add_representer(MonitoringSpec, ServiceSpec.yaml_representer)
1477
1478
1479 class AlertManagerSpec(MonitoringSpec):
1480     def __init__(self,
1481                  service_type: str = 'alertmanager',
1482                  service_id: Optional[str] = None,
1483                  placement: Optional[PlacementSpec] = None,
1484                  unmanaged: bool = False,
1485                  preview_only: bool = False,
1486                  user_data: Optional[Dict[str, Any]] = None,
1487                  config: Optional[Dict[str, str]] = None,
1488                  networks: Optional[List[str]] = None,
1489                  port: Optional[int] = None,
1490                  secure: bool = False,
1491                  extra_container_args: Optional[GeneralArgList] = None,
1492                  extra_entrypoint_args: Optional[GeneralArgList] = None,
1493                  custom_configs: Optional[List[CustomConfig]] = None,
1494                  ):
1495         assert service_type == 'alertmanager'
1496         super(AlertManagerSpec, self).__init__(
1497             'alertmanager', service_id=service_id,
1498             placement=placement, unmanaged=unmanaged,
1499             preview_only=preview_only, config=config, networks=networks, port=port,
1500             extra_container_args=extra_container_args, extra_entrypoint_args=extra_entrypoint_args,
1501             custom_configs=custom_configs)
1502
1503         # Custom configuration.
1504         #
1505         # Example:
1506         # service_type: alertmanager
1507         # service_id: xyz
1508         # user_data:
1509         #   default_webhook_urls:
1510         #   - "https://foo"
1511         #   - "https://bar"
1512         #
1513         # Documentation:
1514         # default_webhook_urls - A list of additional URL's that are
1515         #                        added to the default receivers'
1516         #                        <webhook_configs> configuration.
1517         self.user_data = user_data or {}
1518         self.secure = secure
1519
1520     def get_port_start(self) -> List[int]:
1521         return [self.get_port(), 9094]
1522
1523     def validate(self) -> None:
1524         super(AlertManagerSpec, self).validate()
1525
1526         if self.port == 9094:
1527             raise SpecValidationError(
1528                 'Port 9094 is reserved for AlertManager cluster listen address')
1529
1530
1531 yaml.add_representer(AlertManagerSpec, ServiceSpec.yaml_representer)
1532
1533
1534 class GrafanaSpec(MonitoringSpec):
1535     def __init__(self,
1536                  service_type: str = 'grafana',
1537                  service_id: Optional[str] = None,
1538                  placement: Optional[PlacementSpec] = None,
1539                  unmanaged: bool = False,
1540                  preview_only: bool = False,
1541                  config: Optional[Dict[str, str]] = None,
1542                  networks: Optional[List[str]] = None,
1543                  port: Optional[int] = None,
1544                  protocol: Optional[str] = 'https',
1545                  initial_admin_password: Optional[str] = None,
1546                  anonymous_access: Optional[bool] = True,
1547                  extra_container_args: Optional[GeneralArgList] = None,
1548                  extra_entrypoint_args: Optional[GeneralArgList] = None,
1549                  custom_configs: Optional[List[CustomConfig]] = None,
1550                  ):
1551         assert service_type == 'grafana'
1552         super(GrafanaSpec, self).__init__(
1553             'grafana', service_id=service_id,
1554             placement=placement, unmanaged=unmanaged,
1555             preview_only=preview_only, config=config, networks=networks, port=port,
1556             extra_container_args=extra_container_args, extra_entrypoint_args=extra_entrypoint_args,
1557             custom_configs=custom_configs)
1558
1559         self.initial_admin_password = initial_admin_password
1560         self.anonymous_access = anonymous_access
1561         self.protocol = protocol
1562
1563     def validate(self) -> None:
1564         super(GrafanaSpec, self).validate()
1565         if self.protocol not in ['http', 'https']:
1566             err_msg = f"Invalid protocol '{self.protocol}'. Valid values are: 'http', 'https'."
1567             raise SpecValidationError(err_msg)
1568
1569         if not self.anonymous_access and not self.initial_admin_password:
1570             err_msg = ('Either initial_admin_password must be set or anonymous_access '
1571                        'must be set to true. Otherwise the grafana dashboard will '
1572                        'be inaccessible.')
1573             raise SpecValidationError(err_msg)
1574
1575
1576 yaml.add_representer(GrafanaSpec, ServiceSpec.yaml_representer)
1577
1578
1579 class PrometheusSpec(MonitoringSpec):
1580     def __init__(self,
1581                  service_type: str = 'prometheus',
1582                  service_id: Optional[str] = None,
1583                  placement: Optional[PlacementSpec] = None,
1584                  unmanaged: bool = False,
1585                  preview_only: bool = False,
1586                  config: Optional[Dict[str, str]] = None,
1587                  networks: Optional[List[str]] = None,
1588                  port: Optional[int] = None,
1589                  retention_time: Optional[str] = None,
1590                  retention_size: Optional[str] = None,
1591                  extra_container_args: Optional[GeneralArgList] = None,
1592                  extra_entrypoint_args: Optional[GeneralArgList] = None,
1593                  custom_configs: Optional[List[CustomConfig]] = None,
1594                  ):
1595         assert service_type == 'prometheus'
1596         super(PrometheusSpec, self).__init__(
1597             'prometheus', service_id=service_id,
1598             placement=placement, unmanaged=unmanaged,
1599             preview_only=preview_only, config=config, networks=networks, port=port,
1600             extra_container_args=extra_container_args, extra_entrypoint_args=extra_entrypoint_args,
1601             custom_configs=custom_configs)
1602
1603         self.retention_time = retention_time.strip() if retention_time else None
1604         self.retention_size = retention_size.strip() if retention_size else None
1605
1606     def validate(self) -> None:
1607         super(PrometheusSpec, self).validate()
1608
1609         if self.retention_time:
1610             valid_units = ['y', 'w', 'd', 'h', 'm', 's']
1611             m = re.search(rf"^(\d+)({'|'.join(valid_units)})$", self.retention_time)
1612             if not m:
1613                 units = ', '.join(valid_units)
1614                 raise SpecValidationError(f"Invalid retention time. Valid units are: {units}")
1615         if self.retention_size:
1616             valid_units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB']
1617             m = re.search(rf"^(\d+)({'|'.join(valid_units)})$", self.retention_size)
1618             if not m:
1619                 units = ', '.join(valid_units)
1620                 raise SpecValidationError(f"Invalid retention size. Valid units are: {units}")
1621
1622
1623 yaml.add_representer(PrometheusSpec, ServiceSpec.yaml_representer)
1624
1625
1626 class SNMPGatewaySpec(ServiceSpec):
1627     class SNMPVersion(str, enum.Enum):
1628         V2c = 'V2c'
1629         V3 = 'V3'
1630
1631         def to_json(self) -> str:
1632             return self.value
1633
1634     class SNMPAuthType(str, enum.Enum):
1635         MD5 = 'MD5'
1636         SHA = 'SHA'
1637
1638         def to_json(self) -> str:
1639             return self.value
1640
1641     class SNMPPrivacyType(str, enum.Enum):
1642         DES = 'DES'
1643         AES = 'AES'
1644
1645         def to_json(self) -> str:
1646             return self.value
1647
1648     valid_destination_types = [
1649         'Name:Port',
1650         'IPv4:Port'
1651     ]
1652
1653     def __init__(self,
1654                  service_type: str = 'snmp-gateway',
1655                  snmp_version: Optional[SNMPVersion] = None,
1656                  snmp_destination: str = '',
1657                  credentials: Dict[str, str] = {},
1658                  engine_id: Optional[str] = None,
1659                  auth_protocol: Optional[SNMPAuthType] = None,
1660                  privacy_protocol: Optional[SNMPPrivacyType] = None,
1661                  placement: Optional[PlacementSpec] = None,
1662                  unmanaged: bool = False,
1663                  preview_only: bool = False,
1664                  port: Optional[int] = None,
1665                  extra_container_args: Optional[GeneralArgList] = None,
1666                  extra_entrypoint_args: Optional[GeneralArgList] = None,
1667                  custom_configs: Optional[List[CustomConfig]] = None,
1668                  ):
1669         assert service_type == 'snmp-gateway'
1670
1671         super(SNMPGatewaySpec, self).__init__(
1672             service_type,
1673             placement=placement,
1674             unmanaged=unmanaged,
1675             preview_only=preview_only,
1676             extra_container_args=extra_container_args,
1677             extra_entrypoint_args=extra_entrypoint_args,
1678             custom_configs=custom_configs)
1679
1680         self.service_type = service_type
1681         self.snmp_version = snmp_version
1682         self.snmp_destination = snmp_destination
1683         self.port = port
1684         self.credentials = credentials
1685         self.engine_id = engine_id
1686         self.auth_protocol = auth_protocol
1687         self.privacy_protocol = privacy_protocol
1688
1689     @classmethod
1690     def _from_json_impl(cls, json_spec: dict) -> 'SNMPGatewaySpec':
1691
1692         cpy = json_spec.copy()
1693         types = [
1694             ('snmp_version', SNMPGatewaySpec.SNMPVersion),
1695             ('auth_protocol', SNMPGatewaySpec.SNMPAuthType),
1696             ('privacy_protocol', SNMPGatewaySpec.SNMPPrivacyType),
1697         ]
1698         for d in cpy, cpy.get('spec', {}):
1699             for key, enum_cls in types:
1700                 try:
1701                     if key in d:
1702                         d[key] = enum_cls(d[key])
1703                 except ValueError:
1704                     raise SpecValidationError(f'{key} unsupported. Must be one of '
1705                                               f'{", ".join(enum_cls)}')
1706         return super(SNMPGatewaySpec, cls)._from_json_impl(cpy)
1707
1708     @property
1709     def ports(self) -> List[int]:
1710         return [self.port or 9464]
1711
1712     def get_port_start(self) -> List[int]:
1713         return self.ports
1714
1715     def validate(self) -> None:
1716         super(SNMPGatewaySpec, self).validate()
1717
1718         if not self.credentials:
1719             raise SpecValidationError(
1720                 'Missing authentication information (credentials). '
1721                 'SNMP V2c and V3 require credential information'
1722             )
1723         elif not self.snmp_version:
1724             raise SpecValidationError(
1725                 'Missing SNMP version (snmp_version)'
1726             )
1727
1728         creds_requirement = {
1729             'V2c': ['snmp_community'],
1730             'V3': ['snmp_v3_auth_username', 'snmp_v3_auth_password']
1731         }
1732         if self.privacy_protocol:
1733             creds_requirement['V3'].append('snmp_v3_priv_password')
1734
1735         missing = [parm for parm in creds_requirement[self.snmp_version]
1736                    if parm not in self.credentials]
1737         # check that credentials are correct for the version
1738         if missing:
1739             raise SpecValidationError(
1740                 f'SNMP {self.snmp_version} credentials are incomplete. Missing {", ".join(missing)}'
1741             )
1742
1743         if self.engine_id:
1744             if 10 <= len(self.engine_id) <= 64 and \
1745                is_hex(self.engine_id) and \
1746                len(self.engine_id) % 2 == 0:
1747                 pass
1748             else:
1749                 raise SpecValidationError(
1750                     'engine_id must be a string containing 10-64 hex characters. '
1751                     'Its length must be divisible by 2'
1752                 )
1753
1754         else:
1755             if self.snmp_version == 'V3':
1756                 raise SpecValidationError(
1757                     'Must provide an engine_id for SNMP V3 notifications'
1758                 )
1759
1760         if not self.snmp_destination:
1761             raise SpecValidationError(
1762                 'SNMP destination (snmp_destination) must be provided'
1763             )
1764         else:
1765             valid, description = valid_addr(self.snmp_destination)
1766             if not valid:
1767                 raise SpecValidationError(
1768                     f'SNMP destination (snmp_destination) is invalid: {description}'
1769                 )
1770             if description not in self.valid_destination_types:
1771                 raise SpecValidationError(
1772                     f'SNMP destination (snmp_destination) type ({description}) is invalid. '
1773                     f'Must be either: {", ".join(sorted(self.valid_destination_types))}'
1774                 )
1775
1776
1777 yaml.add_representer(SNMPGatewaySpec, ServiceSpec.yaml_representer)
1778
1779
1780 class MDSSpec(ServiceSpec):
1781     def __init__(self,
1782                  service_type: str = 'mds',
1783                  service_id: Optional[str] = None,
1784                  placement: Optional[PlacementSpec] = None,
1785                  config: Optional[Dict[str, str]] = None,
1786                  unmanaged: bool = False,
1787                  preview_only: bool = False,
1788                  extra_container_args: Optional[GeneralArgList] = None,
1789                  extra_entrypoint_args: Optional[GeneralArgList] = None,
1790                  custom_configs: Optional[List[CustomConfig]] = None,
1791                  ):
1792         assert service_type == 'mds'
1793         super(MDSSpec, self).__init__('mds', service_id=service_id,
1794                                       placement=placement,
1795                                       config=config,
1796                                       unmanaged=unmanaged,
1797                                       preview_only=preview_only,
1798                                       extra_container_args=extra_container_args,
1799                                       extra_entrypoint_args=extra_entrypoint_args,
1800                                       custom_configs=custom_configs)
1801
1802     def validate(self) -> None:
1803         super(MDSSpec, self).validate()
1804
1805         if str(self.service_id)[0].isdigit():
1806             raise SpecValidationError('MDS service id cannot start with a numeric digit')
1807
1808
1809 yaml.add_representer(MDSSpec, ServiceSpec.yaml_representer)
1810
1811
1812 class MONSpec(ServiceSpec):
1813     def __init__(self,
1814                  service_type: str,
1815                  service_id: Optional[str] = None,
1816                  placement: Optional[PlacementSpec] = None,
1817                  count: Optional[int] = None,
1818                  config: Optional[Dict[str, str]] = None,
1819                  unmanaged: bool = False,
1820                  preview_only: bool = False,
1821                  networks: Optional[List[str]] = None,
1822                  extra_container_args: Optional[GeneralArgList] = None,
1823                  custom_configs: Optional[List[CustomConfig]] = None,
1824                  crush_locations: Optional[Dict[str, List[str]]] = None,
1825                  ):
1826         assert service_type == 'mon'
1827         super(MONSpec, self).__init__('mon', service_id=service_id,
1828                                       placement=placement,
1829                                       count=count,
1830                                       config=config,
1831                                       unmanaged=unmanaged,
1832                                       preview_only=preview_only,
1833                                       networks=networks,
1834                                       extra_container_args=extra_container_args,
1835                                       custom_configs=custom_configs)
1836
1837         self.crush_locations = crush_locations
1838         self.validate()
1839
1840     def validate(self) -> None:
1841         if self.crush_locations:
1842             for host, crush_locs in self.crush_locations.items():
1843                 try:
1844                     assert_valid_host(host)
1845                 except SpecValidationError as e:
1846                     err_str = f'Invalid hostname found in spec crush locations: {e}'
1847                     raise SpecValidationError(err_str)
1848                 for cloc in crush_locs:
1849                     if '=' not in cloc or len(cloc.split('=')) != 2:
1850                         err_str = ('Crush locations must be of form <bucket>=<location>. '
1851                                    f'Found crush location: {cloc}')
1852                         raise SpecValidationError(err_str)
1853
1854
1855 yaml.add_representer(MONSpec, ServiceSpec.yaml_representer)
1856
1857
1858 class TracingSpec(ServiceSpec):
1859     SERVICE_TYPES = ['elasticsearch', 'jaeger-collector', 'jaeger-query', 'jaeger-agent']
1860
1861     def __init__(self,
1862                  service_type: str,
1863                  es_nodes: Optional[str] = None,
1864                  without_query: bool = False,
1865                  service_id: Optional[str] = None,
1866                  config: Optional[Dict[str, str]] = None,
1867                  networks: Optional[List[str]] = None,
1868                  placement: Optional[PlacementSpec] = None,
1869                  unmanaged: bool = False,
1870                  preview_only: bool = False
1871                  ):
1872         assert service_type in TracingSpec.SERVICE_TYPES + ['jaeger-tracing']
1873
1874         super(TracingSpec, self).__init__(
1875             service_type, service_id,
1876             placement=placement, unmanaged=unmanaged,
1877             preview_only=preview_only, config=config,
1878             networks=networks)
1879         self.without_query = without_query
1880         self.es_nodes = es_nodes
1881
1882     def get_port_start(self) -> List[int]:
1883         return [self.get_port()]
1884
1885     def get_port(self) -> int:
1886         return {'elasticsearch': 9200,
1887                 'jaeger-agent': 6799,
1888                 'jaeger-collector': 14250,
1889                 'jaeger-query': 16686}[self.service_type]
1890
1891     def get_tracing_specs(self) -> List[ServiceSpec]:
1892         assert self.service_type == 'jaeger-tracing'
1893         specs: List[ServiceSpec] = []
1894         daemons: Dict[str, Optional[PlacementSpec]] = {
1895             daemon: None for daemon in TracingSpec.SERVICE_TYPES}
1896
1897         if self.es_nodes:
1898             del daemons['elasticsearch']
1899         if self.without_query:
1900             del daemons['jaeger-query']
1901         if self.placement:
1902             daemons.update({'jaeger-collector': self.placement})
1903
1904         for daemon, daemon_placement in daemons.items():
1905             specs.append(TracingSpec(service_type=daemon,
1906                                      es_nodes=self.es_nodes,
1907                                      placement=daemon_placement,
1908                                      unmanaged=self.unmanaged,
1909                                      config=self.config,
1910                                      networks=self.networks,
1911                                      preview_only=self.preview_only
1912                                      ))
1913         return specs
1914
1915
1916 yaml.add_representer(TracingSpec, ServiceSpec.yaml_representer)
1917
1918
1919 class TunedProfileSpec():
1920     def __init__(self,
1921                  profile_name: str,
1922                  placement: Optional[PlacementSpec] = None,
1923                  settings: Optional[Dict[str, str]] = None,
1924                  ):
1925         self.profile_name = profile_name
1926         self.placement = placement or PlacementSpec(host_pattern='*')
1927         self.settings = settings or {}
1928         self._last_updated: str = ''
1929
1930     @classmethod
1931     def from_json(cls, spec: Dict[str, Any]) -> 'TunedProfileSpec':
1932         data = {}
1933         if 'profile_name' not in spec:
1934             raise SpecValidationError('Tuned profile spec must include "profile_name" field')
1935         data['profile_name'] = spec['profile_name']
1936         if not isinstance(data['profile_name'], str):
1937             raise SpecValidationError('"profile_name" field must be a string')
1938         if 'placement' in spec:
1939             data['placement'] = PlacementSpec.from_json(spec['placement'])
1940         if 'settings' in spec:
1941             data['settings'] = spec['settings']
1942         return cls(**data)
1943
1944     def to_json(self) -> Dict[str, Any]:
1945         res: Dict[str, Any] = {}
1946         res['profile_name'] = self.profile_name
1947         res['placement'] = self.placement.to_json()
1948         res['settings'] = self.settings
1949         return res
1950
1951     def __eq__(self, other: Any) -> bool:
1952         if isinstance(other, TunedProfileSpec):
1953             if (
1954                 self.placement == other.placement
1955                 and self.profile_name == other.profile_name
1956                 and self.settings == other.settings
1957             ):
1958                 return True
1959             return False
1960         return NotImplemented
1961
1962     def __repr__(self) -> str:
1963         return f'TunedProfile({self.profile_name})'
1964
1965     def copy(self) -> 'TunedProfileSpec':
1966         # for making deep copies so you can edit the settings in one without affecting the other
1967         # mostly for testing purposes
1968         return TunedProfileSpec(self.profile_name, self.placement, self.settings.copy())
1969
1970
1971 class CephExporterSpec(ServiceSpec):
1972     def __init__(self,
1973                  service_type: str = 'ceph-exporter',
1974                  sock_dir: Optional[str] = None,
1975                  addrs: str = '',
1976                  port: Optional[int] = None,
1977                  prio_limit: Optional[int] = 5,
1978                  stats_period: Optional[int] = 5,
1979                  placement: Optional[PlacementSpec] = None,
1980                  unmanaged: bool = False,
1981                  preview_only: bool = False,
1982                  extra_container_args: Optional[GeneralArgList] = None,
1983                  ):
1984         assert service_type == 'ceph-exporter'
1985
1986         super(CephExporterSpec, self).__init__(
1987             service_type,
1988             placement=placement,
1989             unmanaged=unmanaged,
1990             preview_only=preview_only,
1991             extra_container_args=extra_container_args)
1992
1993         self.service_type = service_type
1994         self.sock_dir = sock_dir
1995         self.addrs = addrs
1996         self.port = port
1997         self.prio_limit = prio_limit
1998         self.stats_period = stats_period
1999
2000     def validate(self) -> None:
2001         super(CephExporterSpec, self).validate()
2002
2003         if not isinstance(self.prio_limit, int):
2004             raise SpecValidationError(
2005                     f'prio_limit must be an integer. Got {type(self.prio_limit)}')
2006         if not isinstance(self.stats_period, int):
2007             raise SpecValidationError(
2008                     f'stats_period must be an integer. Got {type(self.stats_period)}')
2009
2010
2011 yaml.add_representer(CephExporterSpec, ServiceSpec.yaml_representer)