ceph/src/python-common/ceph/deployment/service_spec.py

   1 import fnmatch
   2 import re
   3 from collections import namedtuple
   4 from typing import Optional, Dict, Any, List, Union
   5
   6 import six
   7
   8
   9 class ServiceSpecValidationError(Exception):
  10     """
  11     Defining an exception here is a bit problematic, cause you cannot properly catch it,
  12     if it was raised in a different mgr module.
  13     """
  14
  15     def __init__(self, msg):
  16         super(ServiceSpecValidationError, self).__init__(msg)
  17
  18
  19 def assert_valid_host(name):
  20     p = re.compile('^[a-zA-Z0-9-]+$')
  21     try:
  22         assert len(name) <= 250, 'name is too long (max 250 chars)'
  23         for part in name.split('.'):
  24             assert len(part) > 0, '.-delimited name component must not be empty'
  25             assert len(part) <= 63, '.-delimited name component must not be more than 63 chars'
  26             assert p.match(part), 'name component must include only a-z, 0-9, and -'
  27     except AssertionError as e:
  28         raise ServiceSpecValidationError(e)
  29
  30
  31 class HostPlacementSpec(namedtuple('HostPlacementSpec', ['hostname', 'network', 'name'])):
  32     def __str__(self):
  33         res = ''
  34         res += self.hostname
  35         if self.network:
  36             res += ':' + self.network
  37         if self.name:
  38             res += '=' + self.name
  39         return res
  40
  41     @classmethod
  42     def from_json(cls, data):
  43         return cls(**data)
  44
  45     def to_json(self):
  46         return {
  47             'hostname': self.hostname,
  48             'network': self.network,
  49             'name': self.name
  50         }
  51
  52     @classmethod
  53     def parse(cls, host, require_network=True):
  54         # type: (str, bool) -> HostPlacementSpec
  55         """
  56         Split host into host, network, and (optional) daemon name parts.  The network
  57         part can be an IP, CIDR, or ceph addrvec like '[v2:1.2.3.4:3300,v1:1.2.3.4:6789]'.
  58         e.g.,
  59           "myhost"
  60           "myhost=name"
  61           "myhost:1.2.3.4"
  62           "myhost:1.2.3.4=name"
  63           "myhost:1.2.3.0/24"
  64           "myhost:1.2.3.0/24=name"
  65           "myhost:[v2:1.2.3.4:3000]=name"
  66           "myhost:[v2:1.2.3.4:3000,v1:1.2.3.4:6789]=name"
  67         """
  68         # Matches from start to : or = or until end of string
  69         host_re = r'^(.*?)(:|=|$)'
  70         # Matches from : to = or until end of string
  71         ip_re = r':(.*?)(=|$)'
  72         # Matches from = to end of string
  73         name_re = r'=(.*?)$'
  74
  75         # assign defaults
  76         host_spec = cls('', '', '')
  77
  78         match_host = re.search(host_re, host)
  79         if match_host:
  80             host_spec = host_spec._replace(hostname=match_host.group(1))
  81
  82         name_match = re.search(name_re, host)
  83         if name_match:
  84             host_spec = host_spec._replace(name=name_match.group(1))
  85
  86         ip_match = re.search(ip_re, host)
  87         if ip_match:
  88             host_spec = host_spec._replace(network=ip_match.group(1))
  89
  90         if not require_network:
  91             return host_spec
  92
  93         from ipaddress import ip_network, ip_address
  94         networks = list()  # type: List[str]
  95         network = host_spec.network
  96         # in case we have [v2:1.2.3.4:3000,v1:1.2.3.4:6478]
  97         if ',' in network:
  98             networks = [x for x in network.split(',')]
  99         else:
 100             networks.append(network)
 101         for network in networks:
 102             # only if we have versioned network configs
 103             if network.startswith('v') or network.startswith('[v'):
 104                 network = network.split(':')[1]
 105             try:
 106                 # if subnets are defined, also verify the validity
 107                 if '/' in network:
 108                     ip_network(six.text_type(network))
 109                 else:
 110                     ip_address(six.text_type(network))
 111             except ValueError as e:
 112                 # logging?
 113                 raise e
 114         host_spec.validate()
 115         return host_spec
 116
 117     def validate(self):
 118         assert_valid_host(self.hostname)
 119
 120
 121 class PlacementSpec(object):
 122     """
 123     For APIs that need to specify a host subset
 124     """
 125
 126     def __init__(self,
 127                  label=None,  # type: Optional[str]
 128                  hosts=None,  # type: Union[List[str],List[HostPlacementSpec]]
 129                  count=None,  # type: Optional[int]
 130                  host_pattern=None  # type: Optional[str]
 131                  ):
 132         # type: (...) -> None
 133         self.label = label
 134         self.hosts = []  # type: List[HostPlacementSpec]
 135
 136         if hosts:
 137             if all([isinstance(host, HostPlacementSpec) for host in hosts]):
 138                 self.hosts = hosts  # type: ignore
 139             else:
 140                 self.hosts = [HostPlacementSpec.parse(x, require_network=False)  # type: ignore
 141                               for x in hosts if x]
 142
 143         self.count = count  # type: Optional[int]
 144
 145         #: fnmatch patterns to select hosts. Can also be a single host.
 146         self.host_pattern = host_pattern  # type: Optional[str]
 147
 148         self.validate()
 149
 150     def is_empty(self):
 151         return self.label is None and \
 152             not self.hosts and \
 153             not self.host_pattern and \
 154             self.count is None
 155
 156     def set_hosts(self, hosts):
 157         # To backpopulate the .hosts attribute when using labels or count
 158         # in the orchestrator backend.
 159         self.hosts = hosts
 160
 161     def pattern_matches_hosts(self, all_hosts):
 162         # type: (List[str]) -> List[str]
 163         if not self.host_pattern:
 164             return []
 165         return fnmatch.filter(all_hosts, self.host_pattern)
 166
 167     def pretty_str(self):
 168         kv = []
 169         if self.count:
 170             kv.append('count:%d' % self.count)
 171         if self.label:
 172             kv.append('label:%s' % self.label)
 173         if self.hosts:
 174             kv.append('%s' % ','.join([str(h) for h in self.hosts]))
 175         if self.host_pattern:
 176             kv.append(self.host_pattern)
 177         return ' '.join(kv)
 178
 179     def __repr__(self):
 180         kv = []
 181         if self.count:
 182             kv.append('count=%d' % self.count)
 183         if self.label:
 184             kv.append('label=%s' % repr(self.label))
 185         if self.hosts:
 186             kv.append('hosts={!r}'.format(self.hosts))
 187         if self.host_pattern:
 188             kv.append('host_pattern={!r}'.format(self.host_pattern))
 189         return "PlacementSpec(%s)" % ', '.join(kv)
 190
 191     @classmethod
 192     def from_json(cls, data):
 193         hosts = data.get('hosts', [])
 194         if hosts:
 195             data['hosts'] = [HostPlacementSpec.from_json(host) for host in hosts]
 196         _cls = cls(**data)
 197         _cls.validate()
 198         return _cls
 199
 200     def to_json(self):
 201         r = {}
 202         if self.label:
 203             r['label'] = self.label
 204         if self.hosts:
 205             r['hosts'] = [host.to_json() for host in self.hosts]
 206         if self.count:
 207             r['count'] = self.count
 208         if self.host_pattern:
 209             r['host_pattern'] = self.host_pattern
 210         return r
 211
 212     def validate(self):
 213         if self.hosts and self.label:
 214             # TODO: a less generic Exception
 215             raise ServiceSpecValidationError('Host and label are mutually exclusive')
 216         if self.count is not None and self.count <= 0:
 217             raise ServiceSpecValidationError("num/count must be > 1")
 218         if self.host_pattern and self.hosts:
 219             raise ServiceSpecValidationError('cannot combine host patterns and hosts')
 220         for h in self.hosts:
 221             h.validate()
 222
 223     @classmethod
 224     def from_string(cls, arg):
 225         # type: (Optional[str]) -> PlacementSpec
 226         """
 227         A single integer is parsed as a count:
 228         >>> PlacementSpec.from_string('3')
 229         PlacementSpec(count=3)
 230
 231         A list of names is parsed as host specifications:
 232         >>> PlacementSpec.from_string('host1 host2')
 233         PlacementSpec(hosts=[HostPlacementSpec(hostname='host1', network='', name=''), HostPlacemen\
 234 tSpec(hostname='host2', network='', name='')])
 235
 236         You can also prefix the hosts with a count as follows:
 237         >>> PlacementSpec.from_string('2 host1 host2')
 238         PlacementSpec(count=2, hosts=[HostPlacementSpec(hostname='host1', network='', name=''), Hos\
 239 tPlacementSpec(hostname='host2', network='', name='')])
 240
 241         You can spefify labels using `label:<label>`
 242         >>> PlacementSpec.from_string('label:mon')
 243         PlacementSpec(label='mon')
 244
 245         Labels als support a count:
 246         >>> PlacementSpec.from_string('3 label:mon')
 247         PlacementSpec(count=3, label='mon')
 248
 249         fnmatch is also supported:
 250         >>> PlacementSpec.from_string('data[1-3]')
 251         PlacementSpec(host_pattern='data[1-3]')
 252
 253         >>> PlacementSpec.from_string(None)
 254         PlacementSpec()
 255         """
 256         if arg is None or not arg:
 257             strings = []
 258         elif isinstance(arg, str):
 259             if ' ' in arg:
 260                 strings = arg.split(' ')
 261             elif ';' in arg:
 262                 strings = arg.split(';')
 263             elif ',' in arg and '[' not in arg:
 264                 # FIXME: this isn't quite right.  we want to avoid breaking
 265                 # a list of mons with addrvecs... so we're basically allowing
 266                 # , most of the time, except when addrvecs are used.  maybe
 267                 # ok?
 268                 strings = arg.split(',')
 269             else:
 270                 strings = [arg]
 271         else:
 272             raise ServiceSpecValidationError('invalid placement %s' % arg)
 273
 274         count = None
 275         if strings:
 276             try:
 277                 count = int(strings[0])
 278                 strings = strings[1:]
 279             except ValueError:
 280                 pass
 281         for s in strings:
 282             if s.startswith('count:'):
 283                 try:
 284                     count = int(s[6:])
 285                     strings.remove(s)
 286                     break
 287                 except ValueError:
 288                     pass
 289
 290         advanced_hostspecs = [h for h in strings if
 291                               (':' in h or '=' in h or not any(c in '[]?*:=' for c in h)) and
 292                               'label:' not in h]
 293         for a_h in advanced_hostspecs:
 294             strings.remove(a_h)
 295
 296         labels = [x for x in strings if 'label:' in x]
 297         if len(labels) > 1:
 298             raise ServiceSpecValidationError('more than one label provided: {}'.format(labels))
 299         for l in labels:
 300             strings.remove(l)
 301         label = labels[0][6:] if labels else None
 302
 303         host_patterns = strings
 304         if len(host_patterns) > 1:
 305             raise ServiceSpecValidationError(
 306                 'more than one host pattern provided: {}'.format(host_patterns))
 307
 308         ps = PlacementSpec(count=count,
 309                            hosts=advanced_hostspecs,
 310                            label=label,
 311                            host_pattern=host_patterns[0] if host_patterns else None)
 312         return ps
 313
 314
 315 class ServiceSpec(object):
 316     """
 317     Details of service creation.
 318
 319     Request to the orchestrator for a cluster of daemons
 320     such as MDS, RGW, iscsi gateway, MONs, MGRs, Prometheus
 321
 322     This structure is supposed to be enough information to
 323     start the services.
 324
 325     """
 326     KNOWN_SERVICE_TYPES = 'alertmanager crash grafana mds mgr mon nfs ' \
 327                           'node-exporter osd prometheus rbd-mirror rgw'.split()
 328
 329     def __init__(self,
 330                  service_type,     # type: str
 331                  service_id=None,  # type: Optional[str]
 332                  placement=None,   # type: Optional[PlacementSpec]
 333                  count=None,       # type: Optional[int]
 334                  unmanaged=False,  # type: bool
 335                  ):
 336         self.placement = PlacementSpec() if placement is None else placement  # type: PlacementSpec
 337
 338         assert service_type in ServiceSpec.KNOWN_SERVICE_TYPES, service_type
 339         self.service_type = service_type
 340         self.service_id = service_id
 341         self.unmanaged = unmanaged
 342
 343     @classmethod
 344     def from_json(cls, json_spec):
 345         # type: (dict) -> Any
 346         # Python 3:
 347         # >>> ServiceSpecs = TypeVar('Base', bound=ServiceSpec)
 348         # then, the real type is: (dict) -> ServiceSpecs
 349         """
 350         Initialize 'ServiceSpec' object data from a json structure
 351         :param json_spec: A valid dict with ServiceSpec
 352         """
 353         from ceph.deployment.drive_group import DriveGroupSpec
 354
 355         service_type = json_spec.get('service_type', '')
 356         _cls = {
 357             'rgw': RGWSpec,
 358             'nfs': NFSServiceSpec,
 359             'osd': DriveGroupSpec
 360         }.get(service_type, cls)
 361
 362         if _cls == ServiceSpec and not service_type:
 363             raise ServiceSpecValidationError('Spec needs a "service_type" key.')
 364
 365         return _cls._from_json_impl(json_spec)  # type: ignore
 366
 367     @classmethod
 368     def _from_json_impl(cls, json_spec):
 369         args = {}  # type: Dict[str, Dict[Any, Any]]
 370         for k, v in json_spec.items():
 371             if k == 'placement':
 372                 v = PlacementSpec.from_json(v)
 373             if k == 'spec':
 374                 args.update(v)
 375                 continue
 376             args.update({k: v})
 377         return cls(**args)
 378
 379     def service_name(self):
 380         n = self.service_type
 381         if self.service_id:
 382             n += '.' + self.service_id
 383         return n
 384
 385     def to_json(self):
 386         # type: () -> Dict[str, Any]
 387         c = {}
 388         for key, val in self.__dict__.items():
 389             if hasattr(val, 'to_json'):
 390                 val = val.to_json()
 391             if val:
 392                 c[key] = val
 393         return c
 394
 395     def validate(self):
 396         if not self.service_type:
 397             raise ServiceSpecValidationError('Cannot add Service: type required')
 398
 399         if self.placement is not None:
 400             self.placement.validate()
 401
 402     def __repr__(self):
 403         return "{}({!r})".format(self.__class__.__name__, self.__dict__)
 404
 405     def one_line_str(self):
 406         return '<{} for service_name={}>'.format(self.__class__.__name__, self.service_name())
 407
 408
 409 def servicespec_validate_add(self: ServiceSpec):
 410     # This must not be a method of ServiceSpec, otherwise you'll hunt
 411     # sub-interpreter affinity bugs.
 412     ServiceSpec.validate(self)
 413     if self.service_type in ['mds', 'rgw', 'nfs'] and not self.service_id:
 414         raise ServiceSpecValidationError('Cannot add Service: id required')
 415
 416
 417 class NFSServiceSpec(ServiceSpec):
 418     def __init__(self, service_id, pool=None, namespace=None, placement=None,
 419                  service_type='nfs', unmanaged=False):
 420         assert service_type == 'nfs'
 421         super(NFSServiceSpec, self).__init__(
 422             'nfs', service_id=service_id,
 423             placement=placement, unmanaged=unmanaged)
 424
 425         #: RADOS pool where NFS client recovery data is stored.
 426         self.pool = pool
 427
 428         #: RADOS namespace where NFS client recovery data is stored in the pool.
 429         self.namespace = namespace
 430
 431     def validate_add(self):
 432         servicespec_validate_add(self)
 433
 434         if not self.pool:
 435             raise ServiceSpecValidationError('Cannot add NFS: No Pool specified')
 436
 437
 438 class RGWSpec(ServiceSpec):
 439     """
 440     Settings to configure a (multisite) Ceph RGW
 441
 442     """
 443     def __init__(self,
 444                  rgw_realm=None,  # type: Optional[str]
 445                  rgw_zone=None,  # type: Optional[str]
 446                  subcluster=None,  # type: Optional[str]
 447                  service_id=None,  # type: Optional[str]
 448                  placement=None,
 449                  service_type='rgw',
 450                  rgw_frontend_port=None,  # type: Optional[int]
 451                  unmanaged=False,  # type: bool
 452                  ssl=False,   # type: bool
 453                  ):
 454         assert service_type == 'rgw'
 455         if service_id:
 456             a = service_id.split('.', 2)
 457             rgw_realm = a[0]
 458             rgw_zone = a[1]
 459             if len(a) > 2:
 460                 subcluster = a[2]
 461         else:
 462             if subcluster:
 463                 service_id = '%s.%s.%s' % (rgw_realm, rgw_zone, subcluster)
 464             else:
 465                 service_id = '%s.%s' % (rgw_realm, rgw_zone)
 466         super(RGWSpec, self).__init__(
 467             'rgw', service_id=service_id,
 468             placement=placement, unmanaged=unmanaged)
 469
 470         self.rgw_realm = rgw_realm
 471         self.rgw_zone = rgw_zone
 472         self.subcluster = subcluster
 473         self.rgw_frontend_port = rgw_frontend_port
 474         self.ssl = ssl
 475
 476     def get_port(self):
 477         if self.rgw_frontend_port:
 478             return self.rgw_frontend_port
 479         if self.ssl:
 480             return 443
 481         else:
 482             return 80