ceph/src/pybind/mgr/dashboard/controllers/rbd_mirroring.py

   1 # -*- coding: utf-8 -*-
   2 from __future__ import absolute_import
   3
   4 import json
   5 import re
   6 import logging
   7
   8 from functools import partial
   9
  10 import cherrypy
  11
  12 import rbd
  13
  14 from . import ApiController, Endpoint, Task, BaseController, ReadPermission, \
  15     UpdatePermission, RESTController
  16
  17 from .. import mgr
  18 from ..security import Scope
  19 from ..services.ceph_service import CephService
  20 from ..services.rbd import rbd_call
  21 from ..tools import ViewCache
  22 from ..services.exception import handle_rados_error, handle_rbd_error, \
  23     serialize_dashboard_exception
  24
  25 try:
  26     from typing import no_type_check
  27 except ImportError:
  28     no_type_check = object()  # Just for type checking
  29
  30
  31 logger = logging.getLogger('controllers.rbd_mirror')
  32
  33
  34 # pylint: disable=not-callable
  35 def handle_rbd_mirror_error():
  36     def composed_decorator(func):
  37         func = handle_rados_error('rbd-mirroring')(func)
  38         return handle_rbd_error()(func)
  39     return composed_decorator
  40
  41
  42 # pylint: disable=not-callable
  43 def RbdMirroringTask(name, metadata, wait_for):  # noqa: N802
  44     def composed_decorator(func):
  45         func = handle_rbd_mirror_error()(func)
  46         return Task("rbd/mirroring/{}".format(name), metadata, wait_for,
  47                     partial(serialize_dashboard_exception, include_http_status=True))(func)
  48     return composed_decorator
  49
  50
  51 @ViewCache()
  52 def get_daemons_and_pools():  # pylint: disable=R0915
  53     def get_daemons():
  54         daemons = []
  55         for hostname, server in CephService.get_service_map('rbd-mirror').items():
  56             for service in server['services']:
  57                 id = service['id']  # pylint: disable=W0622
  58                 metadata = service['metadata']
  59                 status = service['status'] or {}
  60
  61                 try:
  62                     status = json.loads(status['json'])
  63                 except (ValueError, KeyError):
  64                     status = {}
  65
  66                 instance_id = metadata['instance_id']
  67                 if id == instance_id:
  68                     # new version that supports per-cluster leader elections
  69                     id = metadata['id']
  70
  71                 # extract per-daemon service data and health
  72                 daemon = {
  73                     'id': id,
  74                     'instance_id': instance_id,
  75                     'version': metadata['ceph_version'],
  76                     'server_hostname': hostname,
  77                     'service': service,
  78                     'server': server,
  79                     'metadata': metadata,
  80                     'status': status
  81                 }
  82                 daemon = dict(daemon, **get_daemon_health(daemon))
  83                 daemons.append(daemon)
  84
  85         return sorted(daemons, key=lambda k: k['instance_id'])
  86
  87     def get_daemon_health(daemon):
  88         health = {
  89             'health_color': 'info',
  90             'health': 'Unknown'
  91         }
  92         for _, pool_data in daemon['status'].items():
  93             if (health['health'] != 'error'
  94                     and [k for k, v in pool_data.get('callouts', {}).items()
  95                          if v['level'] == 'error']):
  96                 health = {
  97                     'health_color': 'error',
  98                     'health': 'Error'
  99                 }
 100             elif (health['health'] != 'error'
 101                   and [k for k, v in pool_data.get('callouts', {}).items()
 102                        if v['level'] == 'warning']):
 103                 health = {
 104                     'health_color': 'warning',
 105                     'health': 'Warning'
 106                 }
 107             elif health['health_color'] == 'info':
 108                 health = {
 109                     'health_color': 'success',
 110                     'health': 'OK'
 111                 }
 112         return health
 113
 114     def get_pools(daemons):  # pylint: disable=R0912, R0915
 115         pool_names = [pool['pool_name'] for pool in CephService.get_pool_list('rbd')
 116                       if pool.get('type', 1) == 1]
 117         pool_stats = {}
 118         rbdctx = rbd.RBD()
 119         for pool_name in pool_names:
 120             logger.debug("Constructing IOCtx %s", pool_name)
 121             try:
 122                 ioctx = mgr.rados.open_ioctx(pool_name)
 123             except TypeError:
 124                 logger.exception("Failed to open pool %s", pool_name)
 125                 continue
 126
 127             try:
 128                 mirror_mode = rbdctx.mirror_mode_get(ioctx)
 129                 peer_uuids = [x['uuid'] for x in rbdctx.mirror_peer_list(ioctx)]
 130             except:  # noqa pylint: disable=W0702
 131                 logger.exception("Failed to query mirror settings %s", pool_name)
 132                 mirror_mode = None
 133                 peer_uuids = []
 134
 135             stats = {}
 136             if mirror_mode == rbd.RBD_MIRROR_MODE_DISABLED:
 137                 mirror_mode = "disabled"
 138                 stats['health_color'] = "info"
 139                 stats['health'] = "Disabled"
 140             elif mirror_mode == rbd.RBD_MIRROR_MODE_IMAGE:
 141                 mirror_mode = "image"
 142             elif mirror_mode == rbd.RBD_MIRROR_MODE_POOL:
 143                 mirror_mode = "pool"
 144             else:
 145                 mirror_mode = "unknown"
 146                 stats['health_color'] = "warning"
 147                 stats['health'] = "Warning"
 148
 149             pool_stats[pool_name] = dict(stats, **{
 150                 'mirror_mode': mirror_mode,
 151                 'peer_uuids': peer_uuids
 152             })
 153
 154         for daemon in daemons:
 155             for _, pool_data in daemon['status'].items():
 156                 stats = pool_stats.get(pool_data['name'], None)  # type: ignore
 157                 if stats is None:
 158                     continue
 159
 160                 if pool_data.get('leader', False):
 161                     # leader instance stores image counts
 162                     stats['leader_id'] = daemon['metadata']['instance_id']
 163                     stats['image_local_count'] = pool_data.get('image_local_count', 0)
 164                     stats['image_remote_count'] = pool_data.get('image_remote_count', 0)
 165
 166                 if (stats.get('health_color', '') != 'error'
 167                         and pool_data.get('image_error_count', 0) > 0):
 168                     stats['health_color'] = 'error'
 169                     stats['health'] = 'Error'
 170                 elif (stats.get('health_color', '') != 'error'
 171                       and pool_data.get('image_warning_count', 0) > 0):
 172                     stats['health_color'] = 'warning'
 173                     stats['health'] = 'Warning'
 174                 elif stats.get('health', None) is None:
 175                     stats['health_color'] = 'success'
 176                     stats['health'] = 'OK'
 177
 178         for _, stats in pool_stats.items():
 179             if stats['mirror_mode'] == 'disabled':
 180                 continue
 181             if stats.get('health', None) is None:
 182                 # daemon doesn't know about pool
 183                 stats['health_color'] = 'error'
 184                 stats['health'] = 'Error'
 185             elif stats.get('leader_id', None) is None:
 186                 # no daemons are managing the pool as leader instance
 187                 stats['health_color'] = 'warning'
 188                 stats['health'] = 'Warning'
 189         return pool_stats
 190
 191     daemons = get_daemons()
 192     return {
 193         'daemons': daemons,
 194         'pools': get_pools(daemons)
 195     }
 196
 197
 198 @ViewCache()
 199 @no_type_check
 200 def _get_pool_datum(pool_name):
 201     data = {}
 202     logger.debug("Constructing IOCtx %s", pool_name)
 203     try:
 204         ioctx = mgr.rados.open_ioctx(pool_name)
 205     except TypeError:
 206         logger.exception("Failed to open pool %s", pool_name)
 207         return None
 208
 209     mirror_state = {
 210         'down': {
 211             'health': 'issue',
 212             'state_color': 'warning',
 213             'state': 'Unknown',
 214             'description': None
 215         },
 216         rbd.MIRROR_IMAGE_STATUS_STATE_UNKNOWN: {
 217             'health': 'issue',
 218             'state_color': 'warning',
 219             'state': 'Unknown'
 220         },
 221         rbd.MIRROR_IMAGE_STATUS_STATE_ERROR: {
 222             'health': 'issue',
 223             'state_color': 'error',
 224             'state': 'Error'
 225         },
 226         rbd.MIRROR_IMAGE_STATUS_STATE_SYNCING: {
 227             'health': 'syncing'
 228         },
 229         rbd.MIRROR_IMAGE_STATUS_STATE_STARTING_REPLAY: {
 230             'health': 'ok',
 231             'state_color': 'success',
 232             'state': 'Starting'
 233         },
 234         rbd.MIRROR_IMAGE_STATUS_STATE_REPLAYING: {
 235             'health': 'ok',
 236             'state_color': 'success',
 237             'state': 'Replaying'
 238         },
 239         rbd.MIRROR_IMAGE_STATUS_STATE_STOPPING_REPLAY: {
 240             'health': 'ok',
 241             'state_color': 'success',
 242             'state': 'Stopping'
 243         },
 244         rbd.MIRROR_IMAGE_STATUS_STATE_STOPPED: {
 245             'health': 'ok',
 246             'state_color': 'info',
 247             'state': 'Primary'
 248         }
 249     }
 250
 251     rbdctx = rbd.RBD()
 252     try:
 253         mirror_image_status = rbdctx.mirror_image_status_list(ioctx)
 254         data['mirror_images'] = sorted([
 255             dict({
 256                 'name': image['name'],
 257                 'description': image['description']
 258             }, **mirror_state['down' if not image['up'] else image['state']])
 259             for image in mirror_image_status
 260         ], key=lambda k: k['name'])
 261     except rbd.ImageNotFound:
 262         pass
 263     except:  # noqa pylint: disable=W0702
 264         logger.exception("Failed to list mirror image status %s", pool_name)
 265         raise
 266
 267     return data
 268
 269
 270 @ViewCache()
 271 def _get_content_data():  # pylint: disable=R0914
 272     pool_names = [pool['pool_name'] for pool in CephService.get_pool_list('rbd')
 273                   if pool.get('type', 1) == 1]
 274     _, data = get_daemons_and_pools()
 275     daemons = data.get('daemons', [])
 276     pool_stats = data.get('pools', {})
 277
 278     pools = []
 279     image_error = []
 280     image_syncing = []
 281     image_ready = []
 282     for pool_name in pool_names:
 283         _, pool = _get_pool_datum(pool_name)
 284         if not pool:
 285             pool = {}
 286
 287         stats = pool_stats.get(pool_name, {})
 288         if stats.get('mirror_mode', None) is None:
 289             continue
 290
 291         mirror_images = pool.get('mirror_images', [])
 292         for mirror_image in mirror_images:
 293             image = {
 294                 'pool_name': pool_name,
 295                 'name': mirror_image['name']
 296             }
 297
 298             if mirror_image['health'] == 'ok':
 299                 image.update({
 300                     'state_color': mirror_image['state_color'],
 301                     'state': mirror_image['state'],
 302                     'description': mirror_image['description']
 303                 })
 304                 image_ready.append(image)
 305             elif mirror_image['health'] == 'syncing':
 306                 p = re.compile("bootstrapping, IMAGE_COPY/COPY_OBJECT (.*)%")
 307                 image.update({
 308                     'progress': (p.findall(mirror_image['description']) or [0])[0]
 309                 })
 310                 image_syncing.append(image)
 311             else:
 312                 image.update({
 313                     'state_color': mirror_image['state_color'],
 314                     'state': mirror_image['state'],
 315                     'description': mirror_image['description']
 316                 })
 317                 image_error.append(image)
 318
 319         pools.append(dict({
 320             'name': pool_name
 321         }, **stats))
 322
 323     return {
 324         'daemons': daemons,
 325         'pools': pools,
 326         'image_error': image_error,
 327         'image_syncing': image_syncing,
 328         'image_ready': image_ready
 329     }
 330
 331
 332 def _reset_view_cache():
 333     get_daemons_and_pools.reset()
 334     _get_pool_datum.reset()
 335     _get_content_data.reset()
 336
 337
 338 @ApiController('/block/mirroring', Scope.RBD_MIRRORING)
 339 class RbdMirroring(BaseController):
 340
 341     @Endpoint(method='GET', path='site_name')
 342     @handle_rbd_mirror_error()
 343     @ReadPermission
 344     def get(self):
 345         return self._get_site_name()
 346
 347     @Endpoint(method='PUT', path='site_name')
 348     @handle_rbd_mirror_error()
 349     @UpdatePermission
 350     def set(self, site_name):
 351         rbd.RBD().mirror_site_name_set(mgr.rados, site_name)
 352         return self._get_site_name()
 353
 354     def _get_site_name(self):
 355         return {'site_name': rbd.RBD().mirror_site_name_get(mgr.rados)}
 356
 357
 358 @ApiController('/block/mirroring/summary', Scope.RBD_MIRRORING)
 359 class RbdMirroringSummary(BaseController):
 360
 361     @Endpoint()
 362     @handle_rbd_mirror_error()
 363     @ReadPermission
 364     def __call__(self):
 365         site_name = rbd.RBD().mirror_site_name_get(mgr.rados)
 366
 367         status, content_data = _get_content_data()
 368         return {'site_name': site_name,
 369                 'status': status,
 370                 'content_data': content_data}
 371
 372
 373 @ApiController('/block/mirroring/pool', Scope.RBD_MIRRORING)
 374 class RbdMirroringPoolMode(RESTController):
 375
 376     RESOURCE_ID = "pool_name"
 377     MIRROR_MODES = {
 378         rbd.RBD_MIRROR_MODE_DISABLED: 'disabled',
 379         rbd.RBD_MIRROR_MODE_IMAGE: 'image',
 380         rbd.RBD_MIRROR_MODE_POOL: 'pool'
 381     }
 382
 383     @handle_rbd_mirror_error()
 384     def get(self, pool_name):
 385         ioctx = mgr.rados.open_ioctx(pool_name)
 386         mode = rbd.RBD().mirror_mode_get(ioctx)
 387         data = {
 388             'mirror_mode': self.MIRROR_MODES.get(mode, 'unknown')
 389         }
 390         return data
 391
 392     @RbdMirroringTask('pool/edit', {'pool_name': '{pool_name}'}, 5.0)
 393     def set(self, pool_name, mirror_mode=None):
 394         def _edit(ioctx, mirror_mode=None):
 395             if mirror_mode:
 396                 mode_enum = {x[1]: x[0] for x in
 397                              self.MIRROR_MODES.items()}.get(mirror_mode, None)
 398                 if mode_enum is None:
 399                     raise rbd.Error('invalid mirror mode "{}"'.format(mirror_mode))
 400
 401                 current_mode_enum = rbd.RBD().mirror_mode_get(ioctx)
 402                 if mode_enum != current_mode_enum:
 403                     rbd.RBD().mirror_mode_set(ioctx, mode_enum)
 404                 _reset_view_cache()
 405
 406         return rbd_call(pool_name, None, _edit, mirror_mode)
 407
 408
 409 @ApiController('/block/mirroring/pool/{pool_name}/bootstrap',
 410                Scope.RBD_MIRRORING)
 411 class RbdMirroringPoolBootstrap(BaseController):
 412
 413     @Endpoint(method='POST', path='token')
 414     @handle_rbd_mirror_error()
 415     @UpdatePermission
 416     def create_token(self, pool_name):
 417         ioctx = mgr.rados.open_ioctx(pool_name)
 418         token = rbd.RBD().mirror_peer_bootstrap_create(ioctx)
 419         return {'token': token}
 420
 421     @Endpoint(method='POST', path='peer')
 422     @handle_rbd_mirror_error()
 423     @UpdatePermission
 424     def import_token(self, pool_name, direction, token):
 425         ioctx = mgr.rados.open_ioctx(pool_name)
 426
 427         directions = {
 428             'rx': rbd.RBD_MIRROR_PEER_DIRECTION_RX,
 429             'rx-tx': rbd.RBD_MIRROR_PEER_DIRECTION_RX_TX
 430         }
 431
 432         direction_enum = directions.get(direction)
 433         if direction_enum is None:
 434             raise rbd.Error('invalid direction "{}"'.format(direction))
 435
 436         rbd.RBD().mirror_peer_bootstrap_import(ioctx, direction_enum, token)
 437         return {}
 438
 439
 440 @ApiController('/block/mirroring/pool/{pool_name}/peer', Scope.RBD_MIRRORING)
 441 class RbdMirroringPoolPeer(RESTController):
 442
 443     RESOURCE_ID = "peer_uuid"
 444
 445     @handle_rbd_mirror_error()
 446     def list(self, pool_name):
 447         ioctx = mgr.rados.open_ioctx(pool_name)
 448         peer_list = rbd.RBD().mirror_peer_list(ioctx)
 449         return [x['uuid'] for x in peer_list]
 450
 451     @handle_rbd_mirror_error()
 452     def create(self, pool_name, cluster_name, client_id, mon_host=None,
 453                key=None):
 454         ioctx = mgr.rados.open_ioctx(pool_name)
 455         mode = rbd.RBD().mirror_mode_get(ioctx)
 456         if mode == rbd.RBD_MIRROR_MODE_DISABLED:
 457             raise rbd.Error('mirroring must be enabled')
 458
 459         uuid = rbd.RBD().mirror_peer_add(ioctx, cluster_name,
 460                                          'client.{}'.format(client_id))
 461
 462         attributes = {}
 463         if mon_host is not None:
 464             attributes[rbd.RBD_MIRROR_PEER_ATTRIBUTE_NAME_MON_HOST] = mon_host
 465         if key is not None:
 466             attributes[rbd.RBD_MIRROR_PEER_ATTRIBUTE_NAME_KEY] = key
 467         if attributes:
 468             rbd.RBD().mirror_peer_set_attributes(ioctx, uuid, attributes)
 469
 470         _reset_view_cache()
 471         return {'uuid': uuid}
 472
 473     @handle_rbd_mirror_error()
 474     def get(self, pool_name, peer_uuid):
 475         ioctx = mgr.rados.open_ioctx(pool_name)
 476         peer_list = rbd.RBD().mirror_peer_list(ioctx)
 477         peer = next((x for x in peer_list if x['uuid'] == peer_uuid), None)
 478         if not peer:
 479             raise cherrypy.HTTPError(404)
 480
 481         # convert full client name to just the client id
 482         peer['client_id'] = peer['client_name'].split('.', 1)[-1]
 483         del peer['client_name']
 484
 485         # convert direction enum to string
 486         directions = {
 487             rbd.RBD_MIRROR_PEER_DIRECTION_RX: 'rx',
 488             rbd.RBD_MIRROR_PEER_DIRECTION_TX: 'tx',
 489             rbd.RBD_MIRROR_PEER_DIRECTION_RX_TX: 'rx-tx'
 490         }
 491         peer['direction'] = directions[peer.get('direction', rbd.RBD_MIRROR_PEER_DIRECTION_RX)]
 492
 493         try:
 494             attributes = rbd.RBD().mirror_peer_get_attributes(ioctx, peer_uuid)
 495         except rbd.ImageNotFound:
 496             attributes = {}
 497
 498         peer['mon_host'] = attributes.get(rbd.RBD_MIRROR_PEER_ATTRIBUTE_NAME_MON_HOST, '')
 499         peer['key'] = attributes.get(rbd.RBD_MIRROR_PEER_ATTRIBUTE_NAME_KEY, '')
 500         return peer
 501
 502     @handle_rbd_mirror_error()
 503     def delete(self, pool_name, peer_uuid):
 504         ioctx = mgr.rados.open_ioctx(pool_name)
 505         rbd.RBD().mirror_peer_remove(ioctx, peer_uuid)
 506         _reset_view_cache()
 507
 508     @handle_rbd_mirror_error()
 509     def set(self, pool_name, peer_uuid, cluster_name=None, client_id=None,
 510             mon_host=None, key=None):
 511         ioctx = mgr.rados.open_ioctx(pool_name)
 512         if cluster_name:
 513             rbd.RBD().mirror_peer_set_cluster(ioctx, peer_uuid, cluster_name)
 514         if client_id:
 515             rbd.RBD().mirror_peer_set_client(ioctx, peer_uuid,
 516                                              'client.{}'.format(client_id))
 517
 518         if mon_host is not None or key is not None:
 519             try:
 520                 attributes = rbd.RBD().mirror_peer_get_attributes(ioctx, peer_uuid)
 521             except rbd.ImageNotFound:
 522                 attributes = {}
 523
 524             if mon_host is not None:
 525                 attributes[rbd.RBD_MIRROR_PEER_ATTRIBUTE_NAME_MON_HOST] = mon_host
 526             if key is not None:
 527                 attributes[rbd.RBD_MIRROR_PEER_ATTRIBUTE_NAME_KEY] = key
 528             rbd.RBD().mirror_peer_set_attributes(ioctx, peer_uuid, attributes)
 529
 530         _reset_view_cache()