ceph/src/pybind/mgr/dashboard/controllers/cephfs.py

   1 # -*- coding: utf-8 -*-
   2 from __future__ import absolute_import
   3
   4 from collections import defaultdict
   5
   6 import os
   7
   8 import cherrypy
   9 import cephfs
  10
  11 from . import ApiController, ControllerDoc, RESTController, UiApiController, \
  12     allow_empty_body
  13 from .. import mgr
  14 from ..exceptions import DashboardException
  15 from ..security import Scope
  16 from ..services.cephfs import CephFS as CephFS_
  17 from ..services.ceph_service import CephService
  18 from ..tools import ViewCache
  19
  20
  21 @ApiController('/cephfs', Scope.CEPHFS)
  22 class CephFS(RESTController):
  23     def __init__(self):  # pragma: no cover
  24         super(CephFS, self).__init__()
  25
  26         # Stateful instances of CephFSClients, hold cached results.  Key to
  27         # dict is FSCID
  28         self.cephfs_clients = {}
  29
  30     def list(self):
  31         fsmap = mgr.get("fs_map")
  32         return fsmap['filesystems']
  33
  34     def get(self, fs_id):
  35         fs_id = self.fs_id_to_int(fs_id)
  36         return self.fs_status(fs_id)
  37
  38     @RESTController.Resource('GET')
  39     def clients(self, fs_id):
  40         fs_id = self.fs_id_to_int(fs_id)
  41
  42         return self._clients(fs_id)
  43
  44     @RESTController.Resource('DELETE', path='/client/{client_id}')
  45     def evict(self, fs_id, client_id):
  46         fs_id = self.fs_id_to_int(fs_id)
  47         client_id = self.client_id_to_int(client_id)
  48
  49         return self._evict(fs_id, client_id)
  50
  51     @RESTController.Resource('GET')
  52     def mds_counters(self, fs_id, counters=None):
  53         fs_id = self.fs_id_to_int(fs_id)
  54         return self._mds_counters(fs_id, counters)
  55
  56     def _mds_counters(self, fs_id, counters=None):
  57         """
  58         Result format: map of daemon name to map of counter to list of datapoints
  59         rtype: dict[str, dict[str, list]]
  60         """
  61
  62         if counters is None:
  63             # Opinionated list of interesting performance counters for the GUI
  64             counters = [
  65                 "mds_server.handle_client_request",
  66                 "mds_log.ev",
  67                 "mds_cache.num_strays",
  68                 "mds.exported",
  69                 "mds.exported_inodes",
  70                 "mds.imported",
  71                 "mds.imported_inodes",
  72                 "mds.inodes",
  73                 "mds.caps",
  74                 "mds.subtrees",
  75                 "mds_mem.ino"
  76             ]
  77
  78         result = {}  # type: dict
  79         mds_names = self._get_mds_names(fs_id)
  80
  81         for mds_name in mds_names:
  82             result[mds_name] = {}
  83             for counter in counters:
  84                 data = mgr.get_counter("mds", mds_name, counter)
  85                 if data is not None:
  86                     result[mds_name][counter] = data[counter]
  87                 else:
  88                     result[mds_name][counter] = []
  89
  90         return dict(result)
  91
  92     @staticmethod
  93     def fs_id_to_int(fs_id):
  94         try:
  95             return int(fs_id)
  96         except ValueError:
  97             raise DashboardException(code='invalid_cephfs_id',
  98                                      msg="Invalid cephfs ID {}".format(fs_id),
  99                                      component='cephfs')
 100
 101     @staticmethod
 102     def client_id_to_int(client_id):
 103         try:
 104             return int(client_id)
 105         except ValueError:
 106             raise DashboardException(code='invalid_cephfs_client_id',
 107                                      msg="Invalid cephfs client ID {}".format(client_id),
 108                                      component='cephfs')
 109
 110     def _get_mds_names(self, filesystem_id=None):
 111         names = []
 112
 113         fsmap = mgr.get("fs_map")
 114         for fs in fsmap['filesystems']:
 115             if filesystem_id is not None and fs['id'] != filesystem_id:
 116                 continue
 117             names.extend([info['name']
 118                           for _, info in fs['mdsmap']['info'].items()])
 119
 120         if filesystem_id is None:
 121             names.extend(info['name'] for info in fsmap['standbys'])
 122
 123         return names
 124
 125     def _append_mds_metadata(self, mds_versions, metadata_key):
 126         metadata = mgr.get_metadata('mds', metadata_key)
 127         if metadata is None:
 128             return
 129         mds_versions[metadata.get('ceph_version', 'unknown')].append(metadata_key)
 130
 131     # pylint: disable=too-many-statements,too-many-branches
 132     def fs_status(self, fs_id):
 133         mds_versions = defaultdict(list)  # type: dict
 134
 135         fsmap = mgr.get("fs_map")
 136         filesystem = None
 137         for fs in fsmap['filesystems']:
 138             if fs['id'] == fs_id:
 139                 filesystem = fs
 140                 break
 141
 142         if filesystem is None:
 143             raise cherrypy.HTTPError(404,
 144                                      "CephFS id {0} not found".format(fs_id))
 145
 146         rank_table = []
 147
 148         mdsmap = filesystem['mdsmap']
 149
 150         client_count = 0
 151
 152         for rank in mdsmap["in"]:
 153             up = "mds_{0}".format(rank) in mdsmap["up"]
 154             if up:
 155                 gid = mdsmap['up']["mds_{0}".format(rank)]
 156                 info = mdsmap['info']['gid_{0}'.format(gid)]
 157                 dns = mgr.get_latest("mds", info['name'], "mds_mem.dn")
 158                 inos = mgr.get_latest("mds", info['name'], "mds_mem.ino")
 159
 160                 if rank == 0:
 161                     client_count = mgr.get_latest("mds", info['name'],
 162                                                   "mds_sessions.session_count")
 163                 elif client_count == 0:
 164                     # In case rank 0 was down, look at another rank's
 165                     # sessionmap to get an indication of clients.
 166                     client_count = mgr.get_latest("mds", info['name'],
 167                                                   "mds_sessions.session_count")
 168
 169                 laggy = "laggy_since" in info
 170
 171                 state = info['state'].split(":")[1]
 172                 if laggy:
 173                     state += "(laggy)"
 174
 175                 # Populate based on context of state, e.g. client
 176                 # ops for an active daemon, replay progress, reconnect
 177                 # progress
 178                 if state == "active":
 179                     activity = CephService.get_rate("mds",
 180                                                     info['name'],
 181                                                     "mds_server.handle_client_request")
 182                 else:
 183                     activity = 0.0  # pragma: no cover
 184
 185                 self._append_mds_metadata(mds_versions, info['name'])
 186                 rank_table.append(
 187                     {
 188                         "rank": rank,
 189                         "state": state,
 190                         "mds": info['name'],
 191                         "activity": activity,
 192                         "dns": dns,
 193                         "inos": inos
 194                     }
 195                 )
 196
 197             else:
 198                 rank_table.append(
 199                     {
 200                         "rank": rank,
 201                         "state": "failed",
 202                         "mds": "",
 203                         "activity": 0.0,
 204                         "dns": 0,
 205                         "inos": 0
 206                     }
 207                 )
 208
 209         # Find the standby replays
 210         # pylint: disable=unused-variable
 211         for gid_str, daemon_info in mdsmap['info'].items():
 212             if daemon_info['state'] != "up:standby-replay":
 213                 continue
 214
 215             inos = mgr.get_latest("mds", daemon_info['name'], "mds_mem.ino")
 216             dns = mgr.get_latest("mds", daemon_info['name'], "mds_mem.dn")
 217
 218             activity = CephService.get_rate(
 219                 "mds", daemon_info['name'], "mds_log.replay")
 220
 221             rank_table.append(
 222                 {
 223                     "rank": "{0}-s".format(daemon_info['rank']),
 224                     "state": "standby-replay",
 225                     "mds": daemon_info['name'],
 226                     "activity": activity,
 227                     "dns": dns,
 228                     "inos": inos
 229                 }
 230             )
 231
 232         df = mgr.get("df")
 233         pool_stats = {p['id']: p['stats'] for p in df['pools']}
 234         osdmap = mgr.get("osd_map")
 235         pools = {p['pool']: p for p in osdmap['pools']}
 236         metadata_pool_id = mdsmap['metadata_pool']
 237         data_pool_ids = mdsmap['data_pools']
 238
 239         pools_table = []
 240         for pool_id in [metadata_pool_id] + data_pool_ids:
 241             pool_type = "metadata" if pool_id == metadata_pool_id else "data"
 242             stats = pool_stats[pool_id]
 243             pools_table.append({
 244                 "pool": pools[pool_id]['pool_name'],
 245                 "type": pool_type,
 246                 "used": stats['bytes_used'],
 247                 "avail": stats['max_avail']
 248             })
 249
 250         standby_table = []
 251         for standby in fsmap['standbys']:
 252             self._append_mds_metadata(mds_versions, standby['name'])
 253             standby_table.append({
 254                 'name': standby['name']
 255             })
 256
 257         return {
 258             "cephfs": {
 259                 "id": fs_id,
 260                 "name": mdsmap['fs_name'],
 261                 "client_count": client_count,
 262                 "ranks": rank_table,
 263                 "pools": pools_table
 264             },
 265             "standbys": standby_table,
 266             "versions": mds_versions
 267         }
 268
 269     def _clients(self, fs_id):
 270         cephfs_clients = self.cephfs_clients.get(fs_id, None)
 271         if cephfs_clients is None:
 272             cephfs_clients = CephFSClients(mgr, fs_id)
 273             self.cephfs_clients[fs_id] = cephfs_clients
 274
 275         try:
 276             status, clients = cephfs_clients.get()
 277         except AttributeError:
 278             raise cherrypy.HTTPError(404,
 279                                      "No cephfs with id {0}".format(fs_id))
 280
 281         if clients is None:
 282             raise cherrypy.HTTPError(404,
 283                                      "No cephfs with id {0}".format(fs_id))
 284
 285         # Decorate the metadata with some fields that will be
 286         # indepdendent of whether it's a kernel or userspace
 287         # client, so that the javascript doesn't have to grok that.
 288         for client in clients:
 289             if "ceph_version" in client['client_metadata']:  # pragma: no cover - no complexity
 290                 client['type'] = "userspace"
 291                 client['version'] = client['client_metadata']['ceph_version']
 292                 client['hostname'] = client['client_metadata']['hostname']
 293             elif "kernel_version" in client['client_metadata']:  # pragma: no cover - no complexity
 294                 client['type'] = "kernel"
 295                 client['version'] = client['client_metadata']['kernel_version']
 296                 client['hostname'] = client['client_metadata']['hostname']
 297             else:  # pragma: no cover - no complexity there
 298                 client['type'] = "unknown"
 299                 client['version'] = ""
 300                 client['hostname'] = ""
 301
 302         return {
 303             'status': status,
 304             'data': clients
 305         }
 306
 307     def _evict(self, fs_id, client_id):
 308         clients = self._clients(fs_id)
 309         if not [c for c in clients['data'] if c['id'] == client_id]:
 310             raise cherrypy.HTTPError(404,
 311                                      "Client {0} does not exist in cephfs {1}".format(client_id,
 312                                                                                       fs_id))
 313         CephService.send_command('mds', 'client evict',
 314                                  srv_spec='{0}:0'.format(fs_id), id=client_id)
 315
 316     @staticmethod
 317     def _cephfs_instance(fs_id):
 318         """
 319         :param fs_id: The filesystem identifier.
 320         :type fs_id: int | str
 321         :return: A instance of the CephFS class.
 322         """
 323         fs_name = CephFS_.fs_name_from_id(fs_id)
 324         if fs_name is None:
 325             raise cherrypy.HTTPError(404, "CephFS id {} not found".format(fs_id))
 326         return CephFS_(fs_name)
 327
 328     @RESTController.Resource('GET')
 329     def get_root_directory(self, fs_id):
 330         """
 331         The root directory that can't be fetched using ls_dir (api).
 332         :param fs_id: The filesystem identifier.
 333         :return: The root directory
 334         :rtype: dict
 335         """
 336         try:
 337             return self._get_root_directory(self._cephfs_instance(fs_id))
 338         except (cephfs.PermissionError, cephfs.ObjectNotFound):  # pragma: no cover
 339             return None
 340
 341     def _get_root_directory(self, cfs):
 342         """
 343         The root directory that can't be fetched using ls_dir (api).
 344         It's used in ls_dir (ui-api) and in get_root_directory (api).
 345         :param cfs: CephFS service instance
 346         :type cfs: CephFS
 347         :return: The root directory
 348         :rtype: dict
 349         """
 350         return cfs.get_directory(os.sep.encode())
 351
 352     @RESTController.Resource('GET')
 353     def ls_dir(self, fs_id, path=None, depth=1):
 354         """
 355         List directories of specified path.
 356         :param fs_id: The filesystem identifier.
 357         :param path: The path where to start listing the directory content.
 358           Defaults to '/' if not set.
 359         :type path: str | bytes
 360         :param depth: The number of steps to go down the directory tree.
 361         :type depth: int | str
 362         :return: The names of the directories below the specified path.
 363         :rtype: list
 364         """
 365         path = self._set_ls_dir_path(path)
 366         try:
 367             cfs = self._cephfs_instance(fs_id)
 368             paths = cfs.ls_dir(path, depth)
 369         except (cephfs.PermissionError, cephfs.ObjectNotFound):  # pragma: no cover
 370             paths = []
 371         return paths
 372
 373     def _set_ls_dir_path(self, path):
 374         """
 375         Transforms input path parameter of ls_dir methods (api and ui-api).
 376         :param path: The path where to start listing the directory content.
 377           Defaults to '/' if not set.
 378         :type path: str | bytes
 379         :return: Normalized path or root path
 380         :return: str
 381         """
 382         if path is None:
 383             path = os.sep
 384         else:
 385             path = os.path.normpath(path)
 386         return path
 387
 388     @RESTController.Resource('POST')
 389     @allow_empty_body
 390     def mk_dirs(self, fs_id, path):
 391         """
 392         Create a directory.
 393         :param fs_id: The filesystem identifier.
 394         :param path: The path of the directory.
 395         """
 396         cfs = self._cephfs_instance(fs_id)
 397         cfs.mk_dirs(path)
 398
 399     @RESTController.Resource('POST')
 400     @allow_empty_body
 401     def rm_dir(self, fs_id, path):
 402         """
 403         Remove a directory.
 404         :param fs_id: The filesystem identifier.
 405         :param path: The path of the directory.
 406         """
 407         cfs = self._cephfs_instance(fs_id)
 408         cfs.rm_dir(path)
 409
 410     @RESTController.Resource('POST')
 411     @allow_empty_body
 412     def mk_snapshot(self, fs_id, path, name=None):
 413         """
 414         Create a snapshot.
 415         :param fs_id: The filesystem identifier.
 416         :param path: The path of the directory.
 417         :param name: The name of the snapshot. If not specified,
 418             a name using the current time in RFC3339 UTC format
 419             will be generated.
 420         :return: The name of the snapshot.
 421         :rtype: str
 422         """
 423         cfs = self._cephfs_instance(fs_id)
 424         return cfs.mk_snapshot(path, name)
 425
 426     @RESTController.Resource('POST')
 427     @allow_empty_body
 428     def rm_snapshot(self, fs_id, path, name):
 429         """
 430         Remove a snapshot.
 431         :param fs_id: The filesystem identifier.
 432         :param path: The path of the directory.
 433         :param name: The name of the snapshot.
 434         """
 435         cfs = self._cephfs_instance(fs_id)
 436         cfs.rm_snapshot(path, name)
 437
 438     @RESTController.Resource('GET')
 439     def get_quotas(self, fs_id, path):
 440         """
 441         Get the quotas of the specified path.
 442         :param fs_id: The filesystem identifier.
 443         :param path: The path of the directory/file.
 444         :return: Returns a dictionary containing 'max_bytes'
 445             and 'max_files'.
 446         :rtype: dict
 447         """
 448         cfs = self._cephfs_instance(fs_id)
 449         return cfs.get_quotas(path)
 450
 451     @RESTController.Resource('POST')
 452     @allow_empty_body
 453     def set_quotas(self, fs_id, path, max_bytes=None, max_files=None):
 454         """
 455         Set the quotas of the specified path.
 456         :param fs_id: The filesystem identifier.
 457         :param path: The path of the directory/file.
 458         :param max_bytes: The byte limit.
 459         :param max_files: The file limit.
 460         """
 461         cfs = self._cephfs_instance(fs_id)
 462         return cfs.set_quotas(path, max_bytes, max_files)
 463
 464
 465 class CephFSClients(object):
 466     def __init__(self, module_inst, fscid):
 467         self._module = module_inst
 468         self.fscid = fscid
 469
 470     @ViewCache()
 471     def get(self):
 472         return CephService.send_command('mds', 'session ls', srv_spec='{0}:0'.format(self.fscid))
 473
 474
 475 @UiApiController('/cephfs', Scope.CEPHFS)
 476 @ControllerDoc("Dashboard UI helper function; not part of the public API", "CephFSUi")
 477 class CephFsUi(CephFS):
 478     RESOURCE_ID = 'fs_id'
 479
 480     @RESTController.Resource('GET')
 481     def tabs(self, fs_id):
 482         data = {}
 483         fs_id = self.fs_id_to_int(fs_id)
 484
 485         # Needed for detail tab
 486         fs_status = self.fs_status(fs_id)
 487         for pool in fs_status['cephfs']['pools']:
 488             pool['size'] = pool['used'] + pool['avail']
 489         data['pools'] = fs_status['cephfs']['pools']
 490         data['ranks'] = fs_status['cephfs']['ranks']
 491         data['name'] = fs_status['cephfs']['name']
 492         data['standbys'] = ', '.join([x['name'] for x in fs_status['standbys']])
 493         counters = self._mds_counters(fs_id)
 494         for k, v in counters.items():
 495             v['name'] = k
 496         data['mds_counters'] = counters
 497
 498         # Needed for client tab
 499         data['clients'] = self._clients(fs_id)
 500
 501         return data
 502
 503     @RESTController.Resource('GET')
 504     def ls_dir(self, fs_id, path=None, depth=1):
 505         """
 506         The difference to the API version is that the root directory will be send when listing
 507         the root directory.
 508         To only do one request this endpoint was created.
 509         :param fs_id: The filesystem identifier.
 510         :type fs_id: int | str
 511         :param path: The path where to start listing the directory content.
 512           Defaults to '/' if not set.
 513         :type path: str | bytes
 514         :param depth: The number of steps to go down the directory tree.
 515         :type depth: int | str
 516         :return: The names of the directories below the specified path.
 517         :rtype: list
 518         """
 519         path = self._set_ls_dir_path(path)
 520         try:
 521             cfs = self._cephfs_instance(fs_id)
 522             paths = cfs.ls_dir(path, depth)
 523             if path == os.sep:
 524                 paths = [self._get_root_directory(cfs)] + paths
 525         except (cephfs.PermissionError, cephfs.ObjectNotFound):  # pragma: no cover
 526             paths = []
 527         return paths