ceph/src/pybind/mgr/dashboard/module.py

   1 # -*- coding: utf-8 -*-
   2 """
   3 ceph dashboard mgr plugin (based on CherryPy)
   4 """
   5 from __future__ import absolute_import
   6
   7 import collections
   8 import errno
   9 from distutils.version import StrictVersion
  10 import os
  11 import socket
  12 import tempfile
  13 import threading
  14 import time
  15 from uuid import uuid4
  16 from OpenSSL import crypto, SSL
  17 from mgr_module import MgrModule, MgrStandbyModule, Option, CLIWriteCommand
  18 from mgr_util import get_default_addr
  19
  20 try:
  21     import cherrypy
  22     from cherrypy._cptools import HandlerWrapperTool
  23 except ImportError:
  24     # To be picked up and reported by .can_run()
  25     cherrypy = None
  26
  27 from .services.sso import load_sso_db
  28
  29 # The SSL code in CherryPy 3.5.0 is buggy.  It was fixed long ago,
  30 # but 3.5.0 is still shipping in major linux distributions
  31 # (Fedora 27, Ubuntu Xenial), so we must monkey patch it to get SSL working.
  32 if cherrypy is not None:
  33     v = StrictVersion(cherrypy.__version__)
  34     # It was fixed in 3.7.0.  Exact lower bound version is probably earlier,
  35     # but 3.5.0 is what this monkey patch is tested on.
  36     if StrictVersion("3.5.0") <= v < StrictVersion("3.7.0"):
  37         from cherrypy.wsgiserver.wsgiserver2 import HTTPConnection,\
  38                                                     CP_fileobject
  39
  40         def fixed_init(hc_self, server, sock, makefile=CP_fileobject):
  41             hc_self.server = server
  42             hc_self.socket = sock
  43             hc_self.rfile = makefile(sock, "rb", hc_self.rbufsize)
  44             hc_self.wfile = makefile(sock, "wb", hc_self.wbufsize)
  45             hc_self.requests_seen = 0
  46
  47         HTTPConnection.__init__ = fixed_init
  48
  49 # When the CherryPy server in 3.2.2 (and later) starts it attempts to verify
  50 # that the ports its listening on are in fact bound. When using the any address
  51 # "::" it tries both ipv4 and ipv6, and in some environments (e.g. kubernetes)
  52 # ipv6 isn't yet configured / supported and CherryPy throws an uncaught
  53 # exception.
  54 if cherrypy is not None:
  55     v = StrictVersion(cherrypy.__version__)
  56     # the issue was fixed in 3.2.3. it's present in 3.2.2 (current version on
  57     # centos:7) and back to at least 3.0.0.
  58     if StrictVersion("3.1.2") <= v < StrictVersion("3.2.3"):
  59         # https://github.com/cherrypy/cherrypy/issues/1100
  60         from cherrypy.process import servers
  61         servers.wait_for_occupied_port = lambda host, port: None
  62
  63 if 'COVERAGE_ENABLED' in os.environ:
  64     import coverage
  65     __cov = coverage.Coverage(config_file="{}/.coveragerc".format(os.path.dirname(__file__)),
  66                               data_suffix=True)
  67
  68     cherrypy.engine.subscribe('start', __cov.start)
  69     cherrypy.engine.subscribe('after_request', __cov.save)
  70     cherrypy.engine.subscribe('stop', __cov.stop)
  71
  72 # pylint: disable=wrong-import-position
  73 from . import logger, mgr
  74 from .controllers import generate_routes, json_error_page
  75 from .grafana import push_local_dashboards
  76 from .tools import NotificationQueue, RequestLoggingTool, TaskManager, \
  77                    prepare_url_prefix, str_to_bool
  78 from .services.auth import AuthManager, AuthManagerTool, JwtManager
  79 from .services.sso import SSO_COMMANDS, \
  80                           handle_sso_command
  81 from .services.exception import dashboard_exception_handler
  82 from .settings import options_command_list, options_schema_list, \
  83                       handle_option_command
  84
  85 from .plugins import PLUGIN_MANAGER
  86 from .plugins import feature_toggles  # noqa # pylint: disable=unused-import
  87
  88
  89 # cherrypy likes to sys.exit on error.  don't let it take us down too!
  90 # pylint: disable=W0613
  91 def os_exit_noop(*args):
  92     pass
  93
  94
  95 # pylint: disable=W0212
  96 os._exit = os_exit_noop
  97
  98
  99 class ServerConfigException(Exception):
 100     pass
 101
 102
 103 class CherryPyConfig(object):
 104     """
 105     Class for common server configuration done by both active and
 106     standby module, especially setting up SSL.
 107     """
 108
 109     def __init__(self):
 110         self._stopping = threading.Event()
 111         self._url_prefix = ""
 112
 113         self.cert_tmp = None
 114         self.pkey_tmp = None
 115
 116     def shutdown(self):
 117         self._stopping.set()
 118
 119     @property
 120     def url_prefix(self):
 121         return self._url_prefix
 122
 123     # pylint: disable=too-many-branches
 124     def _configure(self):
 125         """
 126         Configure CherryPy and initialize self.url_prefix
 127
 128         :returns our URI
 129         """
 130         server_addr = self.get_localized_module_option(
 131             'server_addr', get_default_addr())
 132         ssl = self.get_localized_module_option('ssl', True)
 133         if not ssl:
 134             server_port = self.get_localized_module_option('server_port', 8080)
 135         else:
 136             server_port = self.get_localized_module_option('ssl_server_port', 8443)
 137
 138         if server_addr is None:
 139             raise ServerConfigException(
 140                 'no server_addr configured; '
 141                 'try "ceph config set mgr mgr/{}/{}/server_addr <ip>"'
 142                 .format(self.module_name, self.get_mgr_id()))
 143         self.log.info('server: ssl=%s host=%s port=%d', 'yes' if ssl else 'no',
 144                       server_addr, server_port)
 145
 146         # Initialize custom handlers.
 147         cherrypy.tools.authenticate = AuthManagerTool()
 148         cherrypy.tools.plugin_hooks = cherrypy.Tool(
 149             'before_handler',
 150             lambda: PLUGIN_MANAGER.hook.filter_request_before_handler(request=cherrypy.request),
 151             priority=10)
 152         cherrypy.tools.request_logging = RequestLoggingTool()
 153         cherrypy.tools.dashboard_exception_handler = HandlerWrapperTool(dashboard_exception_handler,
 154                                                                         priority=31)
 155
 156         # Apply the 'global' CherryPy configuration.
 157         config = {
 158             'engine.autoreload.on': False,
 159             'server.socket_host': server_addr,
 160             'server.socket_port': int(server_port),
 161             'error_page.default': json_error_page,
 162             'tools.request_logging.on': True,
 163             'tools.gzip.on': True,
 164             'tools.gzip.mime_types': [
 165                 # text/html and text/plain are the default types to compress
 166                 'text/html', 'text/plain',
 167                 # We also want JSON and JavaScript to be compressed
 168                 'application/json',
 169                 'application/javascript',
 170             ],
 171             'tools.json_in.on': True,
 172             'tools.json_in.force': False,
 173             'tools.plugin_hooks.on': True,
 174         }
 175
 176         if ssl:
 177             # SSL initialization
 178             cert = self.get_store("crt")
 179             if cert is not None:
 180                 self.cert_tmp = tempfile.NamedTemporaryFile()
 181                 self.cert_tmp.write(cert.encode('utf-8'))
 182                 self.cert_tmp.flush()  # cert_tmp must not be gc'ed
 183                 cert_fname = self.cert_tmp.name
 184             else:
 185                 cert_fname = self.get_localized_module_option('crt_file')
 186
 187             pkey = self.get_store("key")
 188             if pkey is not None:
 189                 self.pkey_tmp = tempfile.NamedTemporaryFile()
 190                 self.pkey_tmp.write(pkey.encode('utf-8'))
 191                 self.pkey_tmp.flush()  # pkey_tmp must not be gc'ed
 192                 pkey_fname = self.pkey_tmp.name
 193             else:
 194                 pkey_fname = self.get_localized_module_option('key_file')
 195
 196             if not cert_fname or not pkey_fname:
 197                 raise ServerConfigException('no certificate configured')
 198             if not os.path.isfile(cert_fname):
 199                 raise ServerConfigException('certificate %s does not exist' % cert_fname)
 200             if not os.path.isfile(pkey_fname):
 201                 raise ServerConfigException('private key %s does not exist' % pkey_fname)
 202
 203             # Do some validations to the private key and certificate:
 204             # - Check the type and format
 205             # - Check the certificate expiration date
 206             # - Check the consistency of the private key
 207             # - Check that the private key and certificate match up
 208             try:
 209                 with open(cert_fname) as f:
 210                     x509 = crypto.load_certificate(crypto.FILETYPE_PEM, f.read())
 211                     if x509.has_expired():
 212                         self.log.warning(
 213                             'Certificate {} has been expired'.format(cert_fname))
 214             except (ValueError, crypto.Error) as e:
 215                 raise ServerConfigException(
 216                     'Invalid certificate {}: {}'.format(cert_fname, str(e)))
 217             try:
 218                 with open(pkey_fname) as f:
 219                     pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, f.read())
 220                     pkey.check()
 221             except (ValueError, crypto.Error) as e:
 222                 raise ServerConfigException(
 223                     'Invalid private key {}: {}'.format(pkey_fname, str(e)))
 224             try:
 225                 context = SSL.Context(SSL.TLSv1_METHOD)
 226                 context.use_certificate_file(cert_fname, crypto.FILETYPE_PEM)
 227                 context.use_privatekey_file(pkey_fname, crypto.FILETYPE_PEM)
 228                 context.check_privatekey()
 229             except crypto.Error as e:
 230                 self.log.warning(
 231                     'Private key {} and certificate {} do not match up: {}'.format(
 232                         pkey_fname, cert_fname, str(e)))
 233
 234             config['server.ssl_module'] = 'builtin'
 235             config['server.ssl_certificate'] = cert_fname
 236             config['server.ssl_private_key'] = pkey_fname
 237
 238         cherrypy.config.update(config)
 239
 240         self._url_prefix = prepare_url_prefix(self.get_module_option('url_prefix',
 241                                                                      default=''))
 242
 243         uri = "{0}://{1}:{2}{3}/".format(
 244             'https' if ssl else 'http',
 245             socket.getfqdn() if server_addr == "::" else server_addr,
 246             server_port,
 247             self.url_prefix
 248         )
 249
 250         return uri
 251
 252     def await_configuration(self):
 253         """
 254         Block until configuration is ready (i.e. all needed keys are set)
 255         or self._stopping is set.
 256
 257         :returns URI of configured webserver
 258         """
 259         while not self._stopping.is_set():
 260             try:
 261                 uri = self._configure()
 262             except ServerConfigException as e:
 263                 self.log.info("Config not ready to serve, waiting: {0}".format(
 264                     e
 265                 ))
 266                 # Poll until a non-errored config is present
 267                 self._stopping.wait(5)
 268             else:
 269                 self.log.info("Configured CherryPy, starting engine...")
 270                 return uri
 271
 272
 273 class Module(MgrModule, CherryPyConfig):
 274     """
 275     dashboard module entrypoint
 276     """
 277
 278     COMMANDS = [
 279         {
 280             'cmd': 'dashboard set-jwt-token-ttl '
 281                    'name=seconds,type=CephInt',
 282             'desc': 'Set the JWT token TTL in seconds',
 283             'perm': 'w'
 284         },
 285         {
 286             'cmd': 'dashboard get-jwt-token-ttl',
 287             'desc': 'Get the JWT token TTL in seconds',
 288             'perm': 'r'
 289         },
 290         {
 291             "cmd": "dashboard create-self-signed-cert",
 292             "desc": "Create self signed certificate",
 293             "perm": "w"
 294         },
 295         {
 296             "cmd": "dashboard grafana dashboards update",
 297             "desc": "Push dashboards to Grafana",
 298             "perm": "w",
 299         },
 300     ]
 301     COMMANDS.extend(options_command_list())
 302     COMMANDS.extend(SSO_COMMANDS)
 303     PLUGIN_MANAGER.hook.register_commands()
 304
 305     MODULE_OPTIONS = [
 306         Option(name='server_addr', type='str', default=get_default_addr()),
 307         Option(name='server_port', type='int', default=8080),
 308         Option(name='ssl_server_port', type='int', default=8443),
 309         Option(name='jwt_token_ttl', type='int', default=28800),
 310         Option(name='password', type='str', default=''),
 311         Option(name='url_prefix', type='str', default=''),
 312         Option(name='username', type='str', default=''),
 313         Option(name='key_file', type='str', default=''),
 314         Option(name='crt_file', type='str', default=''),
 315         Option(name='ssl', type='bool', default=True)
 316     ]
 317     MODULE_OPTIONS.extend(options_schema_list())
 318     for options in PLUGIN_MANAGER.hook.get_options() or []:
 319         MODULE_OPTIONS.extend(options)
 320
 321     __pool_stats = collections.defaultdict(lambda: collections.defaultdict(
 322         lambda: collections.deque(maxlen=10)))
 323
 324     def __init__(self, *args, **kwargs):
 325         super(Module, self).__init__(*args, **kwargs)
 326         CherryPyConfig.__init__(self)
 327
 328         mgr.init(self)
 329
 330         self._stopping = threading.Event()
 331         self.shutdown_event = threading.Event()
 332
 333         self.ACCESS_CTRL_DB = None
 334         self.SSO_DB = None
 335
 336     @classmethod
 337     def can_run(cls):
 338         if cherrypy is None:
 339             return False, "Missing dependency: cherrypy"
 340
 341         if not os.path.exists(cls.get_frontend_path()):
 342             return False, "Frontend assets not found: incomplete build?"
 343
 344         return True, ""
 345
 346     @classmethod
 347     def get_frontend_path(cls):
 348         current_dir = os.path.dirname(os.path.abspath(__file__))
 349         return os.path.join(current_dir, 'frontend/dist')
 350
 351     def serve(self):
 352         AuthManager.initialize()
 353         load_sso_db()
 354
 355         uri = self.await_configuration()
 356         if uri is None:
 357             # We were shut down while waiting
 358             return
 359
 360         # Publish the URI that others may use to access the service we're
 361         # about to start serving
 362         self.set_uri(uri)
 363
 364         mapper, parent_urls = generate_routes(self.url_prefix)
 365
 366         config = {
 367             self.url_prefix or '/': {
 368                 'tools.staticdir.on': True,
 369                 'tools.staticdir.dir': self.get_frontend_path(),
 370                 'tools.staticdir.index': 'index.html'
 371             }
 372         }
 373         for purl in parent_urls:
 374             config[purl] = {
 375                 'request.dispatch': mapper
 376             }
 377         cherrypy.tree.mount(None, config=config)
 378
 379         PLUGIN_MANAGER.hook.setup()
 380
 381         cherrypy.engine.start()
 382         NotificationQueue.start_queue()
 383         TaskManager.init()
 384         logger.info('Engine started.')
 385         update_dashboards = str_to_bool(
 386             self.get_module_option('GRAFANA_UPDATE_DASHBOARDS', 'False'))
 387         if update_dashboards:
 388             logger.info('Starting Grafana dashboard task')
 389             TaskManager.run(
 390                 'grafana/dashboards/update',
 391                 {},
 392                 push_local_dashboards,
 393                 kwargs=dict(tries=10, sleep=60),
 394             )
 395         # wait for the shutdown event
 396         self.shutdown_event.wait()
 397         self.shutdown_event.clear()
 398         NotificationQueue.stop()
 399         cherrypy.engine.stop()
 400         logger.info('Engine stopped')
 401
 402     def shutdown(self):
 403         super(Module, self).shutdown()
 404         CherryPyConfig.shutdown(self)
 405         logger.info('Stopping engine...')
 406         self.shutdown_event.set()
 407
 408     @CLIWriteCommand("dashboard set-ssl-certificate",
 409                      "name=mgr_id,type=CephString,req=false")
 410     def set_ssl_certificate(self, mgr_id=None, inbuf=None):
 411         if inbuf is None:
 412             return -errno.EINVAL, '',\
 413                    'Please specify the certificate file with "-i" option'
 414         if mgr_id is not None:
 415             self.set_store('{}/crt'.format(mgr_id), inbuf)
 416         else:
 417             self.set_store('crt', inbuf)
 418         return 0, 'SSL certificate updated', ''
 419
 420     @CLIWriteCommand("dashboard set-ssl-certificate-key",
 421                      "name=mgr_id,type=CephString,req=false")
 422     def set_ssl_certificate_key(self, mgr_id=None, inbuf=None):
 423         if inbuf is None:
 424             return -errno.EINVAL, '',\
 425                    'Please specify the certificate key file with "-i" option'
 426         if mgr_id is not None:
 427             self.set_store('{}/key'.format(mgr_id), inbuf)
 428         else:
 429             self.set_store('key', inbuf)
 430         return 0, 'SSL certificate key updated', ''
 431
 432     def handle_command(self, inbuf, cmd):
 433         # pylint: disable=too-many-return-statements
 434         res = handle_option_command(cmd)
 435         if res[0] != -errno.ENOSYS:
 436             return res
 437         res = handle_sso_command(cmd)
 438         if res[0] != -errno.ENOSYS:
 439             return res
 440         if cmd['prefix'] == 'dashboard set-jwt-token-ttl':
 441             self.set_module_option('jwt_token_ttl', str(cmd['seconds']))
 442             return 0, 'JWT token TTL updated', ''
 443         if cmd['prefix'] == 'dashboard get-jwt-token-ttl':
 444             ttl = self.get_module_option('jwt_token_ttl', JwtManager.JWT_TOKEN_TTL)
 445             return 0, str(ttl), ''
 446         if cmd['prefix'] == 'dashboard create-self-signed-cert':
 447             self.create_self_signed_cert()
 448             return 0, 'Self-signed certificate created', ''
 449         if cmd['prefix'] == 'dashboard grafana dashboards update':
 450             push_local_dashboards()
 451             return 0, 'Grafana dashboards updated', ''
 452
 453         return (-errno.EINVAL, '', 'Command not found \'{0}\''
 454                 .format(cmd['prefix']))
 455
 456     def create_self_signed_cert(self):
 457         # create a key pair
 458         pkey = crypto.PKey()
 459         pkey.generate_key(crypto.TYPE_RSA, 2048)
 460
 461         # create a self-signed cert
 462         cert = crypto.X509()
 463         cert.get_subject().O = "IT"
 464         cert.get_subject().CN = "ceph-dashboard"
 465         cert.set_serial_number(int(uuid4()))
 466         cert.gmtime_adj_notBefore(0)
 467         cert.gmtime_adj_notAfter(10*365*24*60*60)
 468         cert.set_issuer(cert.get_subject())
 469         cert.set_pubkey(pkey)
 470         cert.sign(pkey, 'sha512')
 471
 472         cert = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
 473         self.set_store('crt', cert.decode('utf-8'))
 474
 475         pkey = crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey)
 476         self.set_store('key', pkey.decode('utf-8'))
 477
 478     def notify(self, notify_type, notify_id):
 479         NotificationQueue.new_notification(notify_type, notify_id)
 480
 481     def get_updated_pool_stats(self):
 482         df = self.get('df')
 483         pool_stats = {p['id']: p['stats'] for p in df['pools']}
 484         now = time.time()
 485         for pool_id, stats in pool_stats.items():
 486             for stat_name, stat_val in stats.items():
 487                 self.__pool_stats[pool_id][stat_name].append((now, stat_val))
 488
 489         return self.__pool_stats
 490
 491
 492 class StandbyModule(MgrStandbyModule, CherryPyConfig):
 493     def __init__(self, *args, **kwargs):
 494         super(StandbyModule, self).__init__(*args, **kwargs)
 495         CherryPyConfig.__init__(self)
 496         self.shutdown_event = threading.Event()
 497
 498         # We can set the global mgr instance to ourselves even though
 499         # we're just a standby, because it's enough for logging.
 500         mgr.init(self)
 501
 502     def serve(self):
 503         uri = self.await_configuration()
 504         if uri is None:
 505             # We were shut down while waiting
 506             return
 507
 508         module = self
 509
 510         class Root(object):
 511             @cherrypy.expose
 512             def index(self):
 513                 active_uri = module.get_active_uri()
 514                 if active_uri:
 515                     module.log.info("Redirecting to active '%s'", active_uri)
 516                     raise cherrypy.HTTPRedirect(active_uri)
 517                 else:
 518                     template = """
 519                 <html>
 520                     <!-- Note: this is only displayed when the standby
 521                          does not know an active URI to redirect to, otherwise
 522                          a simple redirect is returned instead -->
 523                     <head>
 524                         <title>Ceph</title>
 525                         <meta http-equiv="refresh" content="{delay}">
 526                     </head>
 527                     <body>
 528                         No active ceph-mgr instance is currently running
 529                         the dashboard. A failover may be in progress.
 530                         Retrying in {delay} seconds...
 531                     </body>
 532                 </html>
 533                     """
 534                     return template.format(delay=5)
 535
 536         cherrypy.tree.mount(Root(), "{}/".format(self.url_prefix), {})
 537         self.log.info("Starting engine...")
 538         cherrypy.engine.start()
 539         self.log.info("Engine started...")
 540         # Wait for shutdown event
 541         self.shutdown_event.wait()
 542         self.shutdown_event.clear()
 543         cherrypy.engine.stop()
 544         self.log.info("Engine stopped.")
 545
 546     def shutdown(self):
 547         CherryPyConfig.shutdown(self)
 548
 549         self.log.info("Stopping engine...")
 550         self.shutdown_event.set()
 551         self.log.info("Stopped engine...")