ceph/src/pybind/mgr/dashboard/module.py

   1 # -*- coding: utf-8 -*-
   2 """
   3 ceph dashboard mgr plugin (based on CherryPy)
   4 """
   5 import collections
   6 import errno
   7 import logging
   8 import os
   9 import socket
  10 import ssl
  11 import sys
  12 import tempfile
  13 import threading
  14 import time
  15 from typing import TYPE_CHECKING, Optional
  16 from urllib.parse import urlparse
  17
  18 if TYPE_CHECKING:
  19     if sys.version_info >= (3, 8):
  20         from typing import Literal
  21     else:
  22         from typing_extensions import Literal
  23
  24 from mgr_module import CLIReadCommand, CLIWriteCommand, HandleCommandResult, \
  25     MgrModule, MgrStandbyModule, NotifyType, Option, _get_localized_key
  26 from mgr_util import ServerConfigException, build_url, \
  27     create_self_signed_cert, get_default_addr, verify_tls_files
  28
  29 from . import mgr
  30 from .controllers import Router, json_error_page
  31 from .grafana import push_local_dashboards
  32 from .services.auth import AuthManager, AuthManagerTool, JwtManager
  33 from .services.exception import dashboard_exception_handler
  34 from .services.rgw_client import configure_rgw_credentials
  35 from .services.sso import SSO_COMMANDS, handle_sso_command
  36 from .settings import handle_option_command, options_command_list, options_schema_list
  37 from .tools import NotificationQueue, RequestLoggingTool, TaskManager, \
  38     prepare_url_prefix, str_to_bool
  39
  40 try:
  41     import cherrypy
  42     from cherrypy._cptools import HandlerWrapperTool
  43 except ImportError:
  44     # To be picked up and reported by .can_run()
  45     cherrypy = None
  46
  47 from .services.sso import load_sso_db
  48
  49 if cherrypy is not None:
  50     from .cherrypy_backports import patch_cherrypy
  51     patch_cherrypy(cherrypy.__version__)
  52
  53 # pylint: disable=wrong-import-position
  54 from .plugins import PLUGIN_MANAGER, debug, feature_toggles, motd  # isort:skip # noqa E501 # pylint: disable=unused-import
  55
  56 PLUGIN_MANAGER.hook.init()
  57
  58
  59 # cherrypy likes to sys.exit on error.  don't let it take us down too!
  60 # pylint: disable=W0613
  61 def os_exit_noop(*args):
  62     pass
  63
  64
  65 # pylint: disable=W0212
  66 os._exit = os_exit_noop
  67
  68
  69 logger = logging.getLogger(__name__)
  70
  71
  72 class CherryPyConfig(object):
  73     """
  74     Class for common server configuration done by both active and
  75     standby module, especially setting up SSL.
  76     """
  77
  78     def __init__(self):
  79         self._stopping = threading.Event()
  80         self._url_prefix = ""
  81
  82         self.cert_tmp = None
  83         self.pkey_tmp = None
  84
  85     def shutdown(self):
  86         self._stopping.set()
  87
  88     @property
  89     def url_prefix(self):
  90         return self._url_prefix
  91
  92     @staticmethod
  93     def update_cherrypy_config(config):
  94         PLUGIN_MANAGER.hook.configure_cherrypy(config=config)
  95         cherrypy.config.update(config)
  96
  97     # pylint: disable=too-many-branches
  98     def _configure(self):
  99         """
 100         Configure CherryPy and initialize self.url_prefix
 101
 102         :returns our URI
 103         """
 104         server_addr = self.get_localized_module_option(  # type: ignore
 105             'server_addr', get_default_addr())
 106         use_ssl = self.get_localized_module_option('ssl', True)  # type: ignore
 107         if not use_ssl:
 108             server_port = self.get_localized_module_option('server_port', 8080)  # type: ignore
 109         else:
 110             server_port = self.get_localized_module_option('ssl_server_port', 8443)  # type: ignore
 111
 112         if server_addr is None:
 113             raise ServerConfigException(
 114                 'no server_addr configured; '
 115                 'try "ceph config set mgr mgr/{}/{}/server_addr <ip>"'
 116                 .format(self.module_name, self.get_mgr_id()))  # type: ignore
 117         self.log.info('server: ssl=%s host=%s port=%d', 'yes' if use_ssl else 'no',  # type: ignore
 118                       server_addr, server_port)
 119
 120         # Initialize custom handlers.
 121         cherrypy.tools.authenticate = AuthManagerTool()
 122         self.configure_cors()
 123         cherrypy.tools.plugin_hooks_filter_request = cherrypy.Tool(
 124             'before_handler',
 125             lambda: PLUGIN_MANAGER.hook.filter_request_before_handler(request=cherrypy.request),
 126             priority=1)
 127         cherrypy.tools.request_logging = RequestLoggingTool()
 128         cherrypy.tools.dashboard_exception_handler = HandlerWrapperTool(dashboard_exception_handler,
 129                                                                         priority=31)
 130
 131         cherrypy.log.access_log.propagate = False
 132         cherrypy.log.error_log.propagate = False
 133
 134         # Apply the 'global' CherryPy configuration.
 135         config = {
 136             'engine.autoreload.on': False,
 137             'server.socket_host': server_addr,
 138             'server.socket_port': int(server_port),
 139             'error_page.default': json_error_page,
 140             'tools.request_logging.on': True,
 141             'tools.gzip.on': True,
 142             'tools.gzip.mime_types': [
 143                 # text/html and text/plain are the default types to compress
 144                 'text/html', 'text/plain',
 145                 # We also want JSON and JavaScript to be compressed
 146                 'application/json',
 147                 'application/*+json',
 148                 'application/javascript',
 149             ],
 150             'tools.json_in.on': True,
 151             'tools.json_in.force': True,
 152             'tools.plugin_hooks_filter_request.on': True,
 153         }
 154
 155         if use_ssl:
 156             # SSL initialization
 157             cert = self.get_localized_store("crt")  # type: ignore
 158             if cert is not None:
 159                 self.cert_tmp = tempfile.NamedTemporaryFile()
 160                 self.cert_tmp.write(cert.encode('utf-8'))
 161                 self.cert_tmp.flush()  # cert_tmp must not be gc'ed
 162                 cert_fname = self.cert_tmp.name
 163             else:
 164                 cert_fname = self.get_localized_module_option('crt_file')  # type: ignore
 165
 166             pkey = self.get_localized_store("key")  # type: ignore
 167             if pkey is not None:
 168                 self.pkey_tmp = tempfile.NamedTemporaryFile()
 169                 self.pkey_tmp.write(pkey.encode('utf-8'))
 170                 self.pkey_tmp.flush()  # pkey_tmp must not be gc'ed
 171                 pkey_fname = self.pkey_tmp.name
 172             else:
 173                 pkey_fname = self.get_localized_module_option('key_file')  # type: ignore
 174
 175             verify_tls_files(cert_fname, pkey_fname)
 176
 177             # Create custom SSL context to disable TLS 1.0 and 1.1.
 178             context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
 179             context.load_cert_chain(cert_fname, pkey_fname)
 180             if sys.version_info >= (3, 7):
 181                 context.minimum_version = ssl.TLSVersion.TLSv1_2
 182             else:
 183                 context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1
 184
 185             config['server.ssl_module'] = 'builtin'
 186             config['server.ssl_certificate'] = cert_fname
 187             config['server.ssl_private_key'] = pkey_fname
 188             config['server.ssl_context'] = context
 189
 190         self.update_cherrypy_config(config)
 191
 192         self._url_prefix = prepare_url_prefix(self.get_module_option(  # type: ignore
 193             'url_prefix', default=''))
 194
 195         if server_addr in ['::', '0.0.0.0']:
 196             server_addr = self.get_mgr_ip()  # type: ignore
 197         base_url = build_url(
 198             scheme='https' if use_ssl else 'http',
 199             host=server_addr,
 200             port=server_port,
 201         )
 202         uri = f'{base_url}{self.url_prefix}/'
 203         return uri
 204
 205     def await_configuration(self):
 206         """
 207         Block until configuration is ready (i.e. all needed keys are set)
 208         or self._stopping is set.
 209
 210         :returns URI of configured webserver
 211         """
 212         while not self._stopping.is_set():
 213             try:
 214                 uri = self._configure()
 215             except ServerConfigException as e:
 216                 self.log.info(  # type: ignore
 217                     "Config not ready to serve, waiting: {0}".format(e)
 218                 )
 219                 # Poll until a non-errored config is present
 220                 self._stopping.wait(5)
 221             else:
 222                 self.log.info("Configured CherryPy, starting engine...")  # type: ignore
 223                 return uri
 224
 225     def configure_cors(self):
 226         """
 227         Allow CORS requests if the cross_origin_url option is set.
 228         """
 229         cross_origin_url = mgr.get_localized_module_option('cross_origin_url', '')
 230         if cross_origin_url:
 231             cherrypy.tools.CORS = cherrypy.Tool('before_handler', self.cors_tool)
 232             config = {
 233                 'tools.CORS.on': True,
 234             }
 235             self.update_cherrypy_config(config)
 236
 237     def cors_tool(self):
 238         '''
 239         Handle both simple and complex CORS requests
 240
 241         Add CORS headers to each response. If the request is a CORS preflight
 242         request swap out the default handler with a simple, single-purpose handler
 243         that verifies the request and provides a valid CORS response.
 244         '''
 245         req_head = cherrypy.request.headers
 246         resp_head = cherrypy.response.headers
 247
 248         # Always set response headers necessary for 'simple' CORS.
 249         req_header_cross_origin_url = req_head.get('Access-Control-Allow-Origin')
 250         cross_origin_urls = mgr.get_localized_module_option('cross_origin_url', '')
 251         cross_origin_url_list = [url.strip() for url in cross_origin_urls.split(',')]
 252         if req_header_cross_origin_url in cross_origin_url_list:
 253             resp_head['Access-Control-Allow-Origin'] = req_header_cross_origin_url
 254         resp_head['Access-Control-Expose-Headers'] = 'GET, POST'
 255         resp_head['Access-Control-Allow-Credentials'] = 'true'
 256
 257         # Non-simple CORS preflight request; short-circuit the normal handler.
 258         if cherrypy.request.method == 'OPTIONS':
 259             req_header_origin_url = req_head.get('Origin')
 260             if req_header_origin_url in cross_origin_url_list:
 261                 resp_head['Access-Control-Allow-Origin'] = req_header_origin_url
 262             ac_method = req_head.get('Access-Control-Request-Method', None)
 263
 264             allowed_methods = ['GET', 'POST']
 265             allowed_headers = [
 266                 'Content-Type',
 267                 'Authorization',
 268                 'Accept',
 269                 'Access-Control-Allow-Origin'
 270             ]
 271
 272             if ac_method and ac_method in allowed_methods:
 273                 resp_head['Access-Control-Allow-Methods'] = ', '.join(allowed_methods)
 274                 resp_head['Access-Control-Allow-Headers'] = ', '.join(allowed_headers)
 275
 276                 resp_head['Connection'] = 'keep-alive'
 277                 resp_head['Access-Control-Max-Age'] = '3600'
 278
 279             # CORS requests should short-circuit the other tools.
 280             cherrypy.response.body = ''.encode('utf8')
 281             cherrypy.response.status = 200
 282             cherrypy.serving.request.handler = None
 283
 284             # Needed to avoid the auth_tool check.
 285             if cherrypy.request.config.get('tools.sessions.on', False):
 286                 cherrypy.session['token'] = True
 287             return True
 288
 289
 290 if TYPE_CHECKING:
 291     SslConfigKey = Literal['crt', 'key']
 292
 293
 294 class Module(MgrModule, CherryPyConfig):
 295     """
 296     dashboard module entrypoint
 297     """
 298
 299     COMMANDS = [
 300         {
 301             'cmd': 'dashboard set-jwt-token-ttl '
 302                    'name=seconds,type=CephInt',
 303             'desc': 'Set the JWT token TTL in seconds',
 304             'perm': 'w'
 305         },
 306         {
 307             'cmd': 'dashboard get-jwt-token-ttl',
 308             'desc': 'Get the JWT token TTL in seconds',
 309             'perm': 'r'
 310         },
 311         {
 312             "cmd": "dashboard create-self-signed-cert",
 313             "desc": "Create self signed certificate",
 314             "perm": "w"
 315         },
 316         {
 317             "cmd": "dashboard grafana dashboards update",
 318             "desc": "Push dashboards to Grafana",
 319             "perm": "w",
 320         },
 321     ]
 322     COMMANDS.extend(options_command_list())
 323     COMMANDS.extend(SSO_COMMANDS)
 324     PLUGIN_MANAGER.hook.register_commands()
 325
 326     MODULE_OPTIONS = [
 327         Option(name='server_addr', type='str', default=get_default_addr()),
 328         Option(name='server_port', type='int', default=8080),
 329         Option(name='ssl_server_port', type='int', default=8443),
 330         Option(name='jwt_token_ttl', type='int', default=28800),
 331         Option(name='url_prefix', type='str', default=''),
 332         Option(name='key_file', type='str', default=''),
 333         Option(name='crt_file', type='str', default=''),
 334         Option(name='ssl', type='bool', default=True),
 335         Option(name='standby_behaviour', type='str', default='redirect',
 336                enum_allowed=['redirect', 'error']),
 337         Option(name='standby_error_status_code', type='int', default=500,
 338                min=400, max=599),
 339         Option(name='redirect_resolve_ip_addr', type='bool', default=False),
 340         Option(name='cross_origin_url', type='str', default=''),
 341     ]
 342     MODULE_OPTIONS.extend(options_schema_list())
 343     for options in PLUGIN_MANAGER.hook.get_options() or []:
 344         MODULE_OPTIONS.extend(options)
 345
 346     NOTIFY_TYPES = [NotifyType.clog]
 347
 348     __pool_stats = collections.defaultdict(lambda: collections.defaultdict(
 349         lambda: collections.deque(maxlen=10)))  # type: dict
 350
 351     def __init__(self, *args, **kwargs):
 352         super(Module, self).__init__(*args, **kwargs)
 353         CherryPyConfig.__init__(self)
 354
 355         mgr.init(self)
 356
 357         self._stopping = threading.Event()
 358         self.shutdown_event = threading.Event()
 359         self.ACCESS_CTRL_DB = None
 360         self.SSO_DB = None
 361         self.health_checks = {}
 362
 363     @classmethod
 364     def can_run(cls):
 365         if cherrypy is None:
 366             return False, "Missing dependency: cherrypy"
 367
 368         if not os.path.exists(cls.get_frontend_path()):
 369             return False, ("Frontend assets not found at '{}': incomplete build?"
 370                            .format(cls.get_frontend_path()))
 371
 372         return True, ""
 373
 374     @classmethod
 375     def get_frontend_path(cls):
 376         current_dir = os.path.dirname(os.path.abspath(__file__))
 377         path = os.path.join(current_dir, 'frontend/dist')
 378         if os.path.exists(path):
 379             return path
 380         else:
 381             path = os.path.join(current_dir,
 382                                 '../../../../build',
 383                                 'src/pybind/mgr/dashboard',
 384                                 'frontend/dist')
 385             return os.path.abspath(path)
 386
 387     def serve(self):
 388
 389         if 'COVERAGE_ENABLED' in os.environ:
 390             import coverage
 391             __cov = coverage.Coverage(config_file="{}/.coveragerc"
 392                                       .format(os.path.dirname(__file__)),
 393                                       data_suffix=True)
 394             __cov.start()
 395             cherrypy.engine.subscribe('after_request', __cov.save)
 396             cherrypy.engine.subscribe('stop', __cov.stop)
 397
 398         AuthManager.initialize()
 399         load_sso_db()
 400
 401         uri = self.await_configuration()
 402         if uri is None:
 403             # We were shut down while waiting
 404             return
 405
 406         # Publish the URI that others may use to access the service we're
 407         # about to start serving
 408         self.set_uri(uri)
 409
 410         mapper, parent_urls = Router.generate_routes(self.url_prefix)
 411
 412         config = {}
 413         for purl in parent_urls:
 414             config[purl] = {
 415                 'request.dispatch': mapper
 416             }
 417
 418         cherrypy.tree.mount(None, config=config)
 419
 420         PLUGIN_MANAGER.hook.setup()
 421
 422         cherrypy.engine.start()
 423         NotificationQueue.start_queue()
 424         TaskManager.init()
 425         logger.info('Engine started.')
 426         update_dashboards = str_to_bool(
 427             self.get_module_option('GRAFANA_UPDATE_DASHBOARDS', 'False'))
 428         if update_dashboards:
 429             logger.info('Starting Grafana dashboard task')
 430             TaskManager.run(
 431                 'grafana/dashboards/update',
 432                 {},
 433                 push_local_dashboards,
 434                 kwargs=dict(tries=10, sleep=60),
 435             )
 436         # wait for the shutdown event
 437         self.shutdown_event.wait()
 438         self.shutdown_event.clear()
 439         NotificationQueue.stop()
 440         cherrypy.engine.stop()
 441         logger.info('Engine stopped')
 442
 443     def shutdown(self):
 444         super(Module, self).shutdown()
 445         CherryPyConfig.shutdown(self)
 446         logger.info('Stopping engine...')
 447         self.shutdown_event.set()
 448
 449     def _set_ssl_item(self, item_label: str, item_key: 'SslConfigKey' = 'crt',
 450                       mgr_id: Optional[str] = None, inbuf: Optional[str] = None):
 451         if inbuf is None:
 452             return -errno.EINVAL, '', f'Please specify the {item_label} with "-i" option'
 453
 454         if mgr_id is not None:
 455             self.set_store(_get_localized_key(mgr_id, item_key), inbuf)
 456         else:
 457             self.set_store(item_key, inbuf)
 458         return 0, f'SSL {item_label} updated', ''
 459
 460     @CLIWriteCommand("dashboard set-ssl-certificate")
 461     def set_ssl_certificate(self, mgr_id: Optional[str] = None, inbuf: Optional[str] = None):
 462         return self._set_ssl_item('certificate', 'crt', mgr_id, inbuf)
 463
 464     @CLIWriteCommand("dashboard set-ssl-certificate-key")
 465     def set_ssl_certificate_key(self, mgr_id: Optional[str] = None, inbuf: Optional[str] = None):
 466         return self._set_ssl_item('certificate key', 'key', mgr_id, inbuf)
 467
 468     @CLIWriteCommand("dashboard create-self-signed-cert")
 469     def set_mgr_created_self_signed_cert(self):
 470         cert, pkey = create_self_signed_cert('IT', 'ceph-dashboard')
 471         result = HandleCommandResult(*self.set_ssl_certificate(inbuf=cert))
 472         if result.retval != 0:
 473             return result
 474
 475         result = HandleCommandResult(*self.set_ssl_certificate_key(inbuf=pkey))
 476         if result.retval != 0:
 477             return result
 478         return 0, 'Self-signed certificate created', ''
 479
 480     @CLIWriteCommand("dashboard set-rgw-credentials")
 481     def set_rgw_credentials(self):
 482         try:
 483             configure_rgw_credentials()
 484         except Exception as error:
 485             return -errno.EINVAL, '', str(error)
 486
 487         return 0, 'RGW credentials configured', ''
 488
 489     @CLIWriteCommand("dashboard set-login-banner")
 490     def set_login_banner(self, inbuf: str):
 491         '''
 492         Set the custom login banner read from -i <file>
 493         '''
 494         item_label = 'login banner file'
 495         if inbuf is None:
 496             return HandleCommandResult(
 497                 -errno.EINVAL,
 498                 stderr=f'Please specify the {item_label} with "-i" option'
 499             )
 500         mgr.set_store('custom_login_banner', inbuf)
 501         return HandleCommandResult(stdout=f'{item_label} added')
 502
 503     @CLIReadCommand("dashboard get-login-banner")
 504     def get_login_banner(self):
 505         '''
 506         Get the custom login banner text
 507         '''
 508         banner_text = mgr.get_store('custom_login_banner')
 509         if banner_text is None:
 510             return HandleCommandResult(stdout='No login banner set')
 511         else:
 512             return HandleCommandResult(stdout=banner_text)
 513
 514     @CLIWriteCommand("dashboard unset-login-banner")
 515     def unset_login_banner(self):
 516         '''
 517         Unset the custom login banner
 518         '''
 519         mgr.set_store('custom_login_banner', None)
 520         return HandleCommandResult(stdout='Login banner removed')
 521
 522     def handle_command(self, inbuf, cmd):
 523         # pylint: disable=too-many-return-statements
 524         res = handle_option_command(cmd, inbuf)
 525         if res[0] != -errno.ENOSYS:
 526             return res
 527         res = handle_sso_command(cmd)
 528         if res[0] != -errno.ENOSYS:
 529             return res
 530         if cmd['prefix'] == 'dashboard set-jwt-token-ttl':
 531             self.set_module_option('jwt_token_ttl', str(cmd['seconds']))
 532             return 0, 'JWT token TTL updated', ''
 533         if cmd['prefix'] == 'dashboard get-jwt-token-ttl':
 534             ttl = self.get_module_option('jwt_token_ttl', JwtManager.JWT_TOKEN_TTL)
 535             return 0, str(ttl), ''
 536         if cmd['prefix'] == 'dashboard grafana dashboards update':
 537             push_local_dashboards()
 538             return 0, 'Grafana dashboards updated', ''
 539
 540         return (-errno.EINVAL, '', 'Command not found \'{0}\''
 541                 .format(cmd['prefix']))
 542
 543     def notify(self, notify_type: NotifyType, notify_id):
 544         NotificationQueue.new_notification(str(notify_type), notify_id)
 545
 546     def get_updated_pool_stats(self):
 547         df = self.get('df')
 548         pool_stats = {p['id']: p['stats'] for p in df['pools']}
 549         now = time.time()
 550         for pool_id, stats in pool_stats.items():
 551             for stat_name, stat_val in stats.items():
 552                 self.__pool_stats[pool_id][stat_name].append((now, stat_val))
 553
 554         return self.__pool_stats
 555
 556     def config_notify(self):
 557         """
 558         This method is called whenever one of our config options is changed.
 559         """
 560         PLUGIN_MANAGER.hook.config_notify()
 561
 562     def refresh_health_checks(self):
 563         self.set_health_checks(self.health_checks)
 564
 565
 566 class StandbyModule(MgrStandbyModule, CherryPyConfig):
 567     def __init__(self, *args, **kwargs):
 568         super(StandbyModule, self).__init__(*args, **kwargs)
 569         CherryPyConfig.__init__(self)
 570         self.shutdown_event = threading.Event()
 571
 572         # We can set the global mgr instance to ourselves even though
 573         # we're just a standby, because it's enough for logging.
 574         mgr.init(self)
 575
 576     def serve(self):
 577         uri = self.await_configuration()
 578         if uri is None:
 579             # We were shut down while waiting
 580             return
 581
 582         module = self
 583
 584         class Root(object):
 585             @cherrypy.expose
 586             def default(self, *args, **kwargs):
 587                 if module.get_module_option('standby_behaviour', 'redirect') == 'redirect':
 588                     active_uri = module.get_active_uri()
 589
 590                     if cherrypy.request.path_info.startswith('/api/prometheus_receiver'):
 591                         module.log.debug("Suppressed redirecting alert to active '%s'",
 592                                          active_uri)
 593                         cherrypy.response.status = 204
 594                         return None
 595
 596                     if active_uri:
 597                         if module.get_module_option('redirect_resolve_ip_addr'):
 598                             p_result = urlparse(active_uri)
 599                             hostname = str(p_result.hostname)
 600                             fqdn_netloc = p_result.netloc.replace(
 601                                 hostname, socket.getfqdn(hostname))
 602                             active_uri = p_result._replace(netloc=fqdn_netloc).geturl()
 603
 604                         module.log.info("Redirecting to active '%s'", active_uri)
 605                         raise cherrypy.HTTPRedirect(active_uri)
 606                     else:
 607                         template = """
 608                     <html>
 609                         <!-- Note: this is only displayed when the standby
 610                              does not know an active URI to redirect to, otherwise
 611                              a simple redirect is returned instead -->
 612                         <head>
 613                             <title>Ceph</title>
 614                             <meta http-equiv="refresh" content="{delay}">
 615                         </head>
 616                         <body>
 617                             No active ceph-mgr instance is currently running
 618                             the dashboard. A failover may be in progress.
 619                             Retrying in {delay} seconds...
 620                         </body>
 621                     </html>
 622                         """
 623                         return template.format(delay=5)
 624                 else:
 625                     status = module.get_module_option('standby_error_status_code', 500)
 626                     raise cherrypy.HTTPError(status, message="Keep on looking")
 627
 628         cherrypy.tree.mount(Root(), "{}/".format(self.url_prefix), {})
 629         self.log.info("Starting engine...")
 630         cherrypy.engine.start()
 631         self.log.info("Engine started...")
 632         # Wait for shutdown event
 633         self.shutdown_event.wait()
 634         self.shutdown_event.clear()
 635         cherrypy.engine.stop()
 636         self.log.info("Engine stopped.")
 637
 638     def shutdown(self):
 639         CherryPyConfig.shutdown(self)
 640
 641         self.log.info("Stopping engine...")
 642         self.shutdown_event.set()
 643         self.log.info("Stopped engine...")