ceph/src/pybind/mgr/dashboard/module.py

   1 # -*- coding: utf-8 -*-
   2 """
   3 ceph dashboard mgr plugin (based on CherryPy)
   4 """
   5 import collections
   6 import errno
   7 import logging
   8 import os
   9 import socket
  10 import ssl
  11 import sys
  12 import tempfile
  13 import threading
  14 import time
  15 from typing import TYPE_CHECKING, Optional
  16 from urllib.parse import urlparse
  17
  18 if TYPE_CHECKING:
  19     if sys.version_info >= (3, 8):
  20         from typing import Literal
  21     else:
  22         from typing_extensions import Literal
  23
  24 from mgr_module import CLIReadCommand, CLIWriteCommand, HandleCommandResult, \
  25     MgrModule, MgrStandbyModule, NotifyType, Option, _get_localized_key
  26 from mgr_util import ServerConfigException, build_url, \
  27     create_self_signed_cert, get_default_addr, verify_tls_files
  28
  29 from . import mgr
  30 from .controllers import Router, json_error_page
  31 from .grafana import push_local_dashboards
  32 from .services.auth import AuthManager, AuthManagerTool, JwtManager
  33 from .services.exception import dashboard_exception_handler
  34 from .services.rgw_client import configure_rgw_credentials
  35 from .services.sso import SSO_COMMANDS, handle_sso_command
  36 from .settings import Settings, handle_option_command, options_command_list, options_schema_list
  37 from .tools import NotificationQueue, RequestLoggingTool, TaskManager, \
  38     prepare_url_prefix, str_to_bool
  39
  40 try:
  41     import cherrypy
  42     from cherrypy._cptools import HandlerWrapperTool
  43 except ImportError:
  44     # To be picked up and reported by .can_run()
  45     cherrypy = None
  46
  47 from .services.sso import load_sso_db
  48
  49 if cherrypy is not None:
  50     from .cherrypy_backports import patch_cherrypy
  51     patch_cherrypy(cherrypy.__version__)
  52
  53 # pylint: disable=wrong-import-position
  54 from .plugins import PLUGIN_MANAGER, debug, feature_toggles, motd  # isort:skip # noqa E501 # pylint: disable=unused-import
  55
  56 PLUGIN_MANAGER.hook.init()
  57
  58
  59 # cherrypy likes to sys.exit on error.  don't let it take us down too!
  60 # pylint: disable=W0613
  61 def os_exit_noop(*args):
  62     pass
  63
  64
  65 # pylint: disable=W0212
  66 os._exit = os_exit_noop
  67
  68
  69 logger = logging.getLogger(__name__)
  70
  71
  72 class CherryPyConfig(object):
  73     """
  74     Class for common server configuration done by both active and
  75     standby module, especially setting up SSL.
  76     """
  77
  78     def __init__(self):
  79         self._stopping = threading.Event()
  80         self._url_prefix = ""
  81
  82         self.cert_tmp = None
  83         self.pkey_tmp = None
  84
  85     def shutdown(self):
  86         self._stopping.set()
  87
  88     @property
  89     def url_prefix(self):
  90         return self._url_prefix
  91
  92     @staticmethod
  93     def update_cherrypy_config(config):
  94         PLUGIN_MANAGER.hook.configure_cherrypy(config=config)
  95         cherrypy.config.update(config)
  96
  97     # pylint: disable=too-many-branches
  98     def _configure(self):
  99         """
 100         Configure CherryPy and initialize self.url_prefix
 101
 102         :returns our URI
 103         """
 104         server_addr = self.get_localized_module_option(  # type: ignore
 105             'server_addr', get_default_addr())
 106         use_ssl = self.get_localized_module_option('ssl', True)  # type: ignore
 107         if not use_ssl:
 108             server_port = self.get_localized_module_option('server_port', 8080)  # type: ignore
 109         else:
 110             server_port = self.get_localized_module_option('ssl_server_port', 8443)  # type: ignore
 111
 112         if server_addr is None:
 113             raise ServerConfigException(
 114                 'no server_addr configured; '
 115                 'try "ceph config set mgr mgr/{}/{}/server_addr <ip>"'
 116                 .format(self.module_name, self.get_mgr_id()))  # type: ignore
 117         self.log.info('server: ssl=%s host=%s port=%d', 'yes' if use_ssl else 'no',  # type: ignore
 118                       server_addr, server_port)
 119
 120         # Initialize custom handlers.
 121         cherrypy.tools.authenticate = AuthManagerTool()
 122         self.configure_cors()
 123         cherrypy.tools.plugin_hooks_filter_request = cherrypy.Tool(
 124             'before_handler',
 125             lambda: PLUGIN_MANAGER.hook.filter_request_before_handler(request=cherrypy.request),
 126             priority=1)
 127         cherrypy.tools.request_logging = RequestLoggingTool()
 128         cherrypy.tools.dashboard_exception_handler = HandlerWrapperTool(dashboard_exception_handler,
 129                                                                         priority=31)
 130
 131         cherrypy.log.access_log.propagate = False
 132         cherrypy.log.error_log.propagate = False
 133
 134         # Apply the 'global' CherryPy configuration.
 135         config = {
 136             'engine.autoreload.on': False,
 137             'server.socket_host': server_addr,
 138             'server.socket_port': int(server_port),
 139             'error_page.default': json_error_page,
 140             'tools.request_logging.on': True,
 141             'tools.gzip.on': True,
 142             'tools.gzip.mime_types': [
 143                 # text/html and text/plain are the default types to compress
 144                 'text/html', 'text/plain',
 145                 # We also want JSON and JavaScript to be compressed
 146                 'application/json',
 147                 'application/*+json',
 148                 'application/javascript',
 149             ],
 150             'tools.json_in.on': True,
 151             'tools.json_in.force': True,
 152             'tools.plugin_hooks_filter_request.on': True,
 153         }
 154
 155         if use_ssl:
 156             # SSL initialization
 157             cert = self.get_localized_store("crt")  # type: ignore
 158             if cert is not None:
 159                 self.cert_tmp = tempfile.NamedTemporaryFile()
 160                 self.cert_tmp.write(cert.encode('utf-8'))
 161                 self.cert_tmp.flush()  # cert_tmp must not be gc'ed
 162                 cert_fname = self.cert_tmp.name
 163             else:
 164                 cert_fname = self.get_localized_module_option('crt_file')  # type: ignore
 165
 166             pkey = self.get_localized_store("key")  # type: ignore
 167             if pkey is not None:
 168                 self.pkey_tmp = tempfile.NamedTemporaryFile()
 169                 self.pkey_tmp.write(pkey.encode('utf-8'))
 170                 self.pkey_tmp.flush()  # pkey_tmp must not be gc'ed
 171                 pkey_fname = self.pkey_tmp.name
 172             else:
 173                 pkey_fname = self.get_localized_module_option('key_file')  # type: ignore
 174
 175             verify_tls_files(cert_fname, pkey_fname)
 176
 177             # Create custom SSL context to disable TLS 1.0 and 1.1.
 178             context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
 179             context.load_cert_chain(cert_fname, pkey_fname)
 180             if sys.version_info >= (3, 7):
 181                 if Settings.UNSAFE_TLS_v1_2:
 182                     context.minimum_version = ssl.TLSVersion.TLSv1_2
 183                 else:
 184                     context.minimum_version = ssl.TLSVersion.TLSv1_3
 185             else:
 186                 if Settings.UNSAFE_TLS_v1_2:
 187                     context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1
 188                 else:
 189                     context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 | ssl.OP_NO_TLSv1_2
 190
 191             config['server.ssl_module'] = 'builtin'
 192             config['server.ssl_certificate'] = cert_fname
 193             config['server.ssl_private_key'] = pkey_fname
 194             config['server.ssl_context'] = context
 195
 196         self.update_cherrypy_config(config)
 197
 198         self._url_prefix = prepare_url_prefix(self.get_module_option(  # type: ignore
 199             'url_prefix', default=''))
 200
 201         if server_addr in ['::', '0.0.0.0']:
 202             server_addr = self.get_mgr_ip()  # type: ignore
 203         base_url = build_url(
 204             scheme='https' if use_ssl else 'http',
 205             host=server_addr,
 206             port=server_port,
 207         )
 208         uri = f'{base_url}{self.url_prefix}/'
 209         return uri
 210
 211     def await_configuration(self):
 212         """
 213         Block until configuration is ready (i.e. all needed keys are set)
 214         or self._stopping is set.
 215
 216         :returns URI of configured webserver
 217         """
 218         while not self._stopping.is_set():
 219             try:
 220                 uri = self._configure()
 221             except ServerConfigException as e:
 222                 self.log.info(  # type: ignore
 223                     "Config not ready to serve, waiting: {0}".format(e)
 224                 )
 225                 # Poll until a non-errored config is present
 226                 self._stopping.wait(5)
 227             else:
 228                 self.log.info("Configured CherryPy, starting engine...")  # type: ignore
 229                 return uri
 230
 231     def configure_cors(self):
 232         """
 233         Allow CORS requests if the cross_origin_url option is set.
 234         """
 235         cross_origin_url = mgr.get_localized_module_option('cross_origin_url', '')
 236         if cross_origin_url:
 237             cherrypy.tools.CORS = cherrypy.Tool('before_handler', self.cors_tool)
 238             config = {
 239                 'tools.CORS.on': True,
 240             }
 241             self.update_cherrypy_config(config)
 242
 243     def cors_tool(self):
 244         '''
 245         Handle both simple and complex CORS requests
 246
 247         Add CORS headers to each response. If the request is a CORS preflight
 248         request swap out the default handler with a simple, single-purpose handler
 249         that verifies the request and provides a valid CORS response.
 250         '''
 251         req_head = cherrypy.request.headers
 252         resp_head = cherrypy.response.headers
 253
 254         # Always set response headers necessary for 'simple' CORS.
 255         req_header_cross_origin_url = req_head.get('Access-Control-Allow-Origin')
 256         cross_origin_urls = mgr.get_localized_module_option('cross_origin_url', '')
 257         cross_origin_url_list = [url.strip() for url in cross_origin_urls.split(',')]
 258         if req_header_cross_origin_url in cross_origin_url_list:
 259             resp_head['Access-Control-Allow-Origin'] = req_header_cross_origin_url
 260         resp_head['Access-Control-Expose-Headers'] = 'GET, POST'
 261         resp_head['Access-Control-Allow-Credentials'] = 'true'
 262
 263         # Non-simple CORS preflight request; short-circuit the normal handler.
 264         if cherrypy.request.method == 'OPTIONS':
 265             req_header_origin_url = req_head.get('Origin')
 266             if req_header_origin_url in cross_origin_url_list:
 267                 resp_head['Access-Control-Allow-Origin'] = req_header_origin_url
 268             ac_method = req_head.get('Access-Control-Request-Method', None)
 269
 270             allowed_methods = ['GET', 'POST', 'PUT']
 271             allowed_headers = [
 272                 'Content-Type',
 273                 'Authorization',
 274                 'Accept',
 275                 'Access-Control-Allow-Origin'
 276             ]
 277
 278             if ac_method and ac_method in allowed_methods:
 279                 resp_head['Access-Control-Allow-Methods'] = ', '.join(allowed_methods)
 280                 resp_head['Access-Control-Allow-Headers'] = ', '.join(allowed_headers)
 281
 282                 resp_head['Connection'] = 'keep-alive'
 283                 resp_head['Access-Control-Max-Age'] = '3600'
 284
 285             # CORS requests should short-circuit the other tools.
 286             cherrypy.response.body = ''.encode('utf8')
 287             cherrypy.response.status = 200
 288             cherrypy.serving.request.handler = None
 289
 290             # Needed to avoid the auth_tool check.
 291             if cherrypy.request.config.get('tools.sessions.on', False):
 292                 cherrypy.session['token'] = True
 293             return True
 294
 295
 296 if TYPE_CHECKING:
 297     SslConfigKey = Literal['crt', 'key']
 298
 299
 300 class Module(MgrModule, CherryPyConfig):
 301     """
 302     dashboard module entrypoint
 303     """
 304
 305     COMMANDS = [
 306         {
 307             'cmd': 'dashboard set-jwt-token-ttl '
 308                    'name=seconds,type=CephInt',
 309             'desc': 'Set the JWT token TTL in seconds',
 310             'perm': 'w'
 311         },
 312         {
 313             'cmd': 'dashboard get-jwt-token-ttl',
 314             'desc': 'Get the JWT token TTL in seconds',
 315             'perm': 'r'
 316         },
 317         {
 318             "cmd": "dashboard create-self-signed-cert",
 319             "desc": "Create self signed certificate",
 320             "perm": "w"
 321         },
 322         {
 323             "cmd": "dashboard grafana dashboards update",
 324             "desc": "Push dashboards to Grafana",
 325             "perm": "w",
 326         },
 327     ]
 328     COMMANDS.extend(options_command_list())
 329     COMMANDS.extend(SSO_COMMANDS)
 330     PLUGIN_MANAGER.hook.register_commands()
 331
 332     MODULE_OPTIONS = [
 333         Option(name='server_addr', type='str', default=get_default_addr()),
 334         Option(name='server_port', type='int', default=8080),
 335         Option(name='ssl_server_port', type='int', default=8443),
 336         Option(name='jwt_token_ttl', type='int', default=28800),
 337         Option(name='url_prefix', type='str', default=''),
 338         Option(name='key_file', type='str', default=''),
 339         Option(name='crt_file', type='str', default=''),
 340         Option(name='ssl', type='bool', default=True),
 341         Option(name='standby_behaviour', type='str', default='redirect',
 342                enum_allowed=['redirect', 'error']),
 343         Option(name='standby_error_status_code', type='int', default=500,
 344                min=400, max=599),
 345         Option(name='redirect_resolve_ip_addr', type='bool', default=False),
 346         Option(name='cross_origin_url', type='str', default=''),
 347     ]
 348     MODULE_OPTIONS.extend(options_schema_list())
 349     for options in PLUGIN_MANAGER.hook.get_options() or []:
 350         MODULE_OPTIONS.extend(options)
 351
 352     NOTIFY_TYPES = [NotifyType.clog]
 353
 354     __pool_stats = collections.defaultdict(lambda: collections.defaultdict(
 355         lambda: collections.deque(maxlen=10)))  # type: dict
 356
 357     def __init__(self, *args, **kwargs):
 358         super(Module, self).__init__(*args, **kwargs)
 359         CherryPyConfig.__init__(self)
 360
 361         mgr.init(self)
 362
 363         self._stopping = threading.Event()
 364         self.shutdown_event = threading.Event()
 365         self.ACCESS_CTRL_DB = None
 366         self.SSO_DB = None
 367         self.health_checks = {}
 368
 369     @classmethod
 370     def can_run(cls):
 371         if cherrypy is None:
 372             return False, "Missing dependency: cherrypy"
 373
 374         if not os.path.exists(cls.get_frontend_path()):
 375             return False, ("Frontend assets not found at '{}': incomplete build?"
 376                            .format(cls.get_frontend_path()))
 377
 378         return True, ""
 379
 380     @classmethod
 381     def get_frontend_path(cls):
 382         current_dir = os.path.dirname(os.path.abspath(__file__))
 383         path = os.path.join(current_dir, 'frontend/dist')
 384         if os.path.exists(path):
 385             return path
 386         else:
 387             path = os.path.join(current_dir,
 388                                 '../../../../build',
 389                                 'src/pybind/mgr/dashboard',
 390                                 'frontend/dist')
 391             return os.path.abspath(path)
 392
 393     def serve(self):
 394
 395         if 'COVERAGE_ENABLED' in os.environ:
 396             import coverage
 397             __cov = coverage.Coverage(config_file="{}/.coveragerc"
 398                                       .format(os.path.dirname(__file__)),
 399                                       data_suffix=True)
 400             __cov.start()
 401             cherrypy.engine.subscribe('after_request', __cov.save)
 402             cherrypy.engine.subscribe('stop', __cov.stop)
 403
 404         AuthManager.initialize()
 405         load_sso_db()
 406
 407         uri = self.await_configuration()
 408         if uri is None:
 409             # We were shut down while waiting
 410             return
 411
 412         # Publish the URI that others may use to access the service we're
 413         # about to start serving
 414         self.set_uri(uri)
 415
 416         mapper, parent_urls = Router.generate_routes(self.url_prefix)
 417
 418         config = {}
 419         for purl in parent_urls:
 420             config[purl] = {
 421                 'request.dispatch': mapper
 422             }
 423
 424         cherrypy.tree.mount(None, config=config)
 425
 426         PLUGIN_MANAGER.hook.setup()
 427
 428         cherrypy.engine.start()
 429         NotificationQueue.start_queue()
 430         TaskManager.init()
 431         logger.info('Engine started.')
 432         update_dashboards = str_to_bool(
 433             self.get_module_option('GRAFANA_UPDATE_DASHBOARDS', 'False'))
 434         if update_dashboards:
 435             logger.info('Starting Grafana dashboard task')
 436             TaskManager.run(
 437                 'grafana/dashboards/update',
 438                 {},
 439                 push_local_dashboards,
 440                 kwargs=dict(tries=10, sleep=60),
 441             )
 442         # wait for the shutdown event
 443         self.shutdown_event.wait()
 444         self.shutdown_event.clear()
 445         NotificationQueue.stop()
 446         cherrypy.engine.stop()
 447         logger.info('Engine stopped')
 448
 449     def shutdown(self):
 450         super(Module, self).shutdown()
 451         CherryPyConfig.shutdown(self)
 452         logger.info('Stopping engine...')
 453         self.shutdown_event.set()
 454
 455     def _set_ssl_item(self, item_label: str, item_key: 'SslConfigKey' = 'crt',
 456                       mgr_id: Optional[str] = None, inbuf: Optional[str] = None):
 457         if inbuf is None:
 458             return -errno.EINVAL, '', f'Please specify the {item_label} with "-i" option'
 459
 460         if mgr_id is not None:
 461             self.set_store(_get_localized_key(mgr_id, item_key), inbuf)
 462         else:
 463             self.set_store(item_key, inbuf)
 464         return 0, f'SSL {item_label} updated', ''
 465
 466     @CLIWriteCommand("dashboard set-ssl-certificate")
 467     def set_ssl_certificate(self, mgr_id: Optional[str] = None, inbuf: Optional[str] = None):
 468         return self._set_ssl_item('certificate', 'crt', mgr_id, inbuf)
 469
 470     @CLIWriteCommand("dashboard set-ssl-certificate-key")
 471     def set_ssl_certificate_key(self, mgr_id: Optional[str] = None, inbuf: Optional[str] = None):
 472         return self._set_ssl_item('certificate key', 'key', mgr_id, inbuf)
 473
 474     @CLIWriteCommand("dashboard create-self-signed-cert")
 475     def set_mgr_created_self_signed_cert(self):
 476         cert, pkey = create_self_signed_cert('IT', 'ceph-dashboard')
 477         result = HandleCommandResult(*self.set_ssl_certificate(inbuf=cert))
 478         if result.retval != 0:
 479             return result
 480
 481         result = HandleCommandResult(*self.set_ssl_certificate_key(inbuf=pkey))
 482         if result.retval != 0:
 483             return result
 484         return 0, 'Self-signed certificate created', ''
 485
 486     @CLIWriteCommand("dashboard set-rgw-credentials")
 487     def set_rgw_credentials(self):
 488         try:
 489             configure_rgw_credentials()
 490         except Exception as error:
 491             return -errno.EINVAL, '', str(error)
 492
 493         return 0, 'RGW credentials configured', ''
 494
 495     @CLIWriteCommand("dashboard set-login-banner")
 496     def set_login_banner(self, inbuf: str):
 497         '''
 498         Set the custom login banner read from -i <file>
 499         '''
 500         item_label = 'login banner file'
 501         if inbuf is None:
 502             return HandleCommandResult(
 503                 -errno.EINVAL,
 504                 stderr=f'Please specify the {item_label} with "-i" option'
 505             )
 506         mgr.set_store('custom_login_banner', inbuf)
 507         return HandleCommandResult(stdout=f'{item_label} added')
 508
 509     @CLIReadCommand("dashboard get-login-banner")
 510     def get_login_banner(self):
 511         '''
 512         Get the custom login banner text
 513         '''
 514         banner_text = mgr.get_store('custom_login_banner')
 515         if banner_text is None:
 516             return HandleCommandResult(stdout='No login banner set')
 517         else:
 518             return HandleCommandResult(stdout=banner_text)
 519
 520     @CLIWriteCommand("dashboard unset-login-banner")
 521     def unset_login_banner(self):
 522         '''
 523         Unset the custom login banner
 524         '''
 525         mgr.set_store('custom_login_banner', None)
 526         return HandleCommandResult(stdout='Login banner removed')
 527
 528     def handle_command(self, inbuf, cmd):
 529         # pylint: disable=too-many-return-statements
 530         res = handle_option_command(cmd, inbuf)
 531         if res[0] != -errno.ENOSYS:
 532             return res
 533         res = handle_sso_command(cmd)
 534         if res[0] != -errno.ENOSYS:
 535             return res
 536         if cmd['prefix'] == 'dashboard set-jwt-token-ttl':
 537             self.set_module_option('jwt_token_ttl', str(cmd['seconds']))
 538             return 0, 'JWT token TTL updated', ''
 539         if cmd['prefix'] == 'dashboard get-jwt-token-ttl':
 540             ttl = self.get_module_option('jwt_token_ttl', JwtManager.JWT_TOKEN_TTL)
 541             return 0, str(ttl), ''
 542         if cmd['prefix'] == 'dashboard grafana dashboards update':
 543             push_local_dashboards()
 544             return 0, 'Grafana dashboards updated', ''
 545
 546         return (-errno.EINVAL, '', 'Command not found \'{0}\''
 547                 .format(cmd['prefix']))
 548
 549     def notify(self, notify_type: NotifyType, notify_id):
 550         NotificationQueue.new_notification(str(notify_type), notify_id)
 551
 552     def get_updated_pool_stats(self):
 553         df = self.get('df')
 554         pool_stats = {p['id']: p['stats'] for p in df['pools']}
 555         now = time.time()
 556         for pool_id, stats in pool_stats.items():
 557             for stat_name, stat_val in stats.items():
 558                 self.__pool_stats[pool_id][stat_name].append((now, stat_val))
 559
 560         return self.__pool_stats
 561
 562     def config_notify(self):
 563         """
 564         This method is called whenever one of our config options is changed.
 565         """
 566         PLUGIN_MANAGER.hook.config_notify()
 567
 568     def refresh_health_checks(self):
 569         self.set_health_checks(self.health_checks)
 570
 571
 572 class StandbyModule(MgrStandbyModule, CherryPyConfig):
 573     def __init__(self, *args, **kwargs):
 574         super(StandbyModule, self).__init__(*args, **kwargs)
 575         CherryPyConfig.__init__(self)
 576         self.shutdown_event = threading.Event()
 577
 578         # We can set the global mgr instance to ourselves even though
 579         # we're just a standby, because it's enough for logging.
 580         mgr.init(self)
 581
 582     def serve(self):
 583         uri = self.await_configuration()
 584         if uri is None:
 585             # We were shut down while waiting
 586             return
 587
 588         module = self
 589
 590         class Root(object):
 591             @cherrypy.expose
 592             def default(self, *args, **kwargs):
 593                 if module.get_module_option('standby_behaviour', 'redirect') == 'redirect':
 594                     active_uri = module.get_active_uri()
 595
 596                     if cherrypy.request.path_info.startswith('/api/prometheus_receiver'):
 597                         module.log.debug("Suppressed redirecting alert to active '%s'",
 598                                          active_uri)
 599                         cherrypy.response.status = 204
 600                         return None
 601
 602                     if active_uri:
 603                         if module.get_module_option('redirect_resolve_ip_addr'):
 604                             p_result = urlparse(active_uri)
 605                             hostname = str(p_result.hostname)
 606                             fqdn_netloc = p_result.netloc.replace(
 607                                 hostname, socket.getfqdn(hostname))
 608                             active_uri = p_result._replace(netloc=fqdn_netloc).geturl()
 609
 610                         module.log.info("Redirecting to active '%s'", active_uri)
 611                         raise cherrypy.HTTPRedirect(active_uri)
 612                     else:
 613                         template = """
 614                     <html>
 615                         <!-- Note: this is only displayed when the standby
 616                              does not know an active URI to redirect to, otherwise
 617                              a simple redirect is returned instead -->
 618                         <head>
 619                             <title>Ceph</title>
 620                             <meta http-equiv="refresh" content="{delay}">
 621                         </head>
 622                         <body>
 623                             No active ceph-mgr instance is currently running
 624                             the dashboard. A failover may be in progress.
 625                             Retrying in {delay} seconds...
 626                         </body>
 627                     </html>
 628                         """
 629                         return template.format(delay=5)
 630                 else:
 631                     status = module.get_module_option('standby_error_status_code', 500)
 632                     raise cherrypy.HTTPError(status, message="Keep on looking")
 633
 634         cherrypy.tree.mount(Root(), "{}/".format(self.url_prefix), {})
 635         self.log.info("Starting engine...")
 636         cherrypy.engine.start()
 637         self.log.info("Engine started...")
 638         # Wait for shutdown event
 639         self.shutdown_event.wait()
 640         self.shutdown_event.clear()
 641         cherrypy.engine.stop()
 642         self.log.info("Engine stopped.")
 643
 644     def shutdown(self):
 645         CherryPyConfig.shutdown(self)
 646
 647         self.log.info("Stopping engine...")
 648         self.shutdown_event.set()
 649         self.log.info("Stopped engine...")