ceph/src/pybind/mgr/dashboard/tests/__init__.py

   1 # -*- coding: utf-8 -*-
   2 # pylint: disable=too-many-arguments
   3
   4 import contextlib
   5 import json
   6 import logging
   7 import threading
   8 import time
   9 from typing import Any, Dict, List, Optional
  10 from unittest import mock
  11 from unittest.mock import Mock
  12
  13 import cherrypy
  14 from cherrypy._cptools import HandlerWrapperTool
  15 from cherrypy.test import helper
  16 from mgr_module import HandleCommandResult
  17 from orchestrator import HostSpec, InventoryHost
  18 from pyfakefs import fake_filesystem
  19
  20 from .. import mgr
  21 from ..controllers import generate_controller_routes, json_error_page
  22 from ..controllers._version import APIVersion
  23 from ..module import Module
  24 from ..plugins import PLUGIN_MANAGER, debug, feature_toggles  # noqa
  25 from ..services.auth import AuthManagerTool
  26 from ..services.exception import dashboard_exception_handler
  27 from ..tools import RequestLoggingTool
  28
  29 PLUGIN_MANAGER.hook.init()
  30 PLUGIN_MANAGER.hook.register_commands()
  31
  32
  33 logger = logging.getLogger('tests')
  34
  35
  36 class ModuleTestClass(Module):
  37     """Dashboard module subclass for testing the module methods."""
  38
  39     def __init__(self) -> None:
  40         pass
  41
  42     def _unconfigure_logging(self) -> None:
  43         pass
  44
  45
  46 class CmdException(Exception):
  47     def __init__(self, retcode, message):
  48         super(CmdException, self).__init__(message)
  49         self.retcode = retcode
  50
  51
  52 class KVStoreMockMixin(object):
  53     CONFIG_KEY_DICT = {}
  54
  55     @classmethod
  56     def mock_set_module_option(cls, attr, val):
  57         cls.CONFIG_KEY_DICT[attr] = val
  58
  59     @classmethod
  60     def mock_get_module_option(cls, attr, default=None):
  61         return cls.CONFIG_KEY_DICT.get(attr, default)
  62
  63     @classmethod
  64     def mock_kv_store(cls):
  65         cls.CONFIG_KEY_DICT.clear()
  66         mgr.set_module_option.side_effect = cls.mock_set_module_option
  67         mgr.get_module_option.side_effect = cls.mock_get_module_option
  68         # kludge below
  69         mgr.set_store.side_effect = cls.mock_set_module_option
  70         mgr.get_store.side_effect = cls.mock_get_module_option
  71
  72     @classmethod
  73     def get_key(cls, key):
  74         return cls.CONFIG_KEY_DICT.get(key, None)
  75
  76
  77 # pylint: disable=protected-access
  78 class CLICommandTestMixin(KVStoreMockMixin):
  79     _dashboard_module = ModuleTestClass()
  80
  81     @classmethod
  82     def exec_cmd(cls, cmd, **kwargs):
  83         inbuf = kwargs['inbuf'] if 'inbuf' in kwargs else None
  84         cmd_dict = {'prefix': 'dashboard {}'.format(cmd)}
  85         cmd_dict.update(kwargs)
  86
  87         result = HandleCommandResult(*cls._dashboard_module._handle_command(inbuf, cmd_dict))
  88
  89         if result.retval < 0:
  90             raise CmdException(result.retval, result.stderr)
  91         try:
  92             return json.loads(result.stdout)
  93         except ValueError:
  94             return result.stdout
  95
  96
  97 class FakeFsMixin(object):
  98     fs = fake_filesystem.FakeFilesystem()
  99     f_open = fake_filesystem.FakeFileOpen(fs)
 100     f_os = fake_filesystem.FakeOsModule(fs)
 101     builtins_open = 'builtins.open'
 102
 103
 104 class ControllerTestCase(helper.CPWebCase):
 105     _endpoints_cache = {}
 106
 107     @classmethod
 108     def setup_controllers(cls, ctrl_classes, base_url='', cp_config: Dict[str, Any] = None):
 109         if not isinstance(ctrl_classes, list):
 110             ctrl_classes = [ctrl_classes]
 111         mapper = cherrypy.dispatch.RoutesDispatcher()
 112         endpoint_list = []
 113         for ctrl in ctrl_classes:
 114             ctrl._cp_config = {
 115                 'tools.dashboard_exception_handler.on': True,
 116                 'tools.authenticate.on': False
 117             }
 118             if cp_config:
 119                 ctrl._cp_config.update(cp_config)
 120             inst = ctrl()
 121
 122             # We need to cache the controller endpoints because
 123             # BaseController#endpoints method is not idempontent
 124             # and a controller might be needed by more than one
 125             # unit test.
 126             if ctrl not in cls._endpoints_cache:
 127                 ctrl_endpoints = ctrl.endpoints()
 128                 cls._endpoints_cache[ctrl] = ctrl_endpoints
 129
 130             ctrl_endpoints = cls._endpoints_cache[ctrl]
 131             for endpoint in ctrl_endpoints:
 132                 endpoint.inst = inst
 133                 endpoint_list.append(endpoint)
 134         endpoint_list = sorted(endpoint_list, key=lambda e: e.url)
 135         for endpoint in endpoint_list:
 136             generate_controller_routes(endpoint, mapper, base_url)
 137         if base_url == '':
 138             base_url = '/'
 139         cherrypy.tree.mount(None, config={
 140             base_url: {'request.dispatch': mapper}})
 141
 142     _request_logging = False
 143
 144     @classmethod
 145     def setUpClass(cls):
 146         super().setUpClass()
 147         cherrypy.tools.authenticate = AuthManagerTool()
 148         cherrypy.tools.dashboard_exception_handler = HandlerWrapperTool(dashboard_exception_handler,
 149                                                                         priority=31)
 150         cherrypy.config.update({
 151             'error_page.default': json_error_page,
 152             'tools.json_in.on': True,
 153             'tools.json_in.force': False
 154         })
 155         PLUGIN_MANAGER.hook.configure_cherrypy(config=cherrypy.config)
 156
 157         if cls._request_logging:
 158             cherrypy.tools.request_logging = RequestLoggingTool()
 159             cherrypy.config.update({'tools.request_logging.on': True})
 160
 161     @classmethod
 162     def tearDownClass(cls):
 163         if cls._request_logging:
 164             cherrypy.config.update({'tools.request_logging.on': False})
 165
 166     def _request(self, url, method, data=None, headers=None, version=APIVersion.DEFAULT):
 167         if not data:
 168             b = None
 169             if version:
 170                 h = [('Accept', version.to_mime_type()),
 171                      ('Content-Length', '0')]
 172             else:
 173                 h = None
 174         else:
 175             b = json.dumps(data)
 176             if version is not None:
 177                 h = [('Accept', version.to_mime_type()),
 178                      ('Content-Type', 'application/json'),
 179                      ('Content-Length', str(len(b)))]
 180
 181             else:
 182                 h = [('Content-Type', 'application/json'),
 183                      ('Content-Length', str(len(b)))]
 184
 185         if headers:
 186             h = headers
 187         self.getPage(url, method=method, body=b, headers=h)
 188
 189     def _get(self, url, headers=None, version=APIVersion.DEFAULT):
 190         self._request(url, 'GET', headers=headers, version=version)
 191
 192     def _post(self, url, data=None, version=APIVersion.DEFAULT):
 193         self._request(url, 'POST', data, version=version)
 194
 195     def _delete(self, url, data=None, version=APIVersion.DEFAULT):
 196         self._request(url, 'DELETE', data, version=version)
 197
 198     def _put(self, url, data=None, version=APIVersion.DEFAULT):
 199         self._request(url, 'PUT', data, version=version)
 200
 201     def _task_request(self, method, url, data, timeout, version=APIVersion.DEFAULT):
 202         self._request(url, method, data, version=version)
 203         if self.status != '202 Accepted':
 204             logger.info("task finished immediately")
 205             return
 206
 207         res = self.json_body()
 208         self.assertIsInstance(res, dict)
 209         self.assertIn('name', res)
 210         self.assertIn('metadata', res)
 211
 212         task_name = res['name']
 213         task_metadata = res['metadata']
 214
 215         thread = Waiter(task_name, task_metadata, self, version)
 216         thread.start()
 217         status = thread.ev.wait(timeout)
 218         if not status:
 219             # timeout expired
 220             thread.abort = True
 221             thread.join()
 222             raise Exception("Waiting for task ({}, {}) to finish timed out"
 223                             .format(task_name, task_metadata))
 224         logger.info("task (%s, %s) finished", task_name, task_metadata)
 225         if thread.res_task['success']:
 226             self.body = json.dumps(thread.res_task['ret_value'])
 227             self._set_success_status(method)
 228         else:
 229             if 'status' in thread.res_task['exception']:
 230                 self.status = thread.res_task['exception']['status']
 231             else:
 232                 self.status = 500
 233             self.body = json.dumps(thread.res_task['exception'])
 234
 235     def _set_success_status(self, method):
 236         if method == 'POST':
 237             self.status = '201 Created'
 238         elif method == 'PUT':
 239             self.status = '200 OK'
 240         elif method == 'DELETE':
 241             self.status = '204 No Content'
 242
 243     def _task_post(self, url, data=None, timeout=60, version=APIVersion.DEFAULT):
 244         self._task_request('POST', url, data, timeout, version=version)
 245
 246     def _task_delete(self, url, timeout=60, version=APIVersion.DEFAULT):
 247         self._task_request('DELETE', url, None, timeout, version=version)
 248
 249     def _task_put(self, url, data=None, timeout=60, version=APIVersion.DEFAULT):
 250         self._task_request('PUT', url, data, timeout, version=version)
 251
 252     def json_body(self):
 253         body_str = self.body.decode('utf-8') if isinstance(self.body, bytes) else self.body
 254         return json.loads(body_str)
 255
 256     def assertJsonBody(self, data, msg=None):  # noqa: N802
 257         """Fail if value != self.body."""
 258         json_body = self.json_body()
 259         if data != json_body:
 260             if msg is None:
 261                 msg = 'expected body:\n%r\n\nactual body:\n%r' % (
 262                     data, json_body)
 263             self._handlewebError(msg)
 264
 265     def assertInJsonBody(self, data, msg=None):  # noqa: N802
 266         json_body = self.json_body()
 267         if data not in json_body:
 268             if msg is None:
 269                 msg = 'expected %r to be in %r' % (data, json_body)
 270             self._handlewebError(msg)
 271
 272
 273 class Stub:
 274     """Test class for returning predefined values"""
 275
 276     @classmethod
 277     def get_mgr_no_services(cls):
 278         mgr.get = Mock(return_value={})
 279
 280
 281 class RgwStub(Stub):
 282
 283     @classmethod
 284     def get_daemons(cls):
 285         mgr.get = Mock(return_value={'services': {'rgw': {'daemons': {
 286             '5297': {
 287                 'addr': '192.168.178.3:49774/1534999298',
 288                 'metadata': {
 289                     'frontend_config#0': 'beast port=8000',
 290                     'id': 'daemon1',
 291                     'realm_name': 'realm1',
 292                     'zonegroup_name': 'zonegroup1',
 293                     'zone_name': 'zone1'
 294                 }
 295             },
 296             '5398': {
 297                 'addr': '[2001:db8:85a3::8a2e:370:7334]:49774/1534999298',
 298                 'metadata': {
 299                     'frontend_config#0': 'civetweb port=8002',
 300                     'id': 'daemon2',
 301                     'realm_name': 'realm2',
 302                     'zonegroup_name': 'zonegroup2',
 303                     'zone_name': 'zone2'
 304                 }
 305             }
 306         }}}})
 307
 308     @classmethod
 309     def get_settings(cls):
 310         settings = {
 311             'RGW_API_ACCESS_KEY': 'fake-access-key',
 312             'RGW_API_SECRET_KEY': 'fake-secret-key',
 313         }
 314         mgr.get_module_option = Mock(side_effect=settings.get)
 315
 316
 317 # pylint: disable=protected-access
 318 class Waiter(threading.Thread):
 319     def __init__(self, task_name, task_metadata, tc, version):
 320         super(Waiter, self).__init__()
 321         self.task_name = task_name
 322         self.task_metadata = task_metadata
 323         self.ev = threading.Event()
 324         self.abort = False
 325         self.res_task = None
 326         self.tc = tc
 327         self.version = version
 328
 329     def run(self):
 330         running = True
 331         while running and not self.abort:
 332             logger.info("task (%s, %s) is still executing", self.task_name,
 333                         self.task_metadata)
 334             time.sleep(1)
 335             self.tc._get('/api/task?name={}'.format(self.task_name), version=self.version)
 336             res = self.tc.json_body()
 337             for task in res['finished_tasks']:
 338                 if task['metadata'] == self.task_metadata:
 339                     # task finished
 340                     running = False
 341                     self.res_task = task
 342                     self.ev.set()
 343
 344
 345 @contextlib.contextmanager
 346 def patch_orch(available: bool, missing_features: Optional[List[str]] = None,
 347                hosts: Optional[List[HostSpec]] = None,
 348                inventory: Optional[List[dict]] = None):
 349     with mock.patch('dashboard.controllers.orchestrator.OrchClient.instance') as instance:
 350         fake_client = mock.Mock()
 351         fake_client.available.return_value = available
 352         fake_client.get_missing_features.return_value = missing_features
 353
 354         if hosts is not None:
 355             fake_client.hosts.list.return_value = hosts
 356
 357         if inventory is not None:
 358             def _list_inventory(hosts=None, refresh=False):  # pylint: disable=unused-argument
 359                 inv_hosts = []
 360                 for inv_host in inventory:
 361                     if hosts is None or inv_host['name'] in hosts:
 362                         inv_hosts.append(InventoryHost.from_json(inv_host))
 363                 return inv_hosts
 364             fake_client.inventory.list.side_effect = _list_inventory
 365
 366         instance.return_value = fake_client
 367         yield fake_client