ceph/qa/tasks/mgr/dashboard/helper.py

   1 # -*- coding: utf-8 -*-
   2 # pylint: disable=W0212,too-many-return-statements
   3 from __future__ import absolute_import
   4
   5 import json
   6 import logging
   7 from collections import namedtuple
   8 import time
   9
  10 import requests
  11 import six
  12 from teuthology.exceptions import CommandFailedError
  13
  14 from ..mgr_test_case import MgrTestCase
  15
  16
  17 log = logging.getLogger(__name__)
  18
  19
  20 class DashboardTestCase(MgrTestCase):
  21     MGRS_REQUIRED = 2
  22     MDSS_REQUIRED = 1
  23     REQUIRE_FILESYSTEM = True
  24     CLIENTS_REQUIRED = 1
  25     CEPHFS = False
  26
  27     _session = None  # type: requests.sessions.Session
  28     _token = None
  29     _resp = None  # type: requests.models.Response
  30     _loggedin = False
  31     _base_uri = None
  32
  33     AUTO_AUTHENTICATE = True
  34
  35     AUTH_ROLES = ['administrator']
  36
  37     @classmethod
  38     def create_user(cls, username, password, roles):
  39         try:
  40             cls._ceph_cmd(['dashboard', 'ac-user-show', username])
  41             cls._ceph_cmd(['dashboard', 'ac-user-delete', username])
  42         except CommandFailedError as ex:
  43             if ex.exitstatus != 2:
  44                 raise ex
  45
  46         cls._ceph_cmd(['dashboard', 'ac-user-create', username, password])
  47
  48         set_roles_args = ['dashboard', 'ac-user-set-roles', username]
  49         for idx, role in enumerate(roles):
  50             if isinstance(role, str):
  51                 set_roles_args.append(role)
  52             else:
  53                 assert isinstance(role, dict)
  54                 rolename = 'test_role_{}'.format(idx)
  55                 try:
  56                     cls._ceph_cmd(['dashboard', 'ac-role-show', rolename])
  57                     cls._ceph_cmd(['dashboard', 'ac-role-delete', rolename])
  58                 except CommandFailedError as ex:
  59                     if ex.exitstatus != 2:
  60                         raise ex
  61                 cls._ceph_cmd(['dashboard', 'ac-role-create', rolename])
  62                 for mod, perms in role.items():
  63                     args = ['dashboard', 'ac-role-add-scope-perms', rolename, mod]
  64                     args.extend(perms)
  65                     cls._ceph_cmd(args)
  66                 set_roles_args.append(rolename)
  67         cls._ceph_cmd(set_roles_args)
  68
  69     @classmethod
  70     def login(cls, username, password):
  71         if cls._loggedin:
  72             cls.logout()
  73         cls._post('/api/auth', {'username': username, 'password': password})
  74         cls._token = cls.jsonBody()['token']
  75         cls._loggedin = True
  76
  77     @classmethod
  78     def logout(cls):
  79         if cls._loggedin:
  80             cls._post('/api/auth/logout')
  81             cls._token = None
  82             cls._loggedin = False
  83
  84     @classmethod
  85     def delete_user(cls, username, roles=None):
  86         if roles is None:
  87             roles = []
  88         cls._ceph_cmd(['dashboard', 'ac-user-delete', username])
  89         for idx, role in enumerate(roles):
  90             if isinstance(role, dict):
  91                 cls._ceph_cmd(['dashboard', 'ac-role-delete', 'test_role_{}'.format(idx)])
  92
  93     @classmethod
  94     def RunAs(cls, username, password, roles):
  95         def wrapper(func):
  96             def execute(self, *args, **kwargs):
  97                 self.create_user(username, password, roles)
  98                 self.login(username, password)
  99                 res = func(self, *args, **kwargs)
 100                 self.logout()
 101                 self.delete_user(username, roles)
 102                 return res
 103             return execute
 104         return wrapper
 105
 106     @classmethod
 107     def set_jwt_token(cls, token):
 108         cls._token = token
 109
 110     @classmethod
 111     def setUpClass(cls):
 112         super(DashboardTestCase, cls).setUpClass()
 113         cls._assign_ports("dashboard", "ssl_server_port")
 114         cls._load_module("dashboard")
 115         cls._base_uri = cls._get_uri("dashboard").rstrip('/')
 116
 117         if cls.CEPHFS:
 118             cls.mds_cluster.clear_firewall()
 119
 120             # To avoid any issues with e.g. unlink bugs, we destroy and recreate
 121             # the filesystem rather than just doing a rm -rf of files
 122             cls.mds_cluster.mds_stop()
 123             cls.mds_cluster.mds_fail()
 124             cls.mds_cluster.delete_all_filesystems()
 125             cls.fs = None  # is now invalid!
 126
 127             cls.fs = cls.mds_cluster.newfs(create=True)
 128             cls.fs.mds_restart()
 129
 130             # In case some test messed with auth caps, reset them
 131             # pylint: disable=not-an-iterable
 132             client_mount_ids = [m.client_id for m in cls.mounts]
 133             for client_id in client_mount_ids:
 134                 cls.mds_cluster.mon_manager.raw_cluster_cmd_result(
 135                     'auth', 'caps', "client.{0}".format(client_id),
 136                     'mds', 'allow',
 137                     'mon', 'allow r',
 138                     'osd', 'allow rw pool={0}'.format(cls.fs.get_data_pool_name()))
 139
 140             # wait for mds restart to complete...
 141             cls.fs.wait_for_daemons()
 142
 143         cls._token = None
 144         cls._session = requests.Session()
 145         cls._resp = None
 146
 147         cls.create_user('admin', 'admin', cls.AUTH_ROLES)
 148         if cls.AUTO_AUTHENTICATE:
 149             cls.login('admin', 'admin')
 150
 151     def setUp(self):
 152         if not self._loggedin and self.AUTO_AUTHENTICATE:
 153             self.login('admin', 'admin')
 154         self.wait_for_health_clear(20)
 155
 156     @classmethod
 157     def tearDownClass(cls):
 158         super(DashboardTestCase, cls).tearDownClass()
 159
 160     # pylint: disable=inconsistent-return-statements
 161     @classmethod
 162     def _request(cls, url, method, data=None, params=None):
 163         url = "{}{}".format(cls._base_uri, url)
 164         log.info("Request %s to %s", method, url)
 165         headers = {}
 166         if cls._token:
 167             headers['Authorization'] = "Bearer {}".format(cls._token)
 168
 169         if method == 'GET':
 170             cls._resp = cls._session.get(url, params=params, verify=False,
 171                                          headers=headers)
 172         elif method == 'POST':
 173             cls._resp = cls._session.post(url, json=data, params=params,
 174                                           verify=False, headers=headers)
 175         elif method == 'DELETE':
 176             cls._resp = cls._session.delete(url, json=data, params=params,
 177                                             verify=False, headers=headers)
 178         elif method == 'PUT':
 179             cls._resp = cls._session.put(url, json=data, params=params,
 180                                          verify=False, headers=headers)
 181         else:
 182             assert False
 183         try:
 184             if not cls._resp.ok:
 185                 # Output response for easier debugging.
 186                 log.error("Request response: %s", cls._resp.text)
 187             content_type = cls._resp.headers['content-type']
 188             if content_type == 'application/json' and cls._resp.text and cls._resp.text != "":
 189                 return cls._resp.json()
 190             return cls._resp.text
 191         except ValueError as ex:
 192             log.exception("Failed to decode response: %s", cls._resp.text)
 193             raise ex
 194
 195     @classmethod
 196     def _get(cls, url, params=None):
 197         return cls._request(url, 'GET', params=params)
 198
 199     @classmethod
 200     def _view_cache_get(cls, url, retries=5):
 201         retry = True
 202         while retry and retries > 0:
 203             retry = False
 204             res = cls._get(url)
 205             if isinstance(res, dict):
 206                 res = [res]
 207             for view in res:
 208                 assert 'value' in view
 209                 if not view['value']:
 210                     retry = True
 211             retries -= 1
 212         if retries == 0:
 213             raise Exception("{} view cache exceeded number of retries={}"
 214                             .format(url, retries))
 215         return res
 216
 217     @classmethod
 218     def _post(cls, url, data=None, params=None):
 219         cls._request(url, 'POST', data, params)
 220
 221     @classmethod
 222     def _delete(cls, url, data=None, params=None):
 223         cls._request(url, 'DELETE', data, params)
 224
 225     @classmethod
 226     def _put(cls, url, data=None, params=None):
 227         cls._request(url, 'PUT', data, params)
 228
 229     @classmethod
 230     def _assertEq(cls, v1, v2):
 231         if not v1 == v2:
 232             raise Exception("assertion failed: {} != {}".format(v1, v2))
 233
 234     @classmethod
 235     def _assertIn(cls, v1, v2):
 236         if v1 not in v2:
 237             raise Exception("assertion failed: {} not in {}".format(v1, v2))
 238
 239     @classmethod
 240     def _assertIsInst(cls, v1, v2):
 241         if not isinstance(v1, v2):
 242             raise Exception("assertion failed: {} not instance of {}".format(v1, v2))
 243
 244     # pylint: disable=too-many-arguments
 245     @classmethod
 246     def _task_request(cls, method, url, data, timeout):
 247         res = cls._request(url, method, data)
 248         cls._assertIn(cls._resp.status_code, [200, 201, 202, 204, 400, 403])
 249
 250         if cls._resp.status_code == 403:
 251             return None
 252
 253         if cls._resp.status_code != 202:
 254             log.info("task finished immediately")
 255             return res
 256
 257         cls._assertIn('name', res)
 258         cls._assertIn('metadata', res)
 259         task_name = res['name']
 260         task_metadata = res['metadata']
 261
 262         retries = int(timeout)
 263         res_task = None
 264         while retries > 0 and not res_task:
 265             retries -= 1
 266             log.info("task (%s, %s) is still executing", task_name,
 267                      task_metadata)
 268             time.sleep(1)
 269             _res = cls._get('/api/task?name={}'.format(task_name))
 270             cls._assertEq(cls._resp.status_code, 200)
 271             executing_tasks = [task for task in _res['executing_tasks'] if
 272                                task['metadata'] == task_metadata]
 273             finished_tasks = [task for task in _res['finished_tasks'] if
 274                                task['metadata'] == task_metadata]
 275             if not executing_tasks and finished_tasks:
 276                 res_task = finished_tasks[0]
 277
 278         if retries <= 0:
 279             raise Exception("Waiting for task ({}, {}) to finish timed out. {}"
 280                             .format(task_name, task_metadata, _res))
 281
 282         log.info("task (%s, %s) finished", task_name, task_metadata)
 283         if res_task['success']:
 284             if method == 'POST':
 285                 cls._resp.status_code = 201
 286             elif method == 'PUT':
 287                 cls._resp.status_code = 200
 288             elif method == 'DELETE':
 289                 cls._resp.status_code = 204
 290             return res_task['ret_value']
 291         else:
 292             if 'status' in res_task['exception']:
 293                 cls._resp.status_code = res_task['exception']['status']
 294             else:
 295                 cls._resp.status_code = 500
 296             return res_task['exception']
 297
 298     @classmethod
 299     def _task_post(cls, url, data=None, timeout=60):
 300         return cls._task_request('POST', url, data, timeout)
 301
 302     @classmethod
 303     def _task_delete(cls, url, timeout=60):
 304         return cls._task_request('DELETE', url, None, timeout)
 305
 306     @classmethod
 307     def _task_put(cls, url, data=None, timeout=60):
 308         return cls._task_request('PUT', url, data, timeout)
 309
 310     @classmethod
 311     def cookies(cls):
 312         return cls._resp.cookies
 313
 314     @classmethod
 315     def jsonBody(cls):
 316         return cls._resp.json()
 317
 318     @classmethod
 319     def reset_session(cls):
 320         cls._session = requests.Session()
 321
 322     def assertSubset(self, data, biggerData):
 323         for key, value in data.items():
 324             self.assertEqual(biggerData[key], value)
 325
 326     def assertJsonBody(self, data):
 327         body = self._resp.json()
 328         self.assertEqual(body, data)
 329
 330     def assertJsonSubset(self, data):
 331         self.assertSubset(data, self._resp.json())
 332
 333     def assertSchema(self, data, schema):
 334         try:
 335             return _validate_json(data, schema)
 336         except _ValError as e:
 337             self.assertEqual(data, str(e))
 338
 339     def assertSchemaBody(self, schema):
 340         self.assertSchema(self.jsonBody(), schema)
 341
 342     def assertBody(self, body):
 343         self.assertEqual(self._resp.text, body)
 344
 345     def assertStatus(self, status):
 346         if isinstance(status, list):
 347             self.assertIn(self._resp.status_code, status)
 348         else:
 349             self.assertEqual(self._resp.status_code, status)
 350
 351     def assertHeaders(self, headers):
 352         for name, value in headers.items():
 353             self.assertIn(name, self._resp.headers)
 354             self.assertEqual(self._resp.headers[name], value)
 355
 356     def assertError(self, code=None, component=None, detail=None):
 357         body = self._resp.json()
 358         if code:
 359             self.assertEqual(body['code'], code)
 360         if component:
 361             self.assertEqual(body['component'], component)
 362         if detail:
 363             self.assertEqual(body['detail'], detail)
 364
 365     @classmethod
 366     def _ceph_cmd(cls, cmd):
 367         res = cls.mgr_cluster.mon_manager.raw_cluster_cmd(*cmd)
 368         log.info("command result: %s", res)
 369         return res
 370
 371     def set_config_key(self, key, value):
 372         self._ceph_cmd(['config-key', 'set', key, value])
 373
 374     def get_config_key(self, key):
 375         return self._ceph_cmd(['config-key', 'get', key])
 376
 377     @classmethod
 378     def _rbd_cmd(cls, cmd):
 379         args = [
 380             'rbd'
 381         ]
 382         args.extend(cmd)
 383         cls.mgr_cluster.admin_remote.run(args=args)
 384
 385     @classmethod
 386     def _radosgw_admin_cmd(cls, cmd):
 387         args = [
 388             'radosgw-admin'
 389         ]
 390         args.extend(cmd)
 391         cls.mgr_cluster.admin_remote.run(args=args)
 392
 393     @classmethod
 394     def _rados_cmd(cls, cmd):
 395         args = ['rados']
 396         args.extend(cmd)
 397         cls.mgr_cluster.admin_remote.run(args=args)
 398
 399     @classmethod
 400     def mons(cls):
 401         out = cls.ceph_cluster.mon_manager.raw_cluster_cmd('mon_status')
 402         j = json.loads(out)
 403         return [mon['name'] for mon in j['monmap']['mons']]
 404
 405     @classmethod
 406     def find_object_in_list(cls, key, value, iterable):
 407         """
 408         Get the first occurrence of an object within a list with
 409         the specified key/value.
 410         :param key: The name of the key.
 411         :param value: The value to search for.
 412         :param iterable: The list to process.
 413         :return: Returns the found object or None.
 414         """
 415         for obj in iterable:
 416             if key in obj and obj[key] == value:
 417                 return obj
 418         return None
 419
 420
 421 class JLeaf(namedtuple('JLeaf', ['typ', 'none'])):
 422     def __new__(cls, typ, none=False):
 423         if typ == str:
 424             typ = six.string_types
 425         return super(JLeaf, cls).__new__(cls, typ, none)
 426
 427
 428 JList = namedtuple('JList', ['elem_typ'])
 429
 430 JTuple = namedtuple('JList', ['elem_typs'])
 431
 432
 433 class JObj(namedtuple('JObj', ['sub_elems', 'allow_unknown', 'none', 'unknown_schema'])):
 434     def __new__(cls, sub_elems, allow_unknown=False, none=False, unknown_schema=None):
 435         """
 436         :type sub_elems: dict[str, JAny | JLeaf | JList | JObj | type]
 437         :type allow_unknown: bool
 438         :type none: bool
 439         :type unknown_schema: int, str, JAny | JLeaf | JList | JObj
 440         :return:
 441         """
 442         return super(JObj, cls).__new__(cls, sub_elems, allow_unknown, none, unknown_schema)
 443
 444
 445 JAny = namedtuple('JAny', ['none'])
 446
 447
 448 class _ValError(Exception):
 449     def __init__(self, msg, path):
 450         path_str = ''.join('[{}]'.format(repr(p)) for p in path)
 451         super(_ValError, self).__init__('In `input{}`: {}'.format(path_str, msg))
 452
 453
 454 # pylint: disable=dangerous-default-value,inconsistent-return-statements
 455 def _validate_json(val, schema, path=[]):
 456     """
 457     >>> d = {'a': 1, 'b': 'x', 'c': range(10)}
 458     ... ds = JObj({'a': int, 'b': str, 'c': JList(int)})
 459     ... _validate_json(d, ds)
 460     True
 461     """
 462     if isinstance(schema, JAny):
 463         if not schema.none and val is None:
 464             raise _ValError('val is None', path)
 465         return True
 466     if isinstance(schema, JLeaf):
 467         if schema.none and val is None:
 468             return True
 469         if not isinstance(val, schema.typ):
 470             raise _ValError('val not of type {}'.format(schema.typ), path)
 471         return True
 472     if isinstance(schema, JList):
 473         if not isinstance(val, list):
 474             raise _ValError('val="{}" is not a list'.format(val), path)
 475         return all(_validate_json(e, schema.elem_typ, path + [i]) for i, e in enumerate(val))
 476     if isinstance(schema, JTuple):
 477         return all(_validate_json(val[i], typ, path + [i])
 478                    for i, typ in enumerate(schema.elem_typs))
 479     if isinstance(schema, JObj):
 480         if val is None and schema.none:
 481             return True
 482         elif val is None:
 483             raise _ValError('val is None', path)
 484         if not hasattr(val, 'keys'):
 485             raise _ValError('val="{}" is not a dict'.format(val), path)
 486         missing_keys = set(schema.sub_elems.keys()).difference(set(val.keys()))
 487         if missing_keys:
 488             raise _ValError('missing keys: {}'.format(missing_keys), path)
 489         unknown_keys = set(val.keys()).difference(set(schema.sub_elems.keys()))
 490         if not schema.allow_unknown and unknown_keys:
 491             raise _ValError('unknown keys: {}'.format(unknown_keys), path)
 492         result = all(
 493             _validate_json(val[key], sub_schema, path + [key])
 494             for key, sub_schema in schema.sub_elems.items()
 495         )
 496         if unknown_keys and schema.allow_unknown and schema.unknown_schema:
 497             result += all(
 498                 _validate_json(val[key], schema.unknown_schema, path + [key])
 499                 for key in unknown_keys
 500             )
 501         return result
 502     if schema in [str, int, float, bool, six.string_types]:
 503         return _validate_json(val, JLeaf(schema), path)
 504
 505     assert False, str(path)