ceph/src/pybind/mgr/dashboard/rest_client.py

   1 # -*- coding: utf-8 -*-
   2 """
   3  *   Copyright (c) 2017 SUSE LLC
   4  *
   5  *  openATTIC is free software; you can redistribute it and/or modify it
   6  *  under the terms of the GNU General Public License as published by
   7  *  the Free Software Foundation; version 2.
   8  *
   9  *  This package is distributed in the hope that it will be useful,
  10  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  *  GNU General Public License for more details.
  13 """
  14 from __future__ import absolute_import
  15
  16 import inspect
  17 import logging
  18 import re
  19
  20 import requests
  21 from requests.exceptions import ConnectionError, InvalidURL, Timeout
  22
  23 from .settings import Settings
  24 from .tools import build_url
  25
  26 try:
  27     from requests.packages.urllib3.exceptions import SSLError
  28 except ImportError:
  29     from urllib3.exceptions import SSLError  # type: ignore
  30
  31 try:
  32     from typing import List
  33 except ImportError:
  34     pass  # Just for type checking
  35
  36
  37 logger = logging.getLogger('rest_client')
  38
  39
  40 class TimeoutRequestsSession(requests.Session):
  41     """
  42     Set timeout argument for all requests if this is not already done.
  43     """
  44
  45     def request(self, *args, **kwargs):
  46         if ((args[8] if len(args) > 8 else None) is None) \
  47                 and kwargs.get('timeout') is None:
  48             if Settings.REST_REQUESTS_TIMEOUT > 0:
  49                 kwargs['timeout'] = Settings.REST_REQUESTS_TIMEOUT
  50         return super(TimeoutRequestsSession, self).request(*args, **kwargs)
  51
  52
  53 class RequestException(Exception):
  54     def __init__(self,
  55                  message,
  56                  status_code=None,
  57                  content=None,
  58                  conn_errno=None,
  59                  conn_strerror=None):
  60         super(RequestException, self).__init__(message)
  61         self.status_code = status_code
  62         self.content = content
  63         self.conn_errno = conn_errno
  64         self.conn_strerror = conn_strerror
  65
  66
  67 class BadResponseFormatException(RequestException):
  68     def __init__(self, message):
  69         super(BadResponseFormatException, self).__init__(
  70             "Bad response format" if message is None else message, None)
  71
  72
  73 class _ResponseValidator(object):
  74     """Simple JSON schema validator
  75
  76     This class implements a very simple validator for the JSON formatted
  77     messages received by request responses from a RestClient instance.
  78
  79     The validator validates the JSON response against a "structure" string that
  80     specifies the structure that the JSON response must comply.  The validation
  81     procedure raises a BadResponseFormatException in case of a validation
  82     failure.
  83
  84     The structure syntax is given by the following grammar:
  85
  86     Structure  ::=  Level
  87     Level      ::=  Path | Path '&' Level
  88     Path       ::=  Step | Step '>'+ Path
  89     Step       ::=  Key  | '?' Key | '*' | '(' Level ')'
  90     Key        ::=  <string> | Array+
  91     Array      ::=  '[' <int> ']' | '[' '*' ']' | '[' '+' ']'
  92
  93     The symbols enclosed in ' ' are tokens of the language, and the + symbol
  94     denotes repetition of of the preceding token at least once.
  95
  96     Examples of usage:
  97
  98     Example 1:
  99         Validator args:
 100             structure = "return > *"
 101             response = { 'return': { ... } }
 102
 103         In the above example the structure will validate against any response
 104         that contains a key named "return" in the root of the response
 105         dictionary and its value is also a dictionary.
 106
 107     Example 2:
 108         Validator args:
 109             structure = "[*]"
 110             response = [...]
 111
 112         In the above example the structure will validate against any response
 113         that is an array of any size.
 114
 115     Example 3:
 116         Validator args:
 117             structure = "return[*]"
 118             response = { 'return': [....] }
 119
 120         In the above example the structure will validate against any response
 121         that contains a key named "return" in the root of the response
 122         dictionary and its value is an array.
 123
 124     Example 4:
 125         Validator args:
 126             structure = "return[0] > token"
 127             response = { 'return': [ { 'token': .... } ] }
 128
 129         In the above example the structure will validate against any response
 130         that contains a key named "return" in the root of the response
 131         dictionary and its value is an array, and the first element of the
 132         array is a dictionary that contains the key 'token'.
 133
 134     Example 5:
 135         Validator args:
 136             structure = "return[0][*] > key1"
 137             response = { 'return': [ [ { 'key1': ... } ], ...] }
 138
 139         In the above example the structure will validate against any response
 140         that contains a key named "return" in the root of the response
 141         dictionary where its value is an array, and the first value of this
 142         array is also an array where all it's values must be a dictionary
 143         containing a key named "key1".
 144
 145     Example 6:
 146         Validator args:
 147             structure = "return > (key1[*] & key2 & ?key3 > subkey)"
 148             response = { 'return': { 'key1': [...], 'key2: .... } ] }
 149
 150         In the above example the structure will validate against any response
 151         that contains a key named "return" in the root of the response
 152         dictionary and its value is a dictionary that must contain a key named
 153         "key1" that is an array, a key named "key2", and optionally a key named
 154         "key3" that is a dictionary that contains a key named "subkey".
 155
 156     Example 7:
 157         Validator args:
 158             structure = "return >> roles[*]"
 159             response = { 'return': { 'key1': { 'roles': [...] }, 'key2': { 'roles': [...] } } }
 160
 161         In the above example the structure will validate against any response
 162         that contains a key named "return" in the root of the response
 163         dictionary, and its value is a dictionary that for any key present in
 164         the dictionary their value is also a dictionary that must contain a key
 165         named 'roles' that is an array.  Please note that you can use any
 166         number of successive '>' to denote the level in the JSON tree that you
 167         want to match next step in the path.
 168
 169     """
 170
 171     @staticmethod
 172     def validate(structure, response):
 173         if structure is None:
 174             return
 175
 176         _ResponseValidator._validate_level(structure, response)
 177
 178     @staticmethod
 179     def _validate_level(level, resp):
 180         if not isinstance(resp, dict) and not isinstance(resp, list):
 181             raise BadResponseFormatException(
 182                 "{} is neither a dict nor a list".format(resp))
 183
 184         paths = _ResponseValidator._parse_level_paths(level)
 185         for path in paths:
 186             path_sep = path.find('>')
 187             if path_sep != -1:
 188                 level_next = path[path_sep + 1:].strip()
 189             else:
 190                 path_sep = len(path)
 191                 level_next = None  # type: ignore
 192             key = path[:path_sep].strip()
 193
 194             if key == '*':
 195                 continue
 196             elif key == '':  # check all keys
 197                 for k in resp.keys():  # type: ignore
 198                     _ResponseValidator._validate_key(k, level_next, resp)
 199             else:
 200                 _ResponseValidator._validate_key(key, level_next, resp)
 201
 202     @staticmethod
 203     def _validate_array(array_seq, level_next, resp):
 204         if array_seq:
 205             if not isinstance(resp, list):
 206                 raise BadResponseFormatException(
 207                     "{} is not an array".format(resp))
 208             if array_seq[0].isdigit():
 209                 idx = int(array_seq[0])
 210                 if len(resp) <= idx:
 211                     raise BadResponseFormatException(
 212                         "length of array {} is lower than the index {}".format(
 213                             resp, idx))
 214                 _ResponseValidator._validate_array(array_seq[1:], level_next,
 215                                                    resp[idx])
 216             elif array_seq[0] == '*':
 217                 for r in resp:
 218                     _ResponseValidator._validate_array(array_seq[1:],
 219                                                        level_next, r)
 220             elif array_seq[0] == '+':
 221                 if len(resp) < 1:
 222                     raise BadResponseFormatException(
 223                         "array should not be empty")
 224                 for r in resp:
 225                     _ResponseValidator._validate_array(array_seq[1:],
 226                                                        level_next, r)
 227             else:
 228                 raise Exception(
 229                     "Response structure is invalid: only <int> | '*' are "
 230                     "allowed as array index arguments")
 231         else:
 232             if level_next:
 233                 _ResponseValidator._validate_level(level_next, resp)
 234
 235     @staticmethod
 236     def _validate_key(key, level_next, resp):
 237         array_access = [a.strip() for a in key.split("[")]
 238         key = array_access[0]
 239         if key:
 240             optional = key[0] == '?'
 241             if optional:
 242                 key = key[1:]
 243             if key not in resp:
 244                 if optional:
 245                     return
 246                 raise BadResponseFormatException(
 247                     "key {} is not in dict {}".format(key, resp))
 248             resp_next = resp[key]
 249         else:
 250             resp_next = resp
 251         if len(array_access) > 1:
 252             _ResponseValidator._validate_array(
 253                 [a[0:-1] for a in array_access[1:]], level_next, resp_next)
 254         else:
 255             if level_next:
 256                 _ResponseValidator._validate_level(level_next, resp_next)
 257
 258     @staticmethod
 259     def _parse_level_paths(level):
 260         # type: (str) -> List[str]
 261         level = level.strip()
 262         if level[0] == '(':
 263             level = level[1:]
 264             if level[-1] == ')':
 265                 level = level[:-1]
 266
 267         paths = []
 268         lp = 0
 269         nested = 0
 270         for i, c in enumerate(level):
 271             if c == '&' and nested == 0:
 272                 paths.append(level[lp:i].strip())
 273                 lp = i + 1
 274             elif c == '(':
 275                 nested += 1
 276             elif c == ')':
 277                 nested -= 1
 278         paths.append(level[lp:].strip())
 279         return paths
 280
 281
 282 class _Request(object):
 283     def __init__(self, method, path, path_params, rest_client, resp_structure):
 284         self.method = method
 285         self.path = path
 286         self.path_params = path_params
 287         self.rest_client = rest_client
 288         self.resp_structure = resp_structure
 289
 290     def _gen_path(self):
 291         new_path = self.path
 292         matches = re.finditer(r'\{(\w+?)\}', self.path)
 293         for match in matches:
 294             if match:
 295                 param_key = match.group(1)
 296                 if param_key in self.path_params:
 297                     new_path = new_path.replace(
 298                         match.group(0), self.path_params[param_key])
 299                 else:
 300                     raise RequestException(
 301                         'Invalid path. Param "{}" was not specified'
 302                         .format(param_key), None)
 303         return new_path
 304
 305     def __call__(self,
 306                  req_data=None,
 307                  method=None,
 308                  params=None,
 309                  data=None,
 310                  raw_content=False,
 311                  headers=None):
 312         method = method if method else self.method
 313         if not method:
 314             raise Exception('No HTTP request method specified')
 315         if req_data:
 316             if method == 'get':
 317                 if params:
 318                     raise Exception('Ambiguous source of GET params')
 319                 params = req_data
 320             else:
 321                 if data:
 322                     raise Exception('Ambiguous source of {} data'.format(
 323                         method.upper()))
 324                 data = req_data
 325         resp = self.rest_client.do_request(method, self._gen_path(), params,
 326                                            data, raw_content, headers)
 327         if raw_content and self.resp_structure:
 328             raise Exception("Cannot validate response in raw format")
 329         _ResponseValidator.validate(self.resp_structure, resp)
 330         return resp
 331
 332
 333 class RestClient(object):
 334     def __init__(self, host, port, client_name=None, ssl=False, auth=None, ssl_verify=True):
 335         super(RestClient, self).__init__()
 336         self.client_name = client_name if client_name else ''
 337         self.host = host
 338         self.port = port
 339         self.base_url = build_url(
 340             scheme='https' if ssl else 'http', host=host, port=port)
 341         logger.debug("REST service base URL: %s", self.base_url)
 342         self.headers = {'Accept': 'application/json'}
 343         self.auth = auth
 344         self.session = TimeoutRequestsSession()
 345         self.session.verify = ssl_verify
 346
 347     def _login(self, request=None):
 348         pass
 349
 350     def _is_logged_in(self):
 351         pass
 352
 353     def _reset_login(self):
 354         pass
 355
 356     def is_service_online(self, request=None):
 357         pass
 358
 359     @staticmethod
 360     def requires_login(func):
 361         def func_wrapper(self, *args, **kwargs):
 362             retries = 2
 363             while True:
 364                 try:
 365                     if not self._is_logged_in():
 366                         self._login()
 367                     resp = func(self, *args, **kwargs)
 368                     return resp
 369                 except RequestException as e:
 370                     if isinstance(e, BadResponseFormatException):
 371                         raise e
 372                     retries -= 1
 373                     if e.status_code not in [401, 403] or retries == 0:
 374                         raise e
 375                     self._reset_login()
 376
 377         return func_wrapper
 378
 379     def do_request(self,
 380                    method,
 381                    path,
 382                    params=None,
 383                    data=None,
 384                    raw_content=False,
 385                    headers=None):
 386         url = '{}{}'.format(self.base_url, path)
 387         logger.debug('%s REST API %s req: %s data: %s', self.client_name,
 388                      method.upper(), path, data)
 389         request_headers = self.headers.copy()
 390         if headers:
 391             request_headers.update(headers)
 392         try:
 393             if method.lower() == 'get':
 394                 resp = self.session.get(
 395                     url, headers=request_headers, params=params, auth=self.auth)
 396             elif method.lower() == 'post':
 397                 resp = self.session.post(
 398                     url,
 399                     headers=request_headers,
 400                     params=params,
 401                     data=data,
 402                     auth=self.auth)
 403             elif method.lower() == 'put':
 404                 resp = self.session.put(
 405                     url,
 406                     headers=request_headers,
 407                     params=params,
 408                     data=data,
 409                     auth=self.auth)
 410             elif method.lower() == 'delete':
 411                 resp = self.session.delete(
 412                     url,
 413                     headers=request_headers,
 414                     params=params,
 415                     data=data,
 416                     auth=self.auth)
 417             else:
 418                 raise RequestException('Method "{}" not supported'.format(
 419                     method.upper()), None)
 420             if resp.ok:
 421                 logger.debug("%s REST API %s res status: %s content: %s",
 422                              self.client_name, method.upper(),
 423                              resp.status_code, resp.text)
 424                 if raw_content:
 425                     return resp.content
 426                 try:
 427                     return resp.json() if resp.text else None
 428                 except ValueError:
 429                     logger.error(
 430                         "%s REST API failed %s req while decoding JSON "
 431                         "response : %s",
 432                         self.client_name, method.upper(), resp.text)
 433                     raise RequestException(
 434                         "{} REST API failed request while decoding JSON "
 435                         "response: {}".format(self.client_name, resp.text),
 436                         resp.status_code, resp.text)
 437             else:
 438                 logger.error(
 439                     "%s REST API failed %s req status: %s", self.client_name,
 440                     method.upper(), resp.status_code)
 441                 from pprint import pformat as pf
 442
 443                 raise RequestException(
 444                     "{} REST API failed request with status code {}\n"
 445                     "{}"  # TODO remove
 446                     .format(self.client_name, resp.status_code, pf(
 447                         resp.content)),
 448                     self._handle_response_status_code(resp.status_code),
 449                     resp.content)
 450         except ConnectionError as ex:
 451             if ex.args:
 452                 if isinstance(ex.args[0], SSLError):
 453                     errno = "n/a"
 454                     strerror = "SSL error. Probably trying to access a non " \
 455                                "SSL connection."
 456                     logger.error("%s REST API failed %s, SSL error (url=%s).",
 457                                  self.client_name, method.upper(), ex.request.url)
 458                 else:
 459                     try:
 460                         match = re.match(r'.*: \[Errno (-?\d+)\] (.+)',
 461                                          ex.args[0].reason.args[0])
 462                     except AttributeError:
 463                         match = None
 464                     if match:
 465                         errno = match.group(1)
 466                         strerror = match.group(2)
 467                         logger.error(
 468                             "%s REST API failed %s, connection error (url=%s): "
 469                             "[errno: %s] %s",
 470                             self.client_name, method.upper(), ex.request.url,
 471                             errno, strerror)
 472                     else:
 473                         errno = "n/a"
 474                         strerror = "n/a"
 475                         logger.error(
 476                             "%s REST API failed %s, connection error (url=%s).",
 477                             self.client_name, method.upper(), ex.request.url)
 478             else:
 479                 errno = "n/a"
 480                 strerror = "n/a"
 481                 logger.error("%s REST API failed %s, connection error (url=%s).",
 482                              self.client_name, method.upper(), ex.request.url)
 483
 484             if errno != "n/a":
 485                 ex_msg = (
 486                     "{} REST API cannot be reached: {} [errno {}]. "
 487                     "Please check your configuration and that the API endpoint"
 488                     " is accessible"
 489                     .format(self.client_name, strerror, errno))
 490             else:
 491                 ex_msg = (
 492                     "{} REST API cannot be reached. Please check "
 493                     "your configuration and that the API endpoint is"
 494                     " accessible"
 495                     .format(self.client_name))
 496             raise RequestException(
 497                 ex_msg, conn_errno=errno, conn_strerror=strerror)
 498         except InvalidURL as ex:
 499             logger.exception("%s REST API failed %s: %s", self.client_name,
 500                              method.upper(), str(ex))
 501             raise RequestException(str(ex))
 502         except Timeout as ex:
 503             msg = "{} REST API {} timed out after {} seconds (url={}).".format(
 504                 self.client_name, ex.request.method, Settings.REST_REQUESTS_TIMEOUT,
 505                 ex.request.url)
 506             logger.exception(msg)
 507             raise RequestException(msg)
 508
 509     @staticmethod
 510     def _handle_response_status_code(status_code: int) -> int:
 511         """
 512         Method to be overridden by subclasses that need specific handling.
 513         """
 514         return status_code
 515
 516     @staticmethod
 517     def api(path, **api_kwargs):
 518         def call_decorator(func):
 519             def func_wrapper(self, *args, **kwargs):
 520                 method = api_kwargs.get('method', None)
 521                 resp_structure = api_kwargs.get('resp_structure', None)
 522                 args_name = inspect.getfullargspec(func).args
 523                 args_dict = dict(zip(args_name[1:], args))
 524                 for key, val in kwargs.items():
 525                     args_dict[key] = val
 526                 return func(
 527                     self,
 528                     *args,
 529                     request=_Request(method, path, args_dict, self,
 530                                      resp_structure),
 531                     **kwargs)
 532
 533             return func_wrapper
 534
 535         return call_decorator
 536
 537     @staticmethod
 538     def api_get(path, resp_structure=None):
 539         return RestClient.api(
 540             path, method='get', resp_structure=resp_structure)
 541
 542     @staticmethod
 543     def api_post(path, resp_structure=None):
 544         return RestClient.api(
 545             path, method='post', resp_structure=resp_structure)
 546
 547     @staticmethod
 548     def api_put(path, resp_structure=None):
 549         return RestClient.api(
 550             path, method='put', resp_structure=resp_structure)
 551
 552     @staticmethod
 553     def api_delete(path, resp_structure=None):
 554         return RestClient.api(
 555             path, method='delete', resp_structure=resp_structure)