ceph/src/pybind/mgr/dashboard/rest_client.py

   1 # -*- coding: utf-8 -*-
   2 """
   3  *   Copyright (c) 2017 SUSE LLC
   4  *
   5  *  openATTIC is free software; you can redistribute it and/or modify it
   6  *  under the terms of the GNU General Public License as published by
   7  *  the Free Software Foundation; version 2.
   8  *
   9  *  This package is distributed in the hope that it will be useful,
  10  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  *  GNU General Public License for more details.
  13 """
  14
  15 import inspect
  16 import logging
  17 import re
  18
  19 import requests
  20 from requests.auth import AuthBase
  21 from requests.exceptions import ConnectionError, InvalidURL, Timeout
  22
  23 from .settings import Settings
  24
  25 try:
  26     from requests.packages.urllib3.exceptions import SSLError
  27 except ImportError:
  28     from urllib3.exceptions import SSLError  # type: ignore
  29
  30 from typing import List, Optional
  31
  32 from mgr_util import build_url
  33
  34 logger = logging.getLogger('rest_client')
  35
  36
  37 class TimeoutRequestsSession(requests.Session):
  38     """
  39     Set timeout argument for all requests if this is not already done.
  40     """
  41
  42     def request(self, *args, **kwargs):
  43         if ((args[8] if len(args) > 8 else None) is None) \
  44                 and kwargs.get('timeout') is None:
  45             if Settings.REST_REQUESTS_TIMEOUT > 0:
  46                 kwargs['timeout'] = Settings.REST_REQUESTS_TIMEOUT
  47         return super(TimeoutRequestsSession, self).request(*args, **kwargs)
  48
  49
  50 class RequestException(Exception):
  51     def __init__(self,
  52                  message,
  53                  status_code=None,
  54                  content=None,
  55                  conn_errno=None,
  56                  conn_strerror=None):
  57         super(RequestException, self).__init__(message)
  58         self.status_code = status_code
  59         self.content = content
  60         self.conn_errno = conn_errno
  61         self.conn_strerror = conn_strerror
  62
  63
  64 class BadResponseFormatException(RequestException):
  65     def __init__(self, message):
  66         super(BadResponseFormatException, self).__init__(
  67             "Bad response format" if message is None else message, None)
  68
  69
  70 class _ResponseValidator(object):
  71     """Simple JSON schema validator
  72
  73     This class implements a very simple validator for the JSON formatted
  74     messages received by request responses from a RestClient instance.
  75
  76     The validator validates the JSON response against a "structure" string that
  77     specifies the structure that the JSON response must comply.  The validation
  78     procedure raises a BadResponseFormatException in case of a validation
  79     failure.
  80
  81     The structure syntax is given by the following grammar:
  82
  83     Structure  ::=  Level
  84     Level      ::=  Path | Path '&' Level
  85     Path       ::=  Step | Step '>'+ Path
  86     Step       ::=  Key  | '?' Key | '*' | '(' Level ')'
  87     Key        ::=  <string> | Array+
  88     Array      ::=  '[' <int> ']' | '[' '*' ']' | '[' '+' ']'
  89
  90     The symbols enclosed in ' ' are tokens of the language, and the + symbol
  91     denotes repetition of of the preceding token at least once.
  92
  93     Examples of usage:
  94
  95     Example 1:
  96         Validator args:
  97             structure = "return > *"
  98             response = { 'return': { ... } }
  99
 100         In the above example the structure will validate against any response
 101         that contains a key named "return" in the root of the response
 102         dictionary and its value is also a dictionary.
 103
 104     Example 2:
 105         Validator args:
 106             structure = "[*]"
 107             response = [...]
 108
 109         In the above example the structure will validate against any response
 110         that is an array of any size.
 111
 112     Example 3:
 113         Validator args:
 114             structure = "return[*]"
 115             response = { 'return': [....] }
 116
 117         In the above example the structure will validate against any response
 118         that contains a key named "return" in the root of the response
 119         dictionary and its value is an array.
 120
 121     Example 4:
 122         Validator args:
 123             structure = "return[0] > token"
 124             response = { 'return': [ { 'token': .... } ] }
 125
 126         In the above example the structure will validate against any response
 127         that contains a key named "return" in the root of the response
 128         dictionary and its value is an array, and the first element of the
 129         array is a dictionary that contains the key 'token'.
 130
 131     Example 5:
 132         Validator args:
 133             structure = "return[0][*] > key1"
 134             response = { 'return': [ [ { 'key1': ... } ], ...] }
 135
 136         In the above example the structure will validate against any response
 137         that contains a key named "return" in the root of the response
 138         dictionary where its value is an array, and the first value of this
 139         array is also an array where all it's values must be a dictionary
 140         containing a key named "key1".
 141
 142     Example 6:
 143         Validator args:
 144             structure = "return > (key1[*] & key2 & ?key3 > subkey)"
 145             response = { 'return': { 'key1': [...], 'key2: .... } ] }
 146
 147         In the above example the structure will validate against any response
 148         that contains a key named "return" in the root of the response
 149         dictionary and its value is a dictionary that must contain a key named
 150         "key1" that is an array, a key named "key2", and optionally a key named
 151         "key3" that is a dictionary that contains a key named "subkey".
 152
 153     Example 7:
 154         Validator args:
 155             structure = "return >> roles[*]"
 156             response = { 'return': { 'key1': { 'roles': [...] }, 'key2': { 'roles': [...] } } }
 157
 158         In the above example the structure will validate against any response
 159         that contains a key named "return" in the root of the response
 160         dictionary, and its value is a dictionary that for any key present in
 161         the dictionary their value is also a dictionary that must contain a key
 162         named 'roles' that is an array.  Please note that you can use any
 163         number of successive '>' to denote the level in the JSON tree that you
 164         want to match next step in the path.
 165
 166     """
 167
 168     @staticmethod
 169     def validate(structure, response):
 170         if structure is None:
 171             return
 172
 173         _ResponseValidator._validate_level(structure, response)
 174
 175     @staticmethod
 176     def _validate_level(level, resp):
 177         if not isinstance(resp, dict) and not isinstance(resp, list):
 178             raise BadResponseFormatException(
 179                 "{} is neither a dict nor a list".format(resp))
 180
 181         paths = _ResponseValidator._parse_level_paths(level)
 182         for path in paths:
 183             path_sep = path.find('>')
 184             if path_sep != -1:
 185                 level_next = path[path_sep + 1:].strip()
 186             else:
 187                 path_sep = len(path)
 188                 level_next = None  # type: ignore
 189             key = path[:path_sep].strip()
 190
 191             if key == '*':
 192                 continue
 193             elif key == '':  # check all keys
 194                 for k in resp.keys():  # type: ignore
 195                     _ResponseValidator._validate_key(k, level_next, resp)
 196             else:
 197                 _ResponseValidator._validate_key(key, level_next, resp)
 198
 199     @staticmethod
 200     def _validate_array(array_seq, level_next, resp):
 201         if array_seq:
 202             if not isinstance(resp, list):
 203                 raise BadResponseFormatException(
 204                     "{} is not an array".format(resp))
 205             if array_seq[0].isdigit():
 206                 idx = int(array_seq[0])
 207                 if len(resp) <= idx:
 208                     raise BadResponseFormatException(
 209                         "length of array {} is lower than the index {}".format(
 210                             resp, idx))
 211                 _ResponseValidator._validate_array(array_seq[1:], level_next,
 212                                                    resp[idx])
 213             elif array_seq[0] == '*':
 214                 for r in resp:
 215                     _ResponseValidator._validate_array(array_seq[1:],
 216                                                        level_next, r)
 217             elif array_seq[0] == '+':
 218                 if len(resp) < 1:
 219                     raise BadResponseFormatException(
 220                         "array should not be empty")
 221                 for r in resp:
 222                     _ResponseValidator._validate_array(array_seq[1:],
 223                                                        level_next, r)
 224             else:
 225                 raise Exception(
 226                     "Response structure is invalid: only <int> | '*' are "
 227                     "allowed as array index arguments")
 228         else:
 229             if level_next:
 230                 _ResponseValidator._validate_level(level_next, resp)
 231
 232     @staticmethod
 233     def _validate_key(key, level_next, resp):
 234         array_access = [a.strip() for a in key.split("[")]
 235         key = array_access[0]
 236         if key:
 237             optional = key[0] == '?'
 238             if optional:
 239                 key = key[1:]
 240             if key not in resp:
 241                 if optional:
 242                     return
 243                 raise BadResponseFormatException(
 244                     "key {} is not in dict {}".format(key, resp))
 245             resp_next = resp[key]
 246         else:
 247             resp_next = resp
 248         if len(array_access) > 1:
 249             _ResponseValidator._validate_array(
 250                 [a[0:-1] for a in array_access[1:]], level_next, resp_next)
 251         else:
 252             if level_next:
 253                 _ResponseValidator._validate_level(level_next, resp_next)
 254
 255     @staticmethod
 256     def _parse_level_paths(level):
 257         # type: (str) -> List[str]
 258         level = level.strip()
 259         if level[0] == '(':
 260             level = level[1:]
 261             if level[-1] == ')':
 262                 level = level[:-1]
 263
 264         paths = []
 265         lp = 0
 266         nested = 0
 267         for i, c in enumerate(level):
 268             if c == '&' and nested == 0:
 269                 paths.append(level[lp:i].strip())
 270                 lp = i + 1
 271             elif c == '(':
 272                 nested += 1
 273             elif c == ')':
 274                 nested -= 1
 275         paths.append(level[lp:].strip())
 276         return paths
 277
 278
 279 class _Request(object):
 280     def __init__(self, method, path, path_params, rest_client, resp_structure):
 281         self.method = method
 282         self.path = path
 283         self.path_params = path_params
 284         self.rest_client = rest_client
 285         self.resp_structure = resp_structure
 286
 287     def _gen_path(self):
 288         new_path = self.path
 289         matches = re.finditer(r'\{(\w+?)\}', self.path)
 290         for match in matches:
 291             if match:
 292                 param_key = match.group(1)
 293                 if param_key in self.path_params:
 294                     new_path = new_path.replace(
 295                         match.group(0), self.path_params[param_key])
 296                 else:
 297                     raise RequestException(
 298                         'Invalid path. Param "{}" was not specified'
 299                         .format(param_key), None)
 300         return new_path
 301
 302     def __call__(self,
 303                  req_data=None,
 304                  method=None,
 305                  params=None,
 306                  data=None,
 307                  raw_content=False,
 308                  headers=None):
 309         method = method if method else self.method
 310         if not method:
 311             raise Exception('No HTTP request method specified')
 312         if req_data:
 313             if method == 'get':
 314                 if params:
 315                     raise Exception('Ambiguous source of GET params')
 316                 params = req_data
 317             else:
 318                 if data:
 319                     raise Exception('Ambiguous source of {} data'.format(
 320                         method.upper()))
 321                 data = req_data
 322         resp = self.rest_client.do_request(method, self._gen_path(), params,
 323                                            data, raw_content, headers)
 324         if raw_content and self.resp_structure:
 325             raise Exception("Cannot validate response in raw format")
 326         _ResponseValidator.validate(self.resp_structure, resp)
 327         return resp
 328
 329
 330 class RestClient(object):
 331     def __init__(self,
 332                  host: str,
 333                  port: int,
 334                  client_name: Optional[str] = None,
 335                  ssl: bool = False,
 336                  auth: Optional[AuthBase] = None,
 337                  ssl_verify: bool = True) -> None:
 338         super(RestClient, self).__init__()
 339         self.client_name = client_name if client_name else ''
 340         self.host = host
 341         self.port = port
 342         self.base_url = build_url(
 343             scheme='https' if ssl else 'http', host=host, port=port)
 344         logger.debug("REST service base URL: %s", self.base_url)
 345         self.headers = {'Accept': 'application/json'}
 346         self.auth = auth
 347         self.session = TimeoutRequestsSession()
 348         self.session.verify = ssl_verify
 349
 350     def _login(self, request=None):
 351         pass
 352
 353     def _is_logged_in(self):
 354         pass
 355
 356     def _reset_login(self):
 357         pass
 358
 359     def is_service_online(self, request=None):
 360         pass
 361
 362     @staticmethod
 363     def requires_login(func):
 364         def func_wrapper(self, *args, **kwargs):
 365             retries = 2
 366             while True:
 367                 try:
 368                     if not self._is_logged_in():
 369                         self._login()
 370                     resp = func(self, *args, **kwargs)
 371                     return resp
 372                 except RequestException as e:
 373                     if isinstance(e, BadResponseFormatException):
 374                         raise e
 375                     retries -= 1
 376                     if e.status_code not in [401, 403] or retries == 0:
 377                         raise e
 378                     self._reset_login()
 379
 380         return func_wrapper
 381
 382     def do_request(self,
 383                    method,
 384                    path,
 385                    params=None,
 386                    data=None,
 387                    raw_content=False,
 388                    headers=None):
 389         url = '{}{}'.format(self.base_url, path)
 390         logger.debug('%s REST API %s req: %s data: %s', self.client_name,
 391                      method.upper(), path, data)
 392         request_headers = self.headers.copy()
 393         if headers:
 394             request_headers.update(headers)
 395         try:
 396             if method.lower() == 'get':
 397                 resp = self.session.get(
 398                     url, headers=request_headers, params=params, auth=self.auth)
 399             elif method.lower() == 'post':
 400                 resp = self.session.post(
 401                     url,
 402                     headers=request_headers,
 403                     params=params,
 404                     data=data,
 405                     auth=self.auth)
 406             elif method.lower() == 'put':
 407                 resp = self.session.put(
 408                     url,
 409                     headers=request_headers,
 410                     params=params,
 411                     data=data,
 412                     auth=self.auth)
 413             elif method.lower() == 'delete':
 414                 resp = self.session.delete(
 415                     url,
 416                     headers=request_headers,
 417                     params=params,
 418                     data=data,
 419                     auth=self.auth)
 420             else:
 421                 raise RequestException('Method "{}" not supported'.format(
 422                     method.upper()), None)
 423             if resp.ok:
 424                 logger.debug("%s REST API %s res status: %s content: %s",
 425                              self.client_name, method.upper(),
 426                              resp.status_code, resp.text)
 427                 if raw_content:
 428                     return resp.content
 429                 try:
 430                     return resp.json() if resp.text else None
 431                 except ValueError:
 432                     logger.error(
 433                         "%s REST API failed %s req while decoding JSON "
 434                         "response : %s",
 435                         self.client_name, method.upper(), resp.text)
 436                     raise RequestException(
 437                         "{} REST API failed request while decoding JSON "
 438                         "response: {}".format(self.client_name, resp.text),
 439                         resp.status_code, resp.text)
 440             else:
 441                 logger.error(
 442                     "%s REST API failed %s req status: %s", self.client_name,
 443                     method.upper(), resp.status_code)
 444                 from pprint import pformat as pf
 445
 446                 raise RequestException(
 447                     "{} REST API failed request with status code {}\n"
 448                     "{}"  # TODO remove
 449                     .format(self.client_name, resp.status_code, pf(
 450                         resp.content)),
 451                     self._handle_response_status_code(resp.status_code),
 452                     resp.content)
 453         except ConnectionError as ex:
 454             if ex.args:
 455                 if isinstance(ex.args[0], SSLError):
 456                     errno = "n/a"
 457                     strerror = "SSL error. Probably trying to access a non " \
 458                                "SSL connection."
 459                     logger.error("%s REST API failed %s, SSL error (url=%s).",
 460                                  self.client_name, method.upper(), ex.request.url)
 461                 else:
 462                     try:
 463                         match = re.match(r'.*: \[Errno (-?\d+)\] (.+)',
 464                                          ex.args[0].reason.args[0])
 465                     except AttributeError:
 466                         match = None
 467                     if match:
 468                         errno = match.group(1)
 469                         strerror = match.group(2)
 470                         logger.error(
 471                             "%s REST API failed %s, connection error (url=%s): "
 472                             "[errno: %s] %s",
 473                             self.client_name, method.upper(), ex.request.url,
 474                             errno, strerror)
 475                     else:
 476                         errno = "n/a"
 477                         strerror = "n/a"
 478                         logger.error(
 479                             "%s REST API failed %s, connection error (url=%s).",
 480                             self.client_name, method.upper(), ex.request.url)
 481             else:
 482                 errno = "n/a"
 483                 strerror = "n/a"
 484                 logger.error("%s REST API failed %s, connection error (url=%s).",
 485                              self.client_name, method.upper(), ex.request.url)
 486
 487             if errno != "n/a":
 488                 ex_msg = (
 489                     "{} REST API cannot be reached: {} [errno {}]. "
 490                     "Please check your configuration and that the API endpoint"
 491                     " is accessible"
 492                     .format(self.client_name, strerror, errno))
 493             else:
 494                 ex_msg = (
 495                     "{} REST API cannot be reached. Please check "
 496                     "your configuration and that the API endpoint is"
 497                     " accessible"
 498                     .format(self.client_name))
 499             raise RequestException(
 500                 ex_msg, conn_errno=errno, conn_strerror=strerror)
 501         except InvalidURL as ex:
 502             logger.exception("%s REST API failed %s: %s", self.client_name,
 503                              method.upper(), str(ex))
 504             raise RequestException(str(ex))
 505         except Timeout as ex:
 506             msg = "{} REST API {} timed out after {} seconds (url={}).".format(
 507                 self.client_name, ex.request.method, Settings.REST_REQUESTS_TIMEOUT,
 508                 ex.request.url)
 509             logger.exception(msg)
 510             raise RequestException(msg)
 511
 512     @staticmethod
 513     def _handle_response_status_code(status_code: int) -> int:
 514         """
 515         Method to be overridden by subclasses that need specific handling.
 516         """
 517         return status_code
 518
 519     @staticmethod
 520     def api(path, **api_kwargs):
 521         def call_decorator(func):
 522             def func_wrapper(self, *args, **kwargs):
 523                 method = api_kwargs.get('method', None)
 524                 resp_structure = api_kwargs.get('resp_structure', None)
 525                 args_name = inspect.getfullargspec(func).args
 526                 args_dict = dict(zip(args_name[1:], args))
 527                 for key, val in kwargs.items():
 528                     args_dict[key] = val
 529                 return func(
 530                     self,
 531                     *args,
 532                     request=_Request(method, path, args_dict, self,
 533                                      resp_structure),
 534                     **kwargs)
 535
 536             return func_wrapper
 537
 538         return call_decorator
 539
 540     @staticmethod
 541     def api_get(path, resp_structure=None):
 542         return RestClient.api(
 543             path, method='get', resp_structure=resp_structure)
 544
 545     @staticmethod
 546     def api_post(path, resp_structure=None):
 547         return RestClient.api(
 548             path, method='post', resp_structure=resp_structure)
 549
 550     @staticmethod
 551     def api_put(path, resp_structure=None):
 552         return RestClient.api(
 553             path, method='put', resp_structure=resp_structure)
 554
 555     @staticmethod
 556     def api_delete(path, resp_structure=None):
 557         return RestClient.api(
 558             path, method='delete', resp_structure=resp_structure)