ceph/src/pybind/mgr/dashboard/rest_client.py

   1 # -*- coding: utf-8 -*-
   2 """
   3  *   Copyright (c) 2017 SUSE LLC
   4  *
   5  *  openATTIC is free software; you can redistribute it and/or modify it
   6  *  under the terms of the GNU General Public License as published by
   7  *  the Free Software Foundation; version 2.
   8  *
   9  *  This package is distributed in the hope that it will be useful,
  10  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  *  GNU General Public License for more details.
  13 """
  14 from __future__ import absolute_import
  15
  16 import inspect
  17 import logging
  18 import re
  19 import requests
  20 from requests.exceptions import ConnectionError, InvalidURL, Timeout
  21 from .settings import Settings
  22 from .tools import build_url
  23
  24 try:
  25     from requests.packages.urllib3.exceptions import SSLError
  26 except ImportError:
  27     from urllib3.exceptions import SSLError  # type: ignore
  28
  29 try:
  30     from typing import List
  31 except ImportError:
  32     pass  # Just for type checking
  33
  34
  35 logger = logging.getLogger('rest_client')
  36
  37
  38 class TimeoutRequestsSession(requests.Session):
  39     """
  40     Set timeout argument for all requests if this is not already done.
  41     """
  42     def request(self, *args, **kwargs):
  43         if ((args[8] if len(args) > 8 else None) is None) \
  44                 and kwargs.get('timeout') is None:
  45             if Settings.REST_REQUESTS_TIMEOUT > 0:
  46                 kwargs['timeout'] = Settings.REST_REQUESTS_TIMEOUT
  47         return super(TimeoutRequestsSession, self).request(*args, **kwargs)
  48
  49
  50 class RequestException(Exception):
  51     def __init__(self,
  52                  message,
  53                  status_code=None,
  54                  content=None,
  55                  conn_errno=None,
  56                  conn_strerror=None):
  57         super(RequestException, self).__init__(message)
  58         self.status_code = status_code
  59         self.content = content
  60         self.conn_errno = conn_errno
  61         self.conn_strerror = conn_strerror
  62
  63
  64 class BadResponseFormatException(RequestException):
  65     def __init__(self, message):
  66         super(BadResponseFormatException, self).__init__(
  67             "Bad response format" if message is None else message, None)
  68
  69
  70 class _ResponseValidator(object):
  71     """Simple JSON schema validator
  72
  73     This class implements a very simple validator for the JSON formatted
  74     messages received by request responses from a RestClient instance.
  75
  76     The validator validates the JSON response against a "structure" string that
  77     specifies the structure that the JSON response must comply.  The validation
  78     procedure raises a BadResponseFormatException in case of a validation
  79     failure.
  80
  81     The structure syntax is given by the following grammar:
  82
  83     Structure  ::=  Level
  84     Level      ::=  Path | Path '&' Level
  85     Path       ::=  Step | Step '>'+ Path
  86     Step       ::=  Key  | '?' Key | '*' | '(' Level ')'
  87     Key        ::=  <string> | Array+
  88     Array      ::=  '[' <int> ']' | '[' '*' ']' | '[' '+' ']'
  89
  90     The symbols enclosed in ' ' are tokens of the language, and the + symbol
  91     denotes repetition of of the preceding token at least once.
  92
  93     Examples of usage:
  94
  95     Example 1:
  96         Validator args:
  97             structure = "return > *"
  98             response = { 'return': { ... } }
  99
 100         In the above example the structure will validate against any response
 101         that contains a key named "return" in the root of the response
 102         dictionary and its value is also a dictionary.
 103
 104     Example 2:
 105         Validator args:
 106             structure = "[*]"
 107             response = [...]
 108
 109         In the above example the structure will validate against any response
 110         that is an array of any size.
 111
 112     Example 3:
 113         Validator args:
 114             structure = "return[*]"
 115             response = { 'return': [....] }
 116
 117         In the above example the structure will validate against any response
 118         that contains a key named "return" in the root of the response
 119         dictionary and its value is an array.
 120
 121     Example 4:
 122         Validator args:
 123             structure = "return[0] > token"
 124             response = { 'return': [ { 'token': .... } ] }
 125
 126         In the above example the structure will validate against any response
 127         that contains a key named "return" in the root of the response
 128         dictionary and its value is an array, and the first element of the
 129         array is a dictionary that contains the key 'token'.
 130
 131     Example 5:
 132         Validator args:
 133             structure = "return[0][*] > key1"
 134             response = { 'return': [ [ { 'key1': ... } ], ...] }
 135
 136         In the above example the structure will validate against any response
 137         that contains a key named "return" in the root of the response
 138         dictionary where its value is an array, and the first value of this
 139         array is also an array where all it's values must be a dictionary
 140         containing a key named "key1".
 141
 142     Example 6:
 143         Validator args:
 144             structure = "return > (key1[*] & key2 & ?key3 > subkey)"
 145             response = { 'return': { 'key1': [...], 'key2: .... } ] }
 146
 147         In the above example the structure will validate against any response
 148         that contains a key named "return" in the root of the response
 149         dictionary and its value is a dictionary that must contain a key named
 150         "key1" that is an array, a key named "key2", and optionally a key named
 151         "key3" that is a dictionary that contains a key named "subkey".
 152
 153     Example 7:
 154         Validator args:
 155             structure = "return >> roles[*]"
 156             response = { 'return': { 'key1': { 'roles': [...] }, 'key2': { 'roles': [...] } } }
 157
 158         In the above example the structure will validate against any response
 159         that contains a key named "return" in the root of the response
 160         dictionary, and its value is a dictionary that for any key present in
 161         the dictionary their value is also a dictionary that must contain a key
 162         named 'roles' that is an array.  Please note that you can use any
 163         number of successive '>' to denote the level in the JSON tree that you
 164         want to match next step in the path.
 165
 166     """
 167
 168     @staticmethod
 169     def validate(structure, response):
 170         if structure is None:
 171             return
 172
 173         _ResponseValidator._validate_level(structure, response)
 174
 175     @staticmethod
 176     def _validate_level(level, resp):
 177         if not isinstance(resp, dict) and not isinstance(resp, list):
 178             raise BadResponseFormatException(
 179                 "{} is neither a dict nor a list".format(resp))
 180
 181         paths = _ResponseValidator._parse_level_paths(level)
 182         for path in paths:
 183             path_sep = path.find('>')
 184             if path_sep != -1:
 185                 level_next = path[path_sep + 1:].strip()
 186             else:
 187                 path_sep = len(path)
 188                 level_next = None  # type: ignore
 189             key = path[:path_sep].strip()
 190
 191             if key == '*':
 192                 continue
 193             elif key == '':  # check all keys
 194                 for k in resp.keys():  # type: ignore
 195                     _ResponseValidator._validate_key(k, level_next, resp)
 196             else:
 197                 _ResponseValidator._validate_key(key, level_next, resp)
 198
 199     @staticmethod
 200     def _validate_array(array_seq, level_next, resp):
 201         if array_seq:
 202             if not isinstance(resp, list):
 203                 raise BadResponseFormatException(
 204                     "{} is not an array".format(resp))
 205             if array_seq[0].isdigit():
 206                 idx = int(array_seq[0])
 207                 if len(resp) <= idx:
 208                     raise BadResponseFormatException(
 209                         "length of array {} is lower than the index {}".format(
 210                             resp, idx))
 211                 _ResponseValidator._validate_array(array_seq[1:], level_next,
 212                                                    resp[idx])
 213             elif array_seq[0] == '*':
 214                 for r in resp:
 215                     _ResponseValidator._validate_array(array_seq[1:],
 216                                                        level_next, r)
 217             elif array_seq[0] == '+':
 218                 if len(resp) < 1:
 219                     raise BadResponseFormatException(
 220                         "array should not be empty")
 221                 for r in resp:
 222                     _ResponseValidator._validate_array(array_seq[1:],
 223                                                        level_next, r)
 224             else:
 225                 raise Exception(
 226                     "Response structure is invalid: only <int> | '*' are "
 227                     "allowed as array index arguments")
 228         else:
 229             if level_next:
 230                 _ResponseValidator._validate_level(level_next, resp)
 231
 232     @staticmethod
 233     def _validate_key(key, level_next, resp):
 234         array_access = [a.strip() for a in key.split("[")]
 235         key = array_access[0]
 236         if key:
 237             optional = key[0] == '?'
 238             if optional:
 239                 key = key[1:]
 240             if key not in resp:
 241                 if optional:
 242                     return
 243                 raise BadResponseFormatException(
 244                     "key {} is not in dict {}".format(key, resp))
 245             resp_next = resp[key]
 246         else:
 247             resp_next = resp
 248         if len(array_access) > 1:
 249             _ResponseValidator._validate_array(
 250                 [a[0:-1] for a in array_access[1:]], level_next, resp_next)
 251         else:
 252             if level_next:
 253                 _ResponseValidator._validate_level(level_next, resp_next)
 254
 255     @staticmethod
 256     def _parse_level_paths(level):
 257         # type: (str) -> List[str]
 258         level = level.strip()
 259         if level[0] == '(':
 260             level = level[1:]
 261             if level[-1] == ')':
 262                 level = level[:-1]
 263
 264         paths = []
 265         lp = 0
 266         nested = 0
 267         for i, c in enumerate(level):
 268             if c == '&' and nested == 0:
 269                 paths.append(level[lp:i].strip())
 270                 lp = i + 1
 271             elif c == '(':
 272                 nested += 1
 273             elif c == ')':
 274                 nested -= 1
 275         paths.append(level[lp:].strip())
 276         return paths
 277
 278
 279 class _Request(object):
 280     def __init__(self, method, path, path_params, rest_client, resp_structure):
 281         self.method = method
 282         self.path = path
 283         self.path_params = path_params
 284         self.rest_client = rest_client
 285         self.resp_structure = resp_structure
 286
 287     def _gen_path(self):
 288         new_path = self.path
 289         matches = re.finditer(r'\{(\w+?)\}', self.path)
 290         for match in matches:
 291             if match:
 292                 param_key = match.group(1)
 293                 if param_key in self.path_params:
 294                     new_path = new_path.replace(
 295                         match.group(0), self.path_params[param_key])
 296                 else:
 297                     raise RequestException(
 298                         'Invalid path. Param "{}" was not specified'
 299                         .format(param_key), None)
 300         return new_path
 301
 302     def __call__(self,
 303                  req_data=None,
 304                  method=None,
 305                  params=None,
 306                  data=None,
 307                  raw_content=False,
 308                  headers=None):
 309         method = method if method else self.method
 310         if not method:
 311             raise Exception('No HTTP request method specified')
 312         if req_data:
 313             if method == 'get':
 314                 if params:
 315                     raise Exception('Ambiguous source of GET params')
 316                 params = req_data
 317             else:
 318                 if data:
 319                     raise Exception('Ambiguous source of {} data'.format(
 320                         method.upper()))
 321                 data = req_data
 322         resp = self.rest_client.do_request(method, self._gen_path(), params,
 323                                            data, raw_content, headers)
 324         if raw_content and self.resp_structure:
 325             raise Exception("Cannot validate response in raw format")
 326         _ResponseValidator.validate(self.resp_structure, resp)
 327         return resp
 328
 329
 330 class RestClient(object):
 331     def __init__(self, host, port, client_name=None, ssl=False, auth=None, ssl_verify=True):
 332         super(RestClient, self).__init__()
 333         self.client_name = client_name if client_name else ''
 334         self.host = host
 335         self.port = port
 336         self.base_url = build_url(
 337             scheme='https' if ssl else 'http', host=host, port=port)
 338         logger.debug("REST service base URL: %s", self.base_url)
 339         self.headers = {'Accept': 'application/json'}
 340         self.auth = auth
 341         self.session = TimeoutRequestsSession()
 342         self.session.verify = ssl_verify
 343
 344     def _login(self, request=None):
 345         pass
 346
 347     def _is_logged_in(self):
 348         pass
 349
 350     def _reset_login(self):
 351         pass
 352
 353     def is_service_online(self, request=None):
 354         pass
 355
 356     @staticmethod
 357     def requires_login(func):
 358         def func_wrapper(self, *args, **kwargs):
 359             retries = 2
 360             while True:
 361                 try:
 362                     if not self._is_logged_in():
 363                         self._login()
 364                     resp = func(self, *args, **kwargs)
 365                     return resp
 366                 except RequestException as e:
 367                     if isinstance(e, BadResponseFormatException):
 368                         raise e
 369                     retries -= 1
 370                     if e.status_code not in [401, 403] or retries == 0:
 371                         raise e
 372                     self._reset_login()
 373
 374         return func_wrapper
 375
 376     def do_request(self,
 377                    method,
 378                    path,
 379                    params=None,
 380                    data=None,
 381                    raw_content=False,
 382                    headers=None):
 383         url = '{}{}'.format(self.base_url, path)
 384         logger.debug('%s REST API %s req: %s data: %s', self.client_name,
 385                      method.upper(), path, data)
 386         request_headers = self.headers.copy()
 387         if headers:
 388             request_headers.update(headers)
 389         try:
 390             if method.lower() == 'get':
 391                 resp = self.session.get(
 392                     url, headers=request_headers, params=params, auth=self.auth)
 393             elif method.lower() == 'post':
 394                 resp = self.session.post(
 395                     url,
 396                     headers=request_headers,
 397                     params=params,
 398                     data=data,
 399                     auth=self.auth)
 400             elif method.lower() == 'put':
 401                 resp = self.session.put(
 402                     url,
 403                     headers=request_headers,
 404                     params=params,
 405                     data=data,
 406                     auth=self.auth)
 407             elif method.lower() == 'delete':
 408                 resp = self.session.delete(
 409                     url,
 410                     headers=request_headers,
 411                     params=params,
 412                     data=data,
 413                     auth=self.auth)
 414             else:
 415                 raise RequestException('Method "{}" not supported'.format(
 416                     method.upper()), None)
 417             if resp.ok:
 418                 logger.debug("%s REST API %s res status: %s content: %s",
 419                              self.client_name, method.upper(),
 420                              resp.status_code, resp.text)
 421                 if raw_content:
 422                     return resp.content
 423                 try:
 424                     return resp.json() if resp.text else None
 425                 except ValueError:
 426                     logger.error(
 427                         "%s REST API failed %s req while decoding JSON "
 428                         "response : %s",
 429                         self.client_name, method.upper(), resp.text)
 430                     raise RequestException(
 431                         "{} REST API failed request while decoding JSON "
 432                         "response: {}".format(self.client_name, resp.text),
 433                         resp.status_code, resp.text)
 434             else:
 435                 logger.error(
 436                     "%s REST API failed %s req status: %s", self.client_name,
 437                     method.upper(), resp.status_code)
 438                 from pprint import pformat as pf
 439
 440                 raise RequestException(
 441                     "{} REST API failed request with status code {}\n"
 442                     "{}"  # TODO remove
 443                     .format(self.client_name, resp.status_code, pf(
 444                         resp.content)),
 445                     resp.status_code,
 446                     resp.content)
 447         except ConnectionError as ex:
 448             if ex.args:
 449                 if isinstance(ex.args[0], SSLError):
 450                     errno = "n/a"
 451                     strerror = "SSL error. Probably trying to access a non " \
 452                                "SSL connection."
 453                     logger.error("%s REST API failed %s, SSL error (url=%s).",
 454                                  self.client_name, method.upper(), ex.request.url)
 455                 else:
 456                     try:
 457                         match = re.match(r'.*: \[Errno (-?\d+)\] (.+)',
 458                                          ex.args[0].reason.args[0])
 459                     except AttributeError:
 460                         match = None
 461                     if match:
 462                         errno = match.group(1)
 463                         strerror = match.group(2)
 464                         logger.error(
 465                             "%s REST API failed %s, connection error (url=%s): "
 466                             "[errno: %s] %s",
 467                             self.client_name, method.upper(), ex.request.url,
 468                             errno, strerror)
 469                     else:
 470                         errno = "n/a"
 471                         strerror = "n/a"
 472                         logger.error(
 473                             "%s REST API failed %s, connection error (url=%s).",
 474                             self.client_name, method.upper(), ex.request.url)
 475             else:
 476                 errno = "n/a"
 477                 strerror = "n/a"
 478                 logger.error("%s REST API failed %s, connection error (url=%s).",
 479                              self.client_name, method.upper(), ex.request.url)
 480
 481             if errno != "n/a":
 482                 ex_msg = (
 483                     "{} REST API cannot be reached: {} [errno {}]. "
 484                     "Please check your configuration and that the API endpoint"
 485                     " is accessible"
 486                     .format(self.client_name, strerror, errno))
 487             else:
 488                 ex_msg = (
 489                     "{} REST API cannot be reached. Please check "
 490                     "your configuration and that the API endpoint is"
 491                     " accessible"
 492                     .format(self.client_name))
 493             raise RequestException(
 494                 ex_msg, conn_errno=errno, conn_strerror=strerror)
 495         except InvalidURL as ex:
 496             logger.exception("%s REST API failed %s: %s", self.client_name,
 497                              method.upper(), str(ex))
 498             raise RequestException(str(ex))
 499         except Timeout as ex:
 500             msg = "{} REST API {} timed out after {} seconds (url={}).".format(
 501                 self.client_name, ex.request.method, Settings.REST_REQUESTS_TIMEOUT,
 502                 ex.request.url)
 503             logger.exception(msg)
 504             raise RequestException(msg)
 505
 506     @staticmethod
 507     def api(path, **api_kwargs):
 508         def call_decorator(func):
 509             def func_wrapper(self, *args, **kwargs):
 510                 method = api_kwargs.get('method', None)
 511                 resp_structure = api_kwargs.get('resp_structure', None)
 512                 args_name = inspect.getargspec(func).args
 513                 args_dict = dict(zip(args_name[1:], args))
 514                 for key, val in kwargs:
 515                     args_dict[key] = val
 516                 return func(
 517                     self,
 518                     *args,
 519                     request=_Request(method, path, args_dict, self,
 520                                      resp_structure),
 521                     **kwargs)
 522
 523             return func_wrapper
 524
 525         return call_decorator
 526
 527     @staticmethod
 528     def api_get(path, resp_structure=None):
 529         return RestClient.api(
 530             path, method='get', resp_structure=resp_structure)
 531
 532     @staticmethod
 533     def api_post(path, resp_structure=None):
 534         return RestClient.api(
 535             path, method='post', resp_structure=resp_structure)
 536
 537     @staticmethod
 538     def api_put(path, resp_structure=None):
 539         return RestClient.api(
 540             path, method='put', resp_structure=resp_structure)
 541
 542     @staticmethod
 543     def api_delete(path, resp_structure=None):
 544         return RestClient.api(
 545             path, method='delete', resp_structure=resp_structure)