ceph/src/pybind/mgr/dashboard/rest_client.py

   1 # -*- coding: utf-8 -*-
   2 """
   3  *   Copyright (c) 2017 SUSE LLC
   4  *
   5  *  openATTIC is free software; you can redistribute it and/or modify it
   6  *  under the terms of the GNU General Public License as published by
   7  *  the Free Software Foundation; version 2.
   8  *
   9  *  This package is distributed in the hope that it will be useful,
  10  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  *  GNU General Public License for more details.
  13 """
  14 from __future__ import absolute_import
  15
  16 from .settings import Settings
  17 from .tools import build_url
  18 import inspect
  19 import re
  20 import requests
  21 from requests.exceptions import ConnectionError, InvalidURL, Timeout
  22 from . import logger
  23
  24 try:
  25     from requests.packages.urllib3.exceptions import SSLError
  26 except ImportError:
  27     from urllib3.exceptions import SSLError
  28
  29
  30 class TimeoutRequestsSession(requests.Session):
  31     """
  32     Set timeout argument for all requests if this is not already done.
  33     """
  34     def request(self, *args, **kwargs):
  35         if ((args[8] if len(args) > 8 else None) is None) \
  36                 and kwargs.get('timeout') is None:
  37             if Settings.REST_REQUESTS_TIMEOUT > 0:
  38                 kwargs['timeout'] = Settings.REST_REQUESTS_TIMEOUT
  39         return super(TimeoutRequestsSession, self).request(*args, **kwargs)
  40
  41
  42 class RequestException(Exception):
  43     def __init__(self,
  44                  message,
  45                  status_code=None,
  46                  content=None,
  47                  conn_errno=None,
  48                  conn_strerror=None):
  49         super(RequestException, self).__init__(message)
  50         self.status_code = status_code
  51         self.content = content
  52         self.conn_errno = conn_errno
  53         self.conn_strerror = conn_strerror
  54
  55
  56 class BadResponseFormatException(RequestException):
  57     def __init__(self, message):
  58         super(BadResponseFormatException, self).__init__(
  59             "Bad response format" if message is None else message, None)
  60
  61
  62 class _ResponseValidator(object):
  63     """Simple JSON schema validator
  64
  65     This class implements a very simple validator for the JSON formatted
  66     messages received by request responses from a RestClient instance.
  67
  68     The validator validates the JSON response against a "structure" string that
  69     specifies the structure that the JSON response must comply.  The validation
  70     procedure raises a BadResponseFormatException in case of a validation
  71     failure.
  72
  73     The structure syntax is given by the following grammar:
  74
  75     Structure  ::=  Level
  76     Level      ::=  Path | Path '&' Level
  77     Path       ::=  Step | Step '>'+ Path
  78     Step       ::=  Key  | '?' Key | '*' | '(' Level ')'
  79     Key        ::=  <string> | Array+
  80     Array      ::=  '[' <int> ']' | '[' '*' ']' | '[' '+' ']'
  81
  82     The symbols enclosed in ' ' are tokens of the language, and the + symbol
  83     denotes repetition of of the preceding token at least once.
  84
  85     Examples of usage:
  86
  87     Example 1:
  88         Validator args:
  89             structure = "return > *"
  90             response = { 'return': { ... } }
  91
  92         In the above example the structure will validate against any response
  93         that contains a key named "return" in the root of the response
  94         dictionary and its value is also a dictionary.
  95
  96     Example 2:
  97         Validator args:
  98             structure = "[*]"
  99             response = [...]
 100
 101         In the above example the structure will validate against any response
 102         that is an array of any size.
 103
 104     Example 3:
 105         Validator args:
 106             structure = "return[*]"
 107             response = { 'return': [....] }
 108
 109         In the above example the structure will validate against any response
 110         that contains a key named "return" in the root of the response
 111         dictionary and its value is an array.
 112
 113     Example 4:
 114         Validator args:
 115             structure = "return[0] > token"
 116             response = { 'return': [ { 'token': .... } ] }
 117
 118         In the above example the structure will validate against any response
 119         that contains a key named "return" in the root of the response
 120         dictionary and its value is an array, and the first element of the
 121         array is a dictionary that contains the key 'token'.
 122
 123     Example 5:
 124         Validator args:
 125             structure = "return[0][*] > key1"
 126             response = { 'return': [ [ { 'key1': ... } ], ...] }
 127
 128         In the above example the structure will validate against any response
 129         that contains a key named "return" in the root of the response
 130         dictionary where its value is an array, and the first value of this
 131         array is also an array where all it's values must be a dictionary
 132         containing a key named "key1".
 133
 134     Example 6:
 135         Validator args:
 136             structure = "return > (key1[*] & key2 & ?key3 > subkey)"
 137             response = { 'return': { 'key1': [...], 'key2: .... } ] }
 138
 139         In the above example the structure will validate against any response
 140         that contains a key named "return" in the root of the response
 141         dictionary and its value is a dictionary that must contain a key named
 142         "key1" that is an array, a key named "key2", and optionally a key named
 143         "key3" that is a dictionary that contains a key named "subkey".
 144
 145     Example 7:
 146         Validator args:
 147             structure = "return >> roles[*]"
 148             response = { 'return': { 'key1': { 'roles': [...] }, 'key2': { 'roles': [...] } } }
 149
 150         In the above example the structure will validate against any response
 151         that contains a key named "return" in the root of the response
 152         dictionary, and its value is a dictionary that for any key present in
 153         the dictionary their value is also a dictionary that must contain a key
 154         named 'roles' that is an array.  Please note that you can use any
 155         number of successive '>' to denote the level in the JSON tree that you
 156         want to match next step in the path.
 157
 158     """
 159
 160     @staticmethod
 161     def validate(structure, response):
 162         if structure is None:
 163             return
 164
 165         _ResponseValidator._validate_level(structure, response)
 166
 167     @staticmethod
 168     def _validate_level(level, resp):
 169         if not isinstance(resp, dict) and not isinstance(resp, list):
 170             raise BadResponseFormatException(
 171                 "{} is neither a dict nor a list".format(resp))
 172
 173         paths = _ResponseValidator._parse_level_paths(level)
 174         for path in paths:
 175             path_sep = path.find('>')
 176             if path_sep != -1:
 177                 level_next = path[path_sep + 1:].strip()
 178             else:
 179                 path_sep = len(path)
 180                 level_next = None
 181             key = path[:path_sep].strip()
 182
 183             if key == '*':
 184                 continue
 185             elif key == '':  # check all keys
 186                 for k in resp.keys():
 187                     _ResponseValidator._validate_key(k, level_next, resp)
 188             else:
 189                 _ResponseValidator._validate_key(key, level_next, resp)
 190
 191     @staticmethod
 192     def _validate_array(array_seq, level_next, resp):
 193         if array_seq:
 194             if not isinstance(resp, list):
 195                 raise BadResponseFormatException(
 196                     "{} is not an array".format(resp))
 197             if array_seq[0].isdigit():
 198                 idx = int(array_seq[0])
 199                 if len(resp) <= idx:
 200                     raise BadResponseFormatException(
 201                         "length of array {} is lower than the index {}".format(
 202                             resp, idx))
 203                 _ResponseValidator._validate_array(array_seq[1:], level_next,
 204                                                    resp[idx])
 205             elif array_seq[0] == '*':
 206                 for r in resp:
 207                     _ResponseValidator._validate_array(array_seq[1:],
 208                                                        level_next, r)
 209             elif array_seq[0] == '+':
 210                 if len(resp) < 1:
 211                     raise BadResponseFormatException(
 212                         "array should not be empty")
 213                 for r in resp:
 214                     _ResponseValidator._validate_array(array_seq[1:],
 215                                                        level_next, r)
 216             else:
 217                 raise Exception(
 218                     "Response structure is invalid: only <int> | '*' are "
 219                     "allowed as array index arguments")
 220         else:
 221             if level_next:
 222                 _ResponseValidator._validate_level(level_next, resp)
 223
 224     @staticmethod
 225     def _validate_key(key, level_next, resp):
 226         array_access = [a.strip() for a in key.split("[")]
 227         key = array_access[0]
 228         if key:
 229             optional = key[0] == '?'
 230             if optional:
 231                 key = key[1:]
 232             if key not in resp:
 233                 if optional:
 234                     return
 235                 raise BadResponseFormatException(
 236                     "key {} is not in dict {}".format(key, resp))
 237             resp_next = resp[key]
 238         else:
 239             resp_next = resp
 240         if len(array_access) > 1:
 241             _ResponseValidator._validate_array(
 242                 [a[0:-1] for a in array_access[1:]], level_next, resp_next)
 243         else:
 244             if level_next:
 245                 _ResponseValidator._validate_level(level_next, resp_next)
 246
 247     @staticmethod
 248     def _parse_level_paths(level):
 249         level = level.strip()
 250         if level[0] == '(':
 251             level = level[1:]
 252             if level[-1] == ')':
 253                 level = level[:-1]
 254
 255         paths = []
 256         lp = 0
 257         nested = 0
 258         for i, c in enumerate(level):
 259             if c == '&' and nested == 0:
 260                 paths.append(level[lp:i].strip())
 261                 lp = i + 1
 262             elif c == '(':
 263                 nested += 1
 264             elif c == ')':
 265                 nested -= 1
 266         paths.append(level[lp:].strip())
 267         return paths
 268
 269
 270 class _Request(object):
 271     def __init__(self, method, path, path_params, rest_client, resp_structure):
 272         self.method = method
 273         self.path = path
 274         self.path_params = path_params
 275         self.rest_client = rest_client
 276         self.resp_structure = resp_structure
 277
 278     def _gen_path(self):
 279         new_path = self.path
 280         matches = re.finditer(r'\{(\w+?)\}', self.path)
 281         for match in matches:
 282             if match:
 283                 param_key = match.group(1)
 284                 if param_key in self.path_params:
 285                     new_path = new_path.replace(
 286                         match.group(0), self.path_params[param_key])
 287                 else:
 288                     raise RequestException(
 289                         'Invalid path. Param "{}" was not specified'
 290                         .format(param_key), None)
 291         return new_path
 292
 293     def __call__(self,
 294                  req_data=None,
 295                  method=None,
 296                  params=None,
 297                  data=None,
 298                  raw_content=False):
 299         method = method if method else self.method
 300         if not method:
 301             raise Exception('No HTTP request method specified')
 302         if req_data:
 303             if method == 'get':
 304                 if params:
 305                     raise Exception('Ambiguous source of GET params')
 306                 params = req_data
 307             else:
 308                 if data:
 309                     raise Exception('Ambiguous source of {} data'.format(
 310                         method.upper()))
 311                 data = req_data
 312         resp = self.rest_client.do_request(method, self._gen_path(), params,
 313                                            data, raw_content)
 314         if raw_content and self.resp_structure:
 315             raise Exception("Cannot validate response in raw format")
 316         _ResponseValidator.validate(self.resp_structure, resp)
 317         return resp
 318
 319
 320 class RestClient(object):
 321     def __init__(self, host, port, client_name=None, ssl=False, auth=None, ssl_verify=True):
 322         super(RestClient, self).__init__()
 323         self.client_name = client_name if client_name else ''
 324         self.host = host
 325         self.port = port
 326         self.base_url = build_url(
 327             scheme='https' if ssl else 'http', host=host, port=port)
 328         logger.debug("REST service base URL: %s", self.base_url)
 329         self.headers = {'Accept': 'application/json'}
 330         self.auth = auth
 331         self.session = TimeoutRequestsSession()
 332         self.session.verify = ssl_verify
 333
 334     def _login(self, request=None):
 335         pass
 336
 337     def _is_logged_in(self):
 338         pass
 339
 340     def _reset_login(self):
 341         pass
 342
 343     def is_service_online(self, request=None):
 344         pass
 345
 346     @staticmethod
 347     def requires_login(func):
 348         def func_wrapper(self, *args, **kwargs):
 349             retries = 2
 350             while True:
 351                 try:
 352                     if not self._is_logged_in():
 353                         self._login()
 354                     resp = func(self, *args, **kwargs)
 355                     return resp
 356                 except RequestException as e:
 357                     if isinstance(e, BadResponseFormatException):
 358                         raise e
 359                     retries -= 1
 360                     if e.status_code not in [401, 403] or retries == 0:
 361                         raise e
 362                     self._reset_login()
 363
 364         return func_wrapper
 365
 366     def do_request(self,
 367                    method,
 368                    path,
 369                    params=None,
 370                    data=None,
 371                    raw_content=False):
 372         url = '{}{}'.format(self.base_url, path)
 373         logger.debug('%s REST API %s req: %s data: %s', self.client_name,
 374                      method.upper(), path, data)
 375         try:
 376             if method.lower() == 'get':
 377                 resp = self.session.get(
 378                     url, headers=self.headers, params=params, auth=self.auth)
 379             elif method.lower() == 'post':
 380                 resp = self.session.post(
 381                     url,
 382                     headers=self.headers,
 383                     params=params,
 384                     data=data,
 385                     auth=self.auth)
 386             elif method.lower() == 'put':
 387                 resp = self.session.put(
 388                     url,
 389                     headers=self.headers,
 390                     params=params,
 391                     data=data,
 392                     auth=self.auth)
 393             elif method.lower() == 'delete':
 394                 resp = self.session.delete(
 395                     url,
 396                     headers=self.headers,
 397                     params=params,
 398                     data=data,
 399                     auth=self.auth)
 400             else:
 401                 raise RequestException('Method "{}" not supported'.format(
 402                     method.upper()), None)
 403             if resp.ok:
 404                 logger.debug("%s REST API %s res status: %s content: %s",
 405                              self.client_name, method.upper(),
 406                              resp.status_code, resp.text)
 407                 if raw_content:
 408                     return resp.content
 409                 try:
 410                     return resp.json() if resp.text else None
 411                 except ValueError:
 412                     logger.error(
 413                         "%s REST API failed %s req while decoding JSON "
 414                         "response : %s",
 415                         self.client_name, method.upper(), resp.text)
 416                     raise RequestException(
 417                         "{} REST API failed request while decoding JSON "
 418                         "response: {}".format(self.client_name, resp.text),
 419                         resp.status_code, resp.text)
 420             else:
 421                 logger.error(
 422                     "%s REST API failed %s req status: %s", self.client_name,
 423                     method.upper(), resp.status_code)
 424                 from pprint import pprint as pp
 425                 from pprint import pformat as pf
 426
 427                 raise RequestException(
 428                     "{} REST API failed request with status code {}\n"
 429                     "{}"  # TODO remove
 430                     .format(self.client_name, resp.status_code, pf(
 431                         resp.content)),
 432                     resp.status_code,
 433                     resp.content)
 434         except ConnectionError as ex:
 435             if ex.args:
 436                 if isinstance(ex.args[0], SSLError):
 437                     errno = "n/a"
 438                     strerror = "SSL error. Probably trying to access a non " \
 439                                "SSL connection."
 440                     logger.error("%s REST API failed %s, SSL error.",
 441                                  self.client_name, method.upper())
 442                 else:
 443                     try:
 444                         match = re.match(r'.*: \[Errno (-?\d+)\] (.+)',
 445                                          ex.args[0].reason.args[0])
 446                     except AttributeError:
 447                         match = False
 448                     if match:
 449                         errno = match.group(1)
 450                         strerror = match.group(2)
 451                         logger.error(
 452                             "%s REST API failed %s, connection error: "
 453                             "[errno: %s] %s",
 454                             self.client_name, method.upper(), errno, strerror)
 455                     else:
 456                         errno = "n/a"
 457                         strerror = "n/a"
 458                         logger.error(
 459                             "%s REST API failed %s, connection error.",
 460                             self.client_name, method.upper())
 461             else:
 462                 errno = "n/a"
 463                 strerror = "n/a"
 464                 logger.error("%s REST API failed %s, connection error.",
 465                              self.client_name, method.upper())
 466
 467             if errno != "n/a":
 468                 ex_msg = (
 469                     "{} REST API cannot be reached: {} [errno {}]. "
 470                     "Please check your configuration and that the API endpoint"
 471                     " is accessible"
 472                     .format(self.client_name, strerror, errno))
 473             else:
 474                 ex_msg = (
 475                     "{} REST API cannot be reached. Please check "
 476                     "your configuration and that the API endpoint is"
 477                     " accessible"
 478                     .format(self.client_name))
 479             raise RequestException(
 480                 ex_msg, conn_errno=errno, conn_strerror=strerror)
 481         except InvalidURL as ex:
 482             logger.exception("%s REST API failed %s: %s", self.client_name,
 483                              method.upper(), str(ex))
 484             raise RequestException(str(ex))
 485         except Timeout as ex:
 486             msg = "{} REST API {} timed out after {} seconds (url={}).".format(
 487                 self.client_name, ex.request.method, Settings.REST_REQUESTS_TIMEOUT,
 488                 ex.request.url)
 489             logger.exception(msg)
 490             raise RequestException(msg)
 491
 492     @staticmethod
 493     def api(path, **api_kwargs):
 494         def call_decorator(func):
 495             def func_wrapper(self, *args, **kwargs):
 496                 method = api_kwargs.get('method', None)
 497                 resp_structure = api_kwargs.get('resp_structure', None)
 498                 args_name = inspect.getargspec(func).args
 499                 args_dict = dict(zip(args_name[1:], args))
 500                 for key, val in kwargs:
 501                     args_dict[key] = val
 502                 return func(
 503                     self,
 504                     *args,
 505                     request=_Request(method, path, args_dict, self,
 506                                      resp_structure),
 507                     **kwargs)
 508
 509             return func_wrapper
 510
 511         return call_decorator
 512
 513     @staticmethod
 514     def api_get(path, resp_structure=None):
 515         return RestClient.api(
 516             path, method='get', resp_structure=resp_structure)
 517
 518     @staticmethod
 519     def api_post(path, resp_structure=None):
 520         return RestClient.api(
 521             path, method='post', resp_structure=resp_structure)
 522
 523     @staticmethod
 524     def api_put(path, resp_structure=None):
 525         return RestClient.api(
 526             path, method='put', resp_structure=resp_structure)
 527
 528     @staticmethod
 529     def api_delete(path, resp_structure=None):
 530         return RestClient.api(
 531             path, method='delete', resp_structure=resp_structure)