ceph/src/pybind/mgr/dashboard/rest_client.py

   1 # -*- coding: utf-8 -*-
   2 """
   3  *   Copyright (c) 2017 SUSE LLC
   4  *
   5  *  openATTIC is free software; you can redistribute it and/or modify it
   6  *  under the terms of the GNU General Public License as published by
   7  *  the Free Software Foundation; version 2.
   8  *
   9  *  This package is distributed in the hope that it will be useful,
  10  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  *  GNU General Public License for more details.
  13 """
  14
  15 import inspect
  16 import logging
  17 import re
  18
  19 import requests
  20 from requests.auth import AuthBase
  21 from requests.exceptions import ConnectionError, InvalidURL, Timeout
  22
  23 from .settings import Settings
  24
  25 try:
  26     from requests.packages.urllib3.exceptions import SSLError
  27 except ImportError:
  28     from urllib3.exceptions import SSLError  # type: ignore
  29
  30 from typing import List, Optional
  31
  32 from mgr_util import build_url
  33
  34 logger = logging.getLogger('rest_client')
  35
  36
  37 class TimeoutRequestsSession(requests.Session):
  38     """
  39     Set timeout argument for all requests if this is not already done.
  40     """
  41
  42     def request(self, *args, **kwargs):
  43         if ((args[8] if len(args) > 8 else None) is None) \
  44                 and kwargs.get('timeout') is None:
  45             if Settings.REST_REQUESTS_TIMEOUT > 0:
  46                 kwargs['timeout'] = Settings.REST_REQUESTS_TIMEOUT
  47         return super(TimeoutRequestsSession, self).request(*args, **kwargs)
  48
  49
  50 class RequestException(Exception):
  51     def __init__(self,
  52                  message,
  53                  status_code=None,
  54                  content=None,
  55                  conn_errno=None,
  56                  conn_strerror=None):
  57         super(RequestException, self).__init__(message)
  58         self.status_code = status_code
  59         self.content = content
  60         self.conn_errno = conn_errno
  61         self.conn_strerror = conn_strerror
  62
  63
  64 class BadResponseFormatException(RequestException):
  65     def __init__(self, message):
  66         super(BadResponseFormatException, self).__init__(
  67             "Bad response format" if message is None else message, None)
  68
  69
  70 class _ResponseValidator(object):
  71     """Simple JSON schema validator
  72
  73     This class implements a very simple validator for the JSON formatted
  74     messages received by request responses from a RestClient instance.
  75
  76     The validator validates the JSON response against a "structure" string that
  77     specifies the structure that the JSON response must comply.  The validation
  78     procedure raises a BadResponseFormatException in case of a validation
  79     failure.
  80
  81     The structure syntax is given by the following grammar:
  82
  83     Structure  ::=  Level
  84     Level      ::=  Path | Path '&' Level
  85     Path       ::=  Step | Step '>'+ Path
  86     Step       ::=  Key  | '?' Key | '*' | '(' Level ')'
  87     Key        ::=  <string> | Array+
  88     Array      ::=  '[' <int> ']' | '[' '*' ']' | '[' '+' ']'
  89
  90     The symbols enclosed in ' ' are tokens of the language, and the + symbol
  91     denotes repetition of of the preceding token at least once.
  92
  93     Examples of usage:
  94
  95     Example 1:
  96         Validator args:
  97             structure = "return > *"
  98             response = { 'return': { ... } }
  99
 100         In the above example the structure will validate against any response
 101         that contains a key named "return" in the root of the response
 102         dictionary and its value is also a dictionary.
 103
 104     Example 2:
 105         Validator args:
 106             structure = "[*]"
 107             response = [...]
 108
 109         In the above example the structure will validate against any response
 110         that is an array of any size.
 111
 112     Example 3:
 113         Validator args:
 114             structure = "return[*]"
 115             response = { 'return': [....] }
 116
 117         In the above example the structure will validate against any response
 118         that contains a key named "return" in the root of the response
 119         dictionary and its value is an array.
 120
 121     Example 4:
 122         Validator args:
 123             structure = "return[0] > token"
 124             response = { 'return': [ { 'token': .... } ] }
 125
 126         In the above example the structure will validate against any response
 127         that contains a key named "return" in the root of the response
 128         dictionary and its value is an array, and the first element of the
 129         array is a dictionary that contains the key 'token'.
 130
 131     Example 5:
 132         Validator args:
 133             structure = "return[0][*] > key1"
 134             response = { 'return': [ [ { 'key1': ... } ], ...] }
 135
 136         In the above example the structure will validate against any response
 137         that contains a key named "return" in the root of the response
 138         dictionary where its value is an array, and the first value of this
 139         array is also an array where all it's values must be a dictionary
 140         containing a key named "key1".
 141
 142     Example 6:
 143         Validator args:
 144             structure = "return > (key1[*] & key2 & ?key3 > subkey)"
 145             response = { 'return': { 'key1': [...], 'key2: .... } ] }
 146
 147         In the above example the structure will validate against any response
 148         that contains a key named "return" in the root of the response
 149         dictionary and its value is a dictionary that must contain a key named
 150         "key1" that is an array, a key named "key2", and optionally a key named
 151         "key3" that is a dictionary that contains a key named "subkey".
 152
 153     Example 7:
 154         Validator args:
 155             structure = "return >> roles[*]"
 156             response = { 'return': { 'key1': { 'roles': [...] }, 'key2': { 'roles': [...] } } }
 157
 158         In the above example the structure will validate against any response
 159         that contains a key named "return" in the root of the response
 160         dictionary, and its value is a dictionary that for any key present in
 161         the dictionary their value is also a dictionary that must contain a key
 162         named 'roles' that is an array.  Please note that you can use any
 163         number of successive '>' to denote the level in the JSON tree that you
 164         want to match next step in the path.
 165
 166     """
 167
 168     @staticmethod
 169     def validate(structure, response):
 170         if structure is None:
 171             return
 172
 173         _ResponseValidator._validate_level(structure, response)
 174
 175     @staticmethod
 176     def _validate_level(level, resp):
 177         if not isinstance(resp, dict) and not isinstance(resp, list):
 178             raise BadResponseFormatException(
 179                 "{} is neither a dict nor a list".format(resp))
 180
 181         paths = _ResponseValidator._parse_level_paths(level)
 182         for path in paths:
 183             path_sep = path.find('>')
 184             if path_sep != -1:
 185                 level_next = path[path_sep + 1:].strip()
 186             else:
 187                 path_sep = len(path)
 188                 level_next = None  # type: ignore
 189             key = path[:path_sep].strip()
 190
 191             if key == '*':
 192                 continue
 193             elif key == '':  # check all keys
 194                 for k in resp.keys():  # type: ignore
 195                     _ResponseValidator._validate_key(k, level_next, resp)
 196             else:
 197                 _ResponseValidator._validate_key(key, level_next, resp)
 198
 199     @staticmethod
 200     def _validate_array(array_seq, level_next, resp):
 201         if array_seq:
 202             if not isinstance(resp, list):
 203                 raise BadResponseFormatException(
 204                     "{} is not an array".format(resp))
 205             if array_seq[0].isdigit():
 206                 idx = int(array_seq[0])
 207                 if len(resp) <= idx:
 208                     raise BadResponseFormatException(
 209                         "length of array {} is lower than the index {}".format(
 210                             resp, idx))
 211                 _ResponseValidator._validate_array(array_seq[1:], level_next,
 212                                                    resp[idx])
 213             elif array_seq[0] == '*':
 214                 _ResponseValidator.validate_all_resp(resp, array_seq, level_next)
 215             elif array_seq[0] == '+':
 216                 if len(resp) < 1:
 217                     raise BadResponseFormatException(
 218                         "array should not be empty")
 219                 _ResponseValidator.validate_all_resp(resp, array_seq, level_next)
 220             else:
 221                 raise Exception(
 222                     "Response structure is invalid: only <int> | '*' are "
 223                     "allowed as array index arguments")
 224         else:
 225             if level_next:
 226                 _ResponseValidator._validate_level(level_next, resp)
 227
 228     @staticmethod
 229     def validate_all_resp(resp, array_seq, level_next):
 230         for r in resp:
 231             _ResponseValidator._validate_array(array_seq[1:],
 232                                                level_next, r)
 233
 234     @staticmethod
 235     def _validate_key(key, level_next, resp):
 236         array_access = [a.strip() for a in key.split("[")]
 237         key = array_access[0]
 238         if key:
 239             optional = key[0] == '?'
 240             if optional:
 241                 key = key[1:]
 242             if key not in resp:
 243                 if optional:
 244                     return
 245                 raise BadResponseFormatException(
 246                     "key {} is not in dict {}".format(key, resp))
 247             resp_next = resp[key]
 248         else:
 249             resp_next = resp
 250         if len(array_access) > 1:
 251             _ResponseValidator._validate_array(
 252                 [a[0:-1] for a in array_access[1:]], level_next, resp_next)
 253         else:
 254             if level_next:
 255                 _ResponseValidator._validate_level(level_next, resp_next)
 256
 257     @staticmethod
 258     def _parse_level_paths(level):
 259         # type: (str) -> List[str]
 260         level = level.strip()
 261         if level[0] == '(':
 262             level = level[1:]
 263             if level[-1] == ')':
 264                 level = level[:-1]
 265
 266         paths = []
 267         lp = 0
 268         nested = 0
 269         for i, c in enumerate(level):
 270             if c == '&' and nested == 0:
 271                 paths.append(level[lp:i].strip())
 272                 lp = i + 1
 273             elif c == '(':
 274                 nested += 1
 275             elif c == ')':
 276                 nested -= 1
 277         paths.append(level[lp:].strip())
 278         return paths
 279
 280
 281 class _Request(object):
 282     def __init__(self, method, path, path_params, rest_client, resp_structure):
 283         self.method = method
 284         self.path = path
 285         self.path_params = path_params
 286         self.rest_client = rest_client
 287         self.resp_structure = resp_structure
 288
 289     def _gen_path(self):
 290         new_path = self.path
 291         matches = re.finditer(r'\{(\w+?)\}', self.path)
 292         for match in matches:
 293             if match:
 294                 param_key = match.group(1)
 295                 if param_key in self.path_params:
 296                     new_path = new_path.replace(
 297                         match.group(0), self.path_params[param_key])
 298                 else:
 299                     raise RequestException(
 300                         'Invalid path. Param "{}" was not specified'
 301                         .format(param_key), None)
 302         return new_path
 303
 304     def __call__(self,
 305                  req_data=None,
 306                  method=None,
 307                  params=None,
 308                  data=None,
 309                  raw_content=False,
 310                  headers=None):
 311         method = method if method else self.method
 312         if not method:
 313             raise Exception('No HTTP request method specified')
 314         if req_data:
 315             if method == 'get':
 316                 if params:
 317                     raise Exception('Ambiguous source of GET params')
 318                 params = req_data
 319             else:
 320                 if data:
 321                     raise Exception('Ambiguous source of {} data'.format(
 322                         method.upper()))
 323                 data = req_data
 324         resp = self.rest_client.do_request(method, self._gen_path(), params,
 325                                            data, raw_content, headers)
 326         if raw_content and self.resp_structure:
 327             raise Exception("Cannot validate response in raw format")
 328         _ResponseValidator.validate(self.resp_structure, resp)
 329         return resp
 330
 331
 332 class RestClient(object):
 333     def __init__(self,
 334                  host: str,
 335                  port: int,
 336                  client_name: Optional[str] = None,
 337                  ssl: bool = False,
 338                  auth: Optional[AuthBase] = None,
 339                  ssl_verify: bool = True) -> None:
 340         super(RestClient, self).__init__()
 341         self.client_name = client_name if client_name else ''
 342         self.host = host
 343         self.port = port
 344         self.base_url = build_url(
 345             scheme='https' if ssl else 'http', host=host, port=port)
 346         logger.debug("REST service base URL: %s", self.base_url)
 347         self.headers = {'Accept': 'application/json'}
 348         self.auth = auth
 349         self.session = TimeoutRequestsSession()
 350         self.session.verify = ssl_verify
 351
 352     def _login(self, request=None):
 353         pass
 354
 355     def _is_logged_in(self):
 356         pass
 357
 358     def _reset_login(self):
 359         pass
 360
 361     def is_service_online(self, request=None):
 362         pass
 363
 364     @staticmethod
 365     def requires_login(func):
 366         def func_wrapper(self, *args, **kwargs):
 367             retries = 2
 368             while True:
 369                 try:
 370                     if not self._is_logged_in():
 371                         self._login()
 372                     resp = func(self, *args, **kwargs)
 373                     return resp
 374                 except RequestException as e:
 375                     if isinstance(e, BadResponseFormatException):
 376                         raise e
 377                     retries -= 1
 378                     if e.status_code not in [401, 403] or retries == 0:
 379                         raise e
 380                     self._reset_login()
 381
 382         return func_wrapper
 383
 384     def do_request(self,
 385                    method,
 386                    path,
 387                    params=None,
 388                    data=None,
 389                    raw_content=False,
 390                    headers=None):
 391         url = '{}{}'.format(self.base_url, path)
 392         logger.debug('%s REST API %s req: %s data: %s', self.client_name,
 393                      method.upper(), path, data)
 394         request_headers = self.headers.copy()
 395         if headers:
 396             request_headers.update(headers)
 397         try:
 398             resp = self.send_request(method, url, request_headers, params, data)
 399             if resp.ok:
 400                 logger.debug("%s REST API %s res status: %s content: %s",
 401                              self.client_name, method.upper(),
 402                              resp.status_code, resp.text)
 403                 if raw_content:
 404                     return resp.content
 405                 try:
 406                     return resp.json() if resp.text else None
 407                 except ValueError:
 408                     logger.error(
 409                         "%s REST API failed %s req while decoding JSON "
 410                         "response : %s",
 411                         self.client_name, method.upper(), resp.text)
 412                     raise RequestException(
 413                         "{} REST API failed request while decoding JSON "
 414                         "response: {}".format(self.client_name, resp.text),
 415                         resp.status_code, resp.text)
 416             else:
 417                 logger.error(
 418                     "%s REST API failed %s req status: %s", self.client_name,
 419                     method.upper(), resp.status_code)
 420                 from pprint import pformat as pf
 421
 422                 raise RequestException(
 423                     "{} REST API failed request with status code {}\n"
 424                     "{}"  # TODO remove
 425                     .format(self.client_name, resp.status_code, pf(
 426                         resp.content)),
 427                     self._handle_response_status_code(resp.status_code),
 428                     resp.content)
 429         except ConnectionError as ex:
 430             self.handle_connection_error(ex, method)
 431         except InvalidURL as ex:
 432             logger.exception("%s REST API failed %s: %s", self.client_name,
 433                              method.upper(), str(ex))
 434             raise RequestException(str(ex))
 435         except Timeout as ex:
 436             msg = "{} REST API {} timed out after {} seconds (url={}).".format(
 437                 self.client_name, ex.request.method, Settings.REST_REQUESTS_TIMEOUT,
 438                 ex.request.url)
 439             logger.exception(msg)
 440             raise RequestException(msg)
 441
 442     def send_request(self, method, url, request_headers, params, data):
 443         if method.lower() == 'get':
 444             resp = self.session.get(
 445                 url, headers=request_headers, params=params, auth=self.auth)
 446         elif method.lower() == 'post':
 447             resp = self.session.post(
 448                 url,
 449                 headers=request_headers,
 450                 params=params,
 451                 data=data,
 452                 auth=self.auth)
 453         elif method.lower() == 'put':
 454             resp = self.session.put(
 455                 url,
 456                 headers=request_headers,
 457                 params=params,
 458                 data=data,
 459                 auth=self.auth)
 460         elif method.lower() == 'delete':
 461             resp = self.session.delete(
 462                 url,
 463                 headers=request_headers,
 464                 params=params,
 465                 data=data,
 466                 auth=self.auth)
 467         else:
 468             raise RequestException('Method "{}" not supported'.format(
 469                 method.upper()), None)
 470         return resp
 471
 472     def handle_connection_error(self, exception, method):
 473         if exception.args:
 474             if isinstance(exception.args[0], SSLError):
 475                 errno = "n/a"
 476                 strerror = "SSL error. Probably trying to access a non " \
 477                            "SSL connection."
 478                 logger.error("%s REST API failed %s, SSL error (url=%s).",
 479                              self.client_name, method.upper(), exception.request.url)
 480             else:
 481                 try:
 482                     match = re.match(r'.*: \[Errno (-?\d+)\] (.+)',
 483                                      exception.args[0].reason.args[0])
 484                 except AttributeError:
 485                     match = None
 486                 if match:
 487                     errno = match.group(1)
 488                     strerror = match.group(2)
 489                     logger.error(
 490                         "%s REST API failed %s, connection error (url=%s): "
 491                         "[errno: %s] %s",
 492                         self.client_name, method.upper(), exception.request.url,
 493                         errno, strerror)
 494                 else:
 495                     errno = "n/a"
 496                     strerror = "n/a"
 497                     logger.error(
 498                         "%s REST API failed %s, connection error (url=%s).",
 499                         self.client_name, method.upper(), exception.request.url)
 500         else:
 501             errno = "n/a"
 502             strerror = "n/a"
 503             logger.error("%s REST API failed %s, connection error (url=%s).",
 504                          self.client_name, method.upper(), exception.request.url)
 505         if errno != "n/a":
 506             exception_msg = (
 507                 "{} REST API cannot be reached: {} [errno {}]. "
 508                 "Please check your configuration and that the API endpoint"
 509                 " is accessible"
 510                 .format(self.client_name, strerror, errno))
 511         else:
 512             exception_msg = (
 513                 "{} REST API cannot be reached. Please check "
 514                 "your configuration and that the API endpoint is"
 515                 " accessible"
 516                 .format(self.client_name))
 517         raise RequestException(
 518             exception_msg, conn_errno=errno, conn_strerror=strerror)
 519
 520     @staticmethod
 521     def _handle_response_status_code(status_code: int) -> int:
 522         """
 523         Method to be overridden by subclasses that need specific handling.
 524         """
 525         return status_code
 526
 527     @staticmethod
 528     def api(path, **api_kwargs):
 529         def call_decorator(func):
 530             def func_wrapper(self, *args, **kwargs):
 531                 method = api_kwargs.get('method', None)
 532                 resp_structure = api_kwargs.get('resp_structure', None)
 533                 args_name = inspect.getfullargspec(func).args
 534                 args_dict = dict(zip(args_name[1:], args))
 535                 for key, val in kwargs.items():
 536                     args_dict[key] = val
 537                 return func(
 538                     self,
 539                     *args,
 540                     request=_Request(method, path, args_dict, self,
 541                                      resp_structure),
 542                     **kwargs)
 543
 544             return func_wrapper
 545
 546         return call_decorator
 547
 548     @staticmethod
 549     def api_get(path, resp_structure=None):
 550         return RestClient.api(
 551             path, method='get', resp_structure=resp_structure)
 552
 553     @staticmethod
 554     def api_post(path, resp_structure=None):
 555         return RestClient.api(
 556             path, method='post', resp_structure=resp_structure)
 557
 558     @staticmethod
 559     def api_put(path, resp_structure=None):
 560         return RestClient.api(
 561             path, method='put', resp_structure=resp_structure)
 562
 563     @staticmethod
 564     def api_delete(path, resp_structure=None):
 565         return RestClient.api(
 566             path, method='delete', resp_structure=resp_structure)