ceph/src/pybind/mgr/dashboard/services/rgw_client.py

   1 # -*- coding: utf-8 -*-
   2 from __future__ import absolute_import
   3
   4 import re
   5 from distutils.util import strtobool
   6 from ..awsauth import S3Auth
   7 from ..settings import Settings, Options
   8 from ..rest_client import RestClient, RequestException
   9 from ..tools import build_url, dict_contains_path, is_valid_ip_address
  10 from .. import mgr, logger
  11
  12
  13 class NoCredentialsException(RequestException):
  14     def __init__(self):
  15         super(NoCredentialsException, self).__init__(
  16             'No RGW credentials found, '
  17             'please consult the documentation on how to enable RGW for '
  18             'the dashboard.')
  19
  20
  21 def _determine_rgw_addr():
  22     """
  23     Get a RGW daemon to determine the configured host (IP address) and port.
  24     Note, the service id of the RGW daemons may differ depending on the setup.
  25     Example 1:
  26     {
  27         ...
  28         'services': {
  29             'rgw': {
  30                 'daemons': {
  31                     'summary': '',
  32                     '0': {
  33                         ...
  34                         'addr': '[2001:db8:85a3::8a2e:370:7334]:49774/1534999298',
  35                         'metadata': {
  36                             'frontend_config#0': 'civetweb port=7280',
  37                         }
  38                         ...
  39                     }
  40                 }
  41             }
  42         }
  43     }
  44     Example 2:
  45     {
  46         ...
  47         'services': {
  48             'rgw': {
  49                 'daemons': {
  50                     'summary': '',
  51                     'rgw': {
  52                         ...
  53                         'addr': '192.168.178.3:49774/1534999298',
  54                         'metadata': {
  55                             'frontend_config#0': 'civetweb port=8000',
  56                         }
  57                         ...
  58                     }
  59                 }
  60             }
  61         }
  62     }
  63     """
  64     service_map = mgr.get('service_map')
  65     if not dict_contains_path(service_map, ['services', 'rgw', 'daemons']):
  66         raise LookupError('No RGW found')
  67     daemon = None
  68     daemons = service_map['services']['rgw']['daemons']
  69     for key in daemons.keys():
  70         if dict_contains_path(daemons[key], ['metadata', 'frontend_config#0']):
  71             daemon = daemons[key]
  72             break
  73     if daemon is None:
  74         raise LookupError('No RGW daemon found')
  75
  76     addr = _parse_addr(daemon['addr'])
  77     port, ssl = _parse_frontend_config(daemon['metadata']['frontend_config#0'])
  78
  79     return addr, port, ssl
  80
  81
  82 def _parse_addr(value):
  83     """
  84     Get the IP address the RGW is running on.
  85
  86     >>> _parse_addr('192.168.178.3:49774/1534999298')
  87     '192.168.178.3'
  88
  89     >>> _parse_addr('[2001:db8:85a3::8a2e:370:7334]:49774/1534999298')
  90     '2001:db8:85a3::8a2e:370:7334'
  91
  92     >>> _parse_addr('xyz')
  93     Traceback (most recent call last):
  94     ...
  95     LookupError: Failed to determine RGW address
  96
  97     >>> _parse_addr('192.168.178.a:8080/123456789')
  98     Traceback (most recent call last):
  99     ...
 100     LookupError: Invalid RGW address '192.168.178.a' found
 101
 102     >>> _parse_addr('[2001:0db8:1234]:443/123456789')
 103     Traceback (most recent call last):
 104     ...
 105     LookupError: Invalid RGW address '2001:0db8:1234' found
 106
 107     >>> _parse_addr('2001:0db8::1234:49774/1534999298')
 108     Traceback (most recent call last):
 109     ...
 110     LookupError: Failed to determine RGW address
 111
 112     :param value: The string to process. The syntax is '<HOST>:<PORT>/<NONCE>'.
 113     :type: str
 114     :raises LookupError if parsing fails to determine the IP address.
 115     :return: The IP address.
 116     :rtype: str
 117     """
 118     match = re.search(r'^(\[)?(?(1)([^\]]+)\]|([^:]+)):\d+/\d+?', value)
 119     if match:
 120         # IPv4:
 121         #   Group 0: 192.168.178.3:49774/1534999298
 122         #   Group 3: 192.168.178.3
 123         # IPv6:
 124         #   Group 0: [2001:db8:85a3::8a2e:370:7334]:49774/1534999298
 125         #   Group 1: [
 126         #   Group 2: 2001:db8:85a3::8a2e:370:7334
 127         addr = match.group(3) if match.group(3) else match.group(2)
 128         if not is_valid_ip_address(addr):
 129             raise LookupError('Invalid RGW address \'{}\' found'.format(addr))
 130         return addr
 131     raise LookupError('Failed to determine RGW address')
 132
 133
 134 def _parse_frontend_config(config):
 135     """
 136     Get the port the RGW is running on. Due the complexity of the
 137     syntax not all variations are supported.
 138
 139     Get more details about the configuration syntax here:
 140     http://docs.ceph.com/docs/master/radosgw/frontends/
 141     https://civetweb.github.io/civetweb/UserManual.html
 142
 143     >>> _parse_frontend_config('beast port=8000')
 144     (8000, False)
 145
 146     >>> _parse_frontend_config('civetweb port=8000s')
 147     (8000, True)
 148
 149     >>> _parse_frontend_config('beast port=192.0.2.3:80')
 150     (80, False)
 151
 152     >>> _parse_frontend_config('civetweb port=172.5.2.51:8080s')
 153     (8080, True)
 154
 155     >>> _parse_frontend_config('civetweb port=[::]:8080')
 156     (8080, False)
 157
 158     >>> _parse_frontend_config('civetweb port=ip6-localhost:80s')
 159     (80, True)
 160
 161     >>> _parse_frontend_config('civetweb port=[2001:0db8::1234]:80')
 162     (80, False)
 163
 164     >>> _parse_frontend_config('civetweb port=[::1]:8443s')
 165     (8443, True)
 166
 167     >>> _parse_frontend_config('civetweb port=xyz')
 168     Traceback (most recent call last):
 169     ...
 170     LookupError: Failed to determine RGW port
 171
 172     >>> _parse_frontend_config('civetweb')
 173     Traceback (most recent call last):
 174     ...
 175     LookupError: Failed to determine RGW port
 176
 177     :param config: The configuration string to parse.
 178     :type config: str
 179     :raises LookupError if parsing fails to determine the port.
 180     :return: A tuple containing the port number and the information
 181              whether SSL is used.
 182     :rtype: (int, boolean)
 183     """
 184     match = re.search(r'port=(.*:)?(\d+)(s)?', config)
 185     if match:
 186         port = int(match.group(2))
 187         ssl = match.group(3) == 's'
 188         return port, ssl
 189     raise LookupError('Failed to determine RGW port')
 190
 191
 192 class RgwClient(RestClient):
 193     _SYSTEM_USERID = None
 194     _ADMIN_PATH = None
 195     _host = None
 196     _port = None
 197     _ssl = None
 198     _user_instances = {}
 199
 200     @staticmethod
 201     def _load_settings():
 202         # The API access key and secret key are mandatory for a minimal configuration.
 203         if not (Settings.RGW_API_ACCESS_KEY and Settings.RGW_API_SECRET_KEY):
 204             logger.warning('No credentials found, please consult the '
 205                            'documentation about how to enable RGW for the '
 206                            'dashboard.')
 207             raise NoCredentialsException()
 208
 209         if Options.has_default_value('RGW_API_HOST') and \
 210                 Options.has_default_value('RGW_API_PORT') and \
 211                 Options.has_default_value('RGW_API_SCHEME'):
 212             host, port, ssl = _determine_rgw_addr()
 213         else:
 214             host = Settings.RGW_API_HOST
 215             port = Settings.RGW_API_PORT
 216             ssl = Settings.RGW_API_SCHEME == 'https'
 217
 218         RgwClient._host = host
 219         RgwClient._port = port
 220         RgwClient._ssl = ssl
 221         RgwClient._ADMIN_PATH = Settings.RGW_API_ADMIN_RESOURCE
 222
 223         # Create an instance using the configured settings.
 224         instance = RgwClient(Settings.RGW_API_USER_ID,
 225                              Settings.RGW_API_ACCESS_KEY,
 226                              Settings.RGW_API_SECRET_KEY)
 227
 228         RgwClient._SYSTEM_USERID = instance.userid
 229
 230         # Append the instance to the internal map.
 231         RgwClient._user_instances[RgwClient._SYSTEM_USERID] = instance
 232
 233     @staticmethod
 234     def instance(userid):
 235         if not RgwClient._user_instances:
 236             RgwClient._load_settings()
 237
 238         if not userid:
 239             userid = RgwClient._SYSTEM_USERID
 240
 241         if userid not in RgwClient._user_instances:
 242             # Get the access and secret keys for the specified user.
 243             keys = RgwClient.admin_instance().get_user_keys(userid)
 244             if not keys:
 245                 raise RequestException(
 246                     "User '{}' does not have any keys configured.".format(
 247                         userid))
 248
 249             # Create an instance and append it to the internal map.
 250             RgwClient._user_instances[userid] = RgwClient(userid,
 251                                                           keys['access_key'],
 252                                                           keys['secret_key'])
 253
 254         return RgwClient._user_instances[userid]
 255
 256     @staticmethod
 257     def admin_instance():
 258         return RgwClient.instance(RgwClient._SYSTEM_USERID)
 259
 260     def _reset_login(self):
 261         if self.userid != RgwClient._SYSTEM_USERID:
 262             logger.info("Fetching new keys for user: %s", self.userid)
 263             keys = RgwClient.admin_instance().get_user_keys(self.userid)
 264             self.auth = S3Auth(keys['access_key'], keys['secret_key'],
 265                                service_url=self.service_url)
 266         else:
 267             raise RequestException('Authentication failed for the "{}" user: wrong credentials'
 268                                    .format(self.userid), status_code=401)
 269
 270     def __init__(self,  # pylint: disable-msg=R0913
 271                  userid,
 272                  access_key,
 273                  secret_key,
 274                  host=None,
 275                  port=None,
 276                  admin_path=None,
 277                  ssl=False):
 278
 279         if not host and not RgwClient._host:
 280             RgwClient._load_settings()
 281         host = host if host else RgwClient._host
 282         port = port if port else RgwClient._port
 283         admin_path = admin_path if admin_path else RgwClient._ADMIN_PATH
 284         ssl = ssl if ssl else RgwClient._ssl
 285         ssl_verify = Settings.RGW_API_SSL_VERIFY
 286
 287         self.service_url = build_url(host=host, port=port)
 288         self.admin_path = admin_path
 289
 290         s3auth = S3Auth(access_key, secret_key, service_url=self.service_url)
 291         super(RgwClient, self).__init__(host, port, 'RGW', ssl, s3auth, ssl_verify=ssl_verify)
 292
 293         # If user ID is not set, then try to get it via the RGW Admin Ops API.
 294         self.userid = userid if userid else self._get_user_id(self.admin_path)
 295
 296         logger.info("Created new connection for user: %s", self.userid)
 297
 298     @RestClient.api_get('/', resp_structure='[0] > (ID & DisplayName)')
 299     def is_service_online(self, request=None):
 300         """
 301         Consider the service as online if the response contains the
 302         specified keys. Nothing more is checked here.
 303         """
 304         _ = request({'format': 'json'})
 305         return True
 306
 307     @RestClient.api_get('/{admin_path}/metadata/user?myself',
 308                         resp_structure='data > user_id')
 309     def _get_user_id(self, admin_path, request=None):
 310         # pylint: disable=unused-argument
 311         """
 312         Get the user ID of the user that is used to communicate with the
 313         RGW Admin Ops API.
 314         :rtype: str
 315         :return: The user ID of the user that is used to sign the
 316                  RGW Admin Ops API calls.
 317         """
 318         response = request()
 319         return response['data']['user_id']
 320
 321     @RestClient.api_get('/{admin_path}/metadata/user', resp_structure='[+]')
 322     def _user_exists(self, admin_path, user_id, request=None):
 323         # pylint: disable=unused-argument
 324         response = request()
 325         if user_id:
 326             return user_id in response
 327         return self.userid in response
 328
 329     def user_exists(self, user_id=None):
 330         return self._user_exists(self.admin_path, user_id)
 331
 332     @RestClient.api_get('/{admin_path}/metadata/user?key={userid}',
 333                         resp_structure='data > system')
 334     def _is_system_user(self, admin_path, userid, request=None):
 335         # pylint: disable=unused-argument
 336         response = request()
 337         return strtobool(response['data']['system'])
 338
 339     def is_system_user(self):
 340         return self._is_system_user(self.admin_path, self.userid)
 341
 342     @RestClient.api_get(
 343         '/{admin_path}/user',
 344         resp_structure='tenant & user_id & email & keys[*] > '
 345         ' (user & access_key & secret_key)')
 346     def _admin_get_user_keys(self, admin_path, userid, request=None):
 347         # pylint: disable=unused-argument
 348         colon_idx = userid.find(':')
 349         user = userid if colon_idx == -1 else userid[:colon_idx]
 350         response = request({'uid': user})
 351         for key in response['keys']:
 352             if key['user'] == userid:
 353                 return {
 354                     'access_key': key['access_key'],
 355                     'secret_key': key['secret_key']
 356                 }
 357         return None
 358
 359     def get_user_keys(self, userid):
 360         return self._admin_get_user_keys(self.admin_path, userid)
 361
 362     @RestClient.api('/{admin_path}/{path}')
 363     def _proxy_request(self,  # pylint: disable=too-many-arguments
 364                        admin_path,
 365                        path,
 366                        method,
 367                        params,
 368                        data,
 369                        request=None):
 370         # pylint: disable=unused-argument
 371         return request(
 372             method=method, params=params, data=data, raw_content=True)
 373
 374     def proxy(self, method, path, params, data):
 375         logger.debug("proxying method=%s path=%s params=%s data=%s", method,
 376                      path, params, data)
 377         return self._proxy_request(self.admin_path, path, method, params, data)
 378
 379     @RestClient.api_get('/', resp_structure='[1][*] > Name')
 380     def get_buckets(self, request=None):
 381         """
 382         Get a list of names from all existing buckets of this user.
 383         :return: Returns a list of bucket names.
 384         """
 385         response = request({'format': 'json'})
 386         return [bucket['Name'] for bucket in response[1]]
 387
 388     @RestClient.api_get('/{bucket_name}')
 389     def bucket_exists(self, bucket_name, userid, request=None):
 390         """
 391         Check if the specified bucket exists for this user.
 392         :param bucket_name: The name of the bucket.
 393         :return: Returns True if the bucket exists, otherwise False.
 394         """
 395         # pylint: disable=unused-argument
 396         try:
 397             request()
 398             my_buckets = self.get_buckets()
 399             if bucket_name not in my_buckets:
 400                 raise RequestException(
 401                     'Bucket "{}" belongs to other user'.format(bucket_name),
 402                     403)
 403             return True
 404         except RequestException as e:
 405             if e.status_code == 404:
 406                 return False
 407
 408             raise e
 409
 410     @RestClient.api_put('/{bucket_name}')
 411     def create_bucket(self, bucket_name, request=None):
 412         logger.info("Creating bucket: %s", bucket_name)
 413         return request()