1 # -*- coding: utf-8 -*-
2 from __future__
import absolute_import
7 from distutils
.util
import strtobool
8 import xml
.etree
.ElementTree
as ET
# noqa: N814
10 from ..awsauth
import S3Auth
11 from ..exceptions
import DashboardException
12 from ..settings
import Settings
, Options
13 from ..rest_client
import RestClient
, RequestException
14 from ..tools
import build_url
, dict_contains_path
, json_str_to_object
,\
19 from typing
import Any
, Dict
, List
, Optional
, Tuple
# pylint: disable=unused-import
21 pass # For typing only
23 logger
= logging
.getLogger('rgw_client')
26 class NoCredentialsException(RequestException
):
28 super(NoCredentialsException
, self
).__init
__(
29 'No RGW credentials found, '
30 'please consult the documentation on how to enable RGW for '
34 def _get_daemon_info() -> Dict
[str, Any
]:
36 Retrieve RGW daemon info from MGR.
37 Note, the service id of the RGW daemons may differ depending on the setup.
47 'addr': '[2001:db8:85a3::8a2e:370:7334]:49774/1534999298',
49 'frontend_config#0': 'civetweb port=7280',
66 'addr': '192.168.178.3:49774/1534999298',
68 'frontend_config#0': 'civetweb port=8000',
77 service_map
= mgr
.get('service_map')
78 if not dict_contains_path(service_map
, ['services', 'rgw', 'daemons']):
79 raise LookupError('No RGW found')
81 daemons
= service_map
['services']['rgw']['daemons']
82 for key
in daemons
.keys():
83 if dict_contains_path(daemons
[key
], ['metadata', 'frontend_config#0']):
87 raise LookupError('No RGW daemon found')
92 def _determine_rgw_addr() -> Tuple
[str, int, bool]:
94 Parse RGW daemon info to determine the configured host (IP address) and port.
96 daemon
= _get_daemon_info()
97 addr
= _parse_addr(daemon
['addr'])
98 port
, ssl
= _parse_frontend_config(daemon
['metadata']['frontend_config#0'])
100 logger
.info('Auto-detected RGW configuration: addr=%s, port=%d, ssl=%s',
101 addr
, port
, str(ssl
))
103 return addr
, port
, ssl
106 def _parse_addr(value
) -> str:
108 Get the IP address the RGW is running on.
110 >>> _parse_addr('192.168.178.3:49774/1534999298')
113 >>> _parse_addr('[2001:db8:85a3::8a2e:370:7334]:49774/1534999298')
114 '2001:db8:85a3::8a2e:370:7334'
116 >>> _parse_addr('xyz')
117 Traceback (most recent call last):
119 LookupError: Failed to determine RGW address
121 >>> _parse_addr('192.168.178.a:8080/123456789')
122 Traceback (most recent call last):
124 LookupError: Invalid RGW address '192.168.178.a' found
126 >>> _parse_addr('[2001:0db8:1234]:443/123456789')
127 Traceback (most recent call last):
129 LookupError: Invalid RGW address '2001:0db8:1234' found
131 >>> _parse_addr('2001:0db8::1234:49774/1534999298')
132 Traceback (most recent call last):
134 LookupError: Failed to determine RGW address
136 :param value: The string to process. The syntax is '<HOST>:<PORT>/<NONCE>'.
138 :raises LookupError if parsing fails to determine the IP address.
139 :return: The IP address.
142 match
= re
.search(r
'^(\[)?(?(1)([^\]]+)\]|([^:]+)):\d+/\d+?', value
)
145 # Group 0: 192.168.178.3:49774/1534999298
146 # Group 3: 192.168.178.3
148 # Group 0: [2001:db8:85a3::8a2e:370:7334]:49774/1534999298
150 # Group 2: 2001:db8:85a3::8a2e:370:7334
151 addr
= match
.group(3) if match
.group(3) else match
.group(2)
153 ipaddress
.ip_address(six
.u(addr
))
156 raise LookupError('Invalid RGW address \'{}\' found'.format(addr
))
157 raise LookupError('Failed to determine RGW address')
160 def _parse_frontend_config(config
) -> Tuple
[int, bool]:
162 Get the port the RGW is running on. Due the complexity of the
163 syntax not all variations are supported.
165 If there are multiple (ssl_)ports/(ssl_)endpoints options, then
166 the first found option will be returned.
168 Get more details about the configuration syntax here:
169 http://docs.ceph.com/docs/master/radosgw/frontends/
170 https://civetweb.github.io/civetweb/UserManual.html
172 :param config: The configuration string to parse.
174 :raises LookupError if parsing fails to determine the port.
175 :return: A tuple containing the port number and the information
177 :rtype: (int, boolean)
179 match
= re
.search(r
'^(beast|civetweb)\s+.+$', config
)
181 if match
.group(1) == 'beast':
182 match
= re
.search(r
'(port|ssl_port|endpoint|ssl_endpoint)=(.+)',
185 option_name
= match
.group(1)
186 if option_name
in ['port', 'ssl_port']:
187 match
= re
.search(r
'(\d+)', match
.group(2))
189 port
= int(match
.group(1))
190 ssl
= option_name
== 'ssl_port'
192 if option_name
in ['endpoint', 'ssl_endpoint']:
193 match
= re
.search(r
'([\d.]+|\[.+\])(:(\d+))?',
194 match
.group(2)) # type: ignore
196 port
= int(match
.group(3)) if \
197 match
.group(2) is not None else 443 if \
198 option_name
== 'ssl_endpoint' else \
200 ssl
= option_name
== 'ssl_endpoint'
202 if match
.group(1) == 'civetweb': # type: ignore
203 match
= re
.search(r
'port=(.*:)?(\d+)(s)?', config
)
205 port
= int(match
.group(2))
206 ssl
= match
.group(3) == 's'
208 raise LookupError('Failed to determine RGW port from "{}"'.format(config
))
211 class RgwClient(RestClient
):
212 _SYSTEM_USERID
= None
217 _user_instances
= {} # type: Dict[str, RgwClient]
218 _rgw_settings_snapshot
= None
221 def _load_settings():
222 # The API access key and secret key are mandatory for a minimal configuration.
223 if not (Settings
.RGW_API_ACCESS_KEY
and Settings
.RGW_API_SECRET_KEY
):
224 logger
.warning('No credentials found, please consult the '
225 'documentation about how to enable RGW for the '
227 raise NoCredentialsException()
229 if Options
.has_default_value('RGW_API_HOST') and \
230 Options
.has_default_value('RGW_API_PORT') and \
231 Options
.has_default_value('RGW_API_SCHEME'):
232 host
, port
, ssl
= _determine_rgw_addr()
234 host
= Settings
.RGW_API_HOST
235 port
= Settings
.RGW_API_PORT
236 ssl
= Settings
.RGW_API_SCHEME
== 'https'
238 RgwClient
._host
= host
239 RgwClient
._port
= port
241 RgwClient
._ADMIN
_PATH
= Settings
.RGW_API_ADMIN_RESOURCE
243 # Create an instance using the configured settings.
244 instance
= RgwClient(Settings
.RGW_API_USER_ID
,
245 Settings
.RGW_API_ACCESS_KEY
,
246 Settings
.RGW_API_SECRET_KEY
)
248 RgwClient
._SYSTEM
_USERID
= instance
.userid
250 # Append the instance to the internal map.
251 RgwClient
._user
_instances
[RgwClient
._SYSTEM
_USERID
] = instance
253 def _get_daemon_zone_info(self
): # type: () -> dict
254 return json_str_to_object(self
.proxy('GET', 'config?type=zone', None, None))
256 def _get_realms_info(self
): # type: () -> dict
257 return json_str_to_object(self
.proxy('GET', 'realm?list', None, None))
261 return (Settings
.RGW_API_HOST
,
262 Settings
.RGW_API_PORT
,
263 Settings
.RGW_API_ACCESS_KEY
,
264 Settings
.RGW_API_SECRET_KEY
,
265 Settings
.RGW_API_ADMIN_RESOURCE
,
266 Settings
.RGW_API_SCHEME
,
267 Settings
.RGW_API_USER_ID
,
268 Settings
.RGW_API_SSL_VERIFY
)
271 def instance(userid
):
272 # type: (Optional[str]) -> RgwClient
273 # Discard all cached instances if any rgw setting has changed
274 if RgwClient
._rgw
_settings
_snapshot
!= RgwClient
._rgw
_settings
():
275 RgwClient
._rgw
_settings
_snapshot
= RgwClient
._rgw
_settings
()
276 RgwClient
.drop_instance()
278 if not RgwClient
._user
_instances
:
279 RgwClient
._load
_settings
()
282 userid
= RgwClient
._SYSTEM
_USERID
284 if userid
not in RgwClient
._user
_instances
:
285 # Get the access and secret keys for the specified user.
286 keys
= RgwClient
.admin_instance().get_user_keys(userid
)
288 raise RequestException(
289 "User '{}' does not have any keys configured.".format(
292 # Create an instance and append it to the internal map.
293 RgwClient
._user
_instances
[userid
] = RgwClient(userid
, # type: ignore
297 return RgwClient
._user
_instances
[userid
] # type: ignore
300 def admin_instance():
301 return RgwClient
.instance(RgwClient
._SYSTEM
_USERID
)
304 def drop_instance(userid
: Optional
[str] = None):
306 Drop a cached instance by name or all.
309 RgwClient
._user
_instances
.pop(userid
, None)
311 RgwClient
._user
_instances
.clear()
313 def _reset_login(self
):
314 if self
.userid
!= RgwClient
._SYSTEM
_USERID
:
315 logger
.info("Fetching new keys for user: %s", self
.userid
)
316 keys
= RgwClient
.admin_instance().get_user_keys(self
.userid
)
317 self
.auth
= S3Auth(keys
['access_key'], keys
['secret_key'],
318 service_url
=self
.service_url
)
320 raise RequestException('Authentication failed for the "{}" user: wrong credentials'
321 .format(self
.userid
), status_code
=401)
323 def __init__(self
, # pylint: disable-msg=R0913
332 if not host
and not RgwClient
._host
:
333 RgwClient
._load
_settings
()
334 host
= host
if host
else RgwClient
._host
335 port
= port
if port
else RgwClient
._port
336 admin_path
= admin_path
if admin_path
else RgwClient
._ADMIN
_PATH
337 ssl
= ssl
if ssl
else RgwClient
._ssl
338 ssl_verify
= Settings
.RGW_API_SSL_VERIFY
340 self
.service_url
= build_url(host
=host
, port
=port
)
341 self
.admin_path
= admin_path
343 s3auth
= S3Auth(access_key
, secret_key
, service_url
=self
.service_url
)
344 super(RgwClient
, self
).__init
__(host
, port
, 'RGW', ssl
, s3auth
, ssl_verify
=ssl_verify
)
346 # If user ID is not set, then try to get it via the RGW Admin Ops API.
347 self
.userid
= userid
if userid
else self
._get
_user
_id
(self
.admin_path
) # type: str
349 self
._zonegroup
_name
: str = _get_daemon_info()['metadata']['zonegroup_name']
351 logger
.info("Created new connection: user=%s, host=%s, port=%s, ssl=%d, sslverify=%d",
352 self
.userid
, host
, port
, ssl
, ssl_verify
)
354 @RestClient.api_get('/', resp_structure
='[0] > (ID & DisplayName)')
355 def is_service_online(self
, request
=None):
357 Consider the service as online if the response contains the
358 specified keys. Nothing more is checked here.
360 _
= request({'format': 'json'})
363 @RestClient.api_get('/{admin_path}/metadata/user?myself',
364 resp_structure
='data > user_id')
365 def _get_user_id(self
, admin_path
, request
=None):
366 # pylint: disable=unused-argument
368 Get the user ID of the user that is used to communicate with the
371 :return: The user ID of the user that is used to sign the
372 RGW Admin Ops API calls.
375 return response
['data']['user_id']
377 @RestClient.api_get('/{admin_path}/metadata/user', resp_structure
='[+]')
378 def _user_exists(self
, admin_path
, user_id
, request
=None):
379 # pylint: disable=unused-argument
382 return user_id
in response
383 return self
.userid
in response
385 def user_exists(self
, user_id
=None):
386 return self
._user
_exists
(self
.admin_path
, user_id
)
388 @RestClient.api_get('/{admin_path}/metadata/user?key={userid}',
389 resp_structure
='data > system')
390 def _is_system_user(self
, admin_path
, userid
, request
=None):
391 # pylint: disable=unused-argument
393 return strtobool(response
['data']['system'])
395 def is_system_user(self
):
396 return self
._is
_system
_user
(self
.admin_path
, self
.userid
)
399 '/{admin_path}/user',
400 resp_structure
='tenant & user_id & email & keys[*] > '
401 ' (user & access_key & secret_key)')
402 def _admin_get_user_keys(self
, admin_path
, userid
, request
=None):
403 # pylint: disable=unused-argument
404 colon_idx
= userid
.find(':')
405 user
= userid
if colon_idx
== -1 else userid
[:colon_idx
]
406 response
= request({'uid': user
})
407 for key
in response
['keys']:
408 if key
['user'] == userid
:
410 'access_key': key
['access_key'],
411 'secret_key': key
['secret_key']
415 def get_user_keys(self
, userid
):
416 return self
._admin
_get
_user
_keys
(self
.admin_path
, userid
)
418 @RestClient.api('/{admin_path}/{path}')
420 self
, # pylint: disable=too-many-arguments
427 # pylint: disable=unused-argument
428 return request(method
=method
, params
=params
, data
=data
,
431 def proxy(self
, method
, path
, params
, data
):
432 logger
.debug("proxying method=%s path=%s params=%s data=%s",
433 method
, path
, params
, data
)
434 return self
._proxy
_request
(self
.admin_path
, path
, method
,
437 @RestClient.api_get('/', resp_structure
='[1][*] > Name')
438 def get_buckets(self
, request
=None):
440 Get a list of names from all existing buckets of this user.
441 :return: Returns a list of bucket names.
443 response
= request({'format': 'json'})
444 return [bucket
['Name'] for bucket
in response
[1]]
446 @RestClient.api_get('/{bucket_name}')
447 def bucket_exists(self
, bucket_name
, userid
, request
=None):
449 Check if the specified bucket exists for this user.
450 :param bucket_name: The name of the bucket.
451 :return: Returns True if the bucket exists, otherwise False.
453 # pylint: disable=unused-argument
456 my_buckets
= self
.get_buckets()
457 if bucket_name
not in my_buckets
:
458 raise RequestException(
459 'Bucket "{}" belongs to other user'.format(bucket_name
),
462 except RequestException
as e
:
463 if e
.status_code
== 404:
468 @RestClient.api_put('/{bucket_name}')
469 def create_bucket(self
, bucket_name
, zonegroup
=None,
470 placement_target
=None, lock_enabled
=False,
472 logger
.info("Creating bucket: %s, zonegroup: %s, placement_target: %s",
473 bucket_name
, zonegroup
, placement_target
)
475 if zonegroup
and placement_target
:
476 create_bucket_configuration
= ET
.Element('CreateBucketConfiguration')
477 location_constraint
= ET
.SubElement(create_bucket_configuration
, 'LocationConstraint')
478 location_constraint
.text
= '{}:{}'.format(zonegroup
, placement_target
)
479 data
= ET
.tostring(create_bucket_configuration
, encoding
='unicode')
481 headers
= None # type: Optional[dict]
483 headers
= {'x-amz-bucket-object-lock-enabled': 'true'}
485 return request(data
=data
, headers
=headers
)
487 def get_placement_targets(self
): # type: () -> dict
488 zone
= self
._get
_daemon
_zone
_info
()
489 placement_targets
= [] # type: List[Dict]
490 for placement_pool
in zone
['placement_pools']:
491 placement_targets
.append(
493 'name': placement_pool
['key'],
494 'data_pool': placement_pool
['val']['storage_classes']['STANDARD']['data_pool']
498 return {'zonegroup': self
._zonegroup
_name
, 'placement_targets': placement_targets
}
500 def get_realms(self
): # type: () -> List
501 realms_info
= self
._get
_realms
_info
()
502 if 'realms' in realms_info
and realms_info
['realms']:
503 return realms_info
['realms']
507 @RestClient.api_get('/{bucket_name}?versioning')
508 def get_bucket_versioning(self
, bucket_name
, request
=None):
510 Get bucket versioning.
511 :param str bucket_name: the name of the bucket.
512 :return: versioning info
515 # pylint: disable=unused-argument
517 if 'Status' not in result
:
518 result
['Status'] = 'Suspended'
519 if 'MfaDelete' not in result
:
520 result
['MfaDelete'] = 'Disabled'
523 @RestClient.api_put('/{bucket_name}?versioning')
524 def set_bucket_versioning(self
, bucket_name
, versioning_state
, mfa_delete
,
525 mfa_token_serial
, mfa_token_pin
, request
=None):
527 Set bucket versioning.
528 :param str bucket_name: the name of the bucket.
529 :param str versioning_state:
530 https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTVersioningStatus.html
531 :param str mfa_delete: MFA Delete state.
532 :param str mfa_token_serial:
533 https://docs.ceph.com/docs/master/radosgw/mfa/
534 :param str mfa_token_pin: value of a TOTP token at a certain time (auth code)
537 # pylint: disable=unused-argument
538 versioning_configuration
= ET
.Element('VersioningConfiguration')
539 status_element
= ET
.SubElement(versioning_configuration
, 'Status')
540 status_element
.text
= versioning_state
543 if mfa_delete
and mfa_token_serial
and mfa_token_pin
:
544 headers
['x-amz-mfa'] = '{} {}'.format(mfa_token_serial
, mfa_token_pin
)
545 mfa_delete_element
= ET
.SubElement(versioning_configuration
, 'MfaDelete')
546 mfa_delete_element
.text
= mfa_delete
548 data
= ET
.tostring(versioning_configuration
, encoding
='unicode')
551 request(data
=data
, headers
=headers
)
552 except RequestException
as error
:
554 if error
.status_code
== 403:
555 msg
= 'Bad MFA credentials: {}'.format(msg
)
556 # Avoid dashboard GUI redirections caused by status code (403, ...):
557 http_status_code
= 400 if 400 <= error
.status_code
< 500 else error
.status_code
558 raise DashboardException(msg
=msg
,
559 http_status_code
=http_status_code
,
562 @RestClient.api_get('/{bucket_name}?object-lock')
563 def get_bucket_locking(self
, bucket_name
, request
=None):
564 # type: (str, Optional[object]) -> dict
566 Gets the locking configuration for a bucket. The locking
567 configuration will be applied by default to every new object
568 placed in the specified bucket.
569 :param bucket_name: The name of the bucket.
570 :type bucket_name: str
571 :return: The locking configuration.
574 # pylint: disable=unused-argument
576 # Try to get the Object Lock configuration. If there is none,
577 # then return default values.
579 result
= request() # type: ignore
581 'lock_enabled': dict_get(result
, 'ObjectLockEnabled') == 'Enabled',
582 'lock_mode': dict_get(result
, 'Rule.DefaultRetention.Mode'),
583 'lock_retention_period_days': dict_get(result
, 'Rule.DefaultRetention.Days', 0),
584 'lock_retention_period_years': dict_get(result
, 'Rule.DefaultRetention.Years', 0)
586 except RequestException
as e
:
588 content
= json_str_to_object(e
.content
)
590 'Code') == 'ObjectLockConfigurationNotFoundError':
592 'lock_enabled': False,
593 'lock_mode': 'compliance',
594 'lock_retention_period_days': None,
595 'lock_retention_period_years': None
599 @RestClient.api_put('/{bucket_name}?object-lock')
600 def set_bucket_locking(self
,
603 retention_period_days
,
604 retention_period_years
,
606 # type: (str, str, int, int, Optional[object]) -> None
608 Places the locking configuration on the specified bucket. The
609 locking configuration will be applied by default to every new
610 object placed in the specified bucket.
611 :param bucket_name: The name of the bucket.
612 :type bucket_name: str
613 :param mode: The lock mode, e.g. `COMPLIANCE` or `GOVERNANCE`.
615 :param retention_period_days:
616 :type retention_period_days: int
617 :param retention_period_years:
618 :type retention_period_years: int
621 # pylint: disable=unused-argument
623 # Do some validations.
624 if retention_period_days
and retention_period_years
:
625 # https://docs.aws.amazon.com/AmazonS3/latest/API/archive-RESTBucketPUTObjectLockConfiguration.html
626 msg
= "Retention period requires either Days or Years. "\
627 "You can't specify both at the same time."
628 raise DashboardException(msg
=msg
, component
='rgw')
629 if not retention_period_days
and not retention_period_years
:
630 msg
= "Retention period requires either Days or Years. "\
631 "You must specify at least one."
632 raise DashboardException(msg
=msg
, component
='rgw')
634 # Generate the XML data like this:
635 # <ObjectLockConfiguration>
636 # <ObjectLockEnabled>string</ObjectLockEnabled>
639 # <Days>integer</Days>
640 # <Mode>string</Mode>
641 # <Years>integer</Years>
642 # </DefaultRetention>
644 # </ObjectLockConfiguration>
645 locking_configuration
= ET
.Element('ObjectLockConfiguration')
646 enabled_element
= ET
.SubElement(locking_configuration
,
648 enabled_element
.text
= 'Enabled' # Locking can't be disabled.
649 rule_element
= ET
.SubElement(locking_configuration
, 'Rule')
650 default_retention_element
= ET
.SubElement(rule_element
,
652 mode_element
= ET
.SubElement(default_retention_element
, 'Mode')
653 mode_element
.text
= mode
.upper()
654 if retention_period_days
:
655 days_element
= ET
.SubElement(default_retention_element
, 'Days')
656 days_element
.text
= str(retention_period_days
)
657 if retention_period_years
:
658 years_element
= ET
.SubElement(default_retention_element
, 'Years')
659 years_element
.text
= str(retention_period_years
)
661 data
= ET
.tostring(locking_configuration
, encoding
='unicode')
664 _
= request(data
=data
) # type: ignore
665 except RequestException
as e
:
666 raise DashboardException(msg
=str(e
), component
='rgw')