1 # -*- coding: utf-8 -*-
2 from __future__
import absolute_import
7 from distutils
.util
import strtobool
8 import xml
.etree
.ElementTree
as ET
# noqa: N814
10 from ..awsauth
import S3Auth
11 from ..exceptions
import DashboardException
12 from ..settings
import Settings
, Options
13 from ..rest_client
import RestClient
, RequestException
14 from ..tools
import build_url
, dict_contains_path
, json_str_to_object
,\
15 partial_dict
, dict_get
19 from typing
import Dict
, List
, Optional
# pylint: disable=unused-import
21 pass # For typing only
23 logger
= logging
.getLogger('rgw_client')
26 class NoCredentialsException(RequestException
):
28 super(NoCredentialsException
, self
).__init
__(
29 'No RGW credentials found, '
30 'please consult the documentation on how to enable RGW for '
34 def _determine_rgw_addr():
36 Get a RGW daemon to determine the configured host (IP address) and port.
37 Note, the service id of the RGW daemons may differ depending on the setup.
47 'addr': '[2001:db8:85a3::8a2e:370:7334]:49774/1534999298',
49 'frontend_config#0': 'civetweb port=7280',
66 'addr': '192.168.178.3:49774/1534999298',
68 'frontend_config#0': 'civetweb port=8000',
77 service_map
= mgr
.get('service_map')
78 if not dict_contains_path(service_map
, ['services', 'rgw', 'daemons']):
79 raise LookupError('No RGW found')
81 daemons
= service_map
['services']['rgw']['daemons']
82 for key
in daemons
.keys():
83 if dict_contains_path(daemons
[key
], ['metadata', 'frontend_config#0']):
87 raise LookupError('No RGW daemon found')
89 addr
= _parse_addr(daemon
['addr'])
90 port
, ssl
= _parse_frontend_config(daemon
['metadata']['frontend_config#0'])
92 return addr
, port
, ssl
95 def _parse_addr(value
):
97 Get the IP address the RGW is running on.
99 >>> _parse_addr('192.168.178.3:49774/1534999298')
102 >>> _parse_addr('[2001:db8:85a3::8a2e:370:7334]:49774/1534999298')
103 '2001:db8:85a3::8a2e:370:7334'
105 >>> _parse_addr('xyz')
106 Traceback (most recent call last):
108 LookupError: Failed to determine RGW address
110 >>> _parse_addr('192.168.178.a:8080/123456789')
111 Traceback (most recent call last):
113 LookupError: Invalid RGW address '192.168.178.a' found
115 >>> _parse_addr('[2001:0db8:1234]:443/123456789')
116 Traceback (most recent call last):
118 LookupError: Invalid RGW address '2001:0db8:1234' found
120 >>> _parse_addr('2001:0db8::1234:49774/1534999298')
121 Traceback (most recent call last):
123 LookupError: Failed to determine RGW address
125 :param value: The string to process. The syntax is '<HOST>:<PORT>/<NONCE>'.
127 :raises LookupError if parsing fails to determine the IP address.
128 :return: The IP address.
131 match
= re
.search(r
'^(\[)?(?(1)([^\]]+)\]|([^:]+)):\d+/\d+?', value
)
134 # Group 0: 192.168.178.3:49774/1534999298
135 # Group 3: 192.168.178.3
137 # Group 0: [2001:db8:85a3::8a2e:370:7334]:49774/1534999298
139 # Group 2: 2001:db8:85a3::8a2e:370:7334
140 addr
= match
.group(3) if match
.group(3) else match
.group(2)
142 ipaddress
.ip_address(six
.u(addr
))
145 raise LookupError('Invalid RGW address \'{}\' found'.format(addr
))
146 raise LookupError('Failed to determine RGW address')
149 def _parse_frontend_config(config
):
151 Get the port the RGW is running on. Due the complexity of the
152 syntax not all variations are supported.
154 If there are multiple (ssl_)ports/(ssl_)endpoints options, then
155 the first found option will be returned.
157 Get more details about the configuration syntax here:
158 http://docs.ceph.com/docs/master/radosgw/frontends/
159 https://civetweb.github.io/civetweb/UserManual.html
161 :param config: The configuration string to parse.
163 :raises LookupError if parsing fails to determine the port.
164 :return: A tuple containing the port number and the information
166 :rtype: (int, boolean)
168 match
= re
.search(r
'^(beast|civetweb)\s+.+$', config
)
170 if match
.group(1) == 'beast':
171 match
= re
.search(r
'(port|ssl_port|endpoint|ssl_endpoint)=(.+)',
174 option_name
= match
.group(1)
175 if option_name
in ['port', 'ssl_port']:
176 match
= re
.search(r
'(\d+)', match
.group(2))
178 port
= int(match
.group(1))
179 ssl
= option_name
== 'ssl_port'
181 if option_name
in ['endpoint', 'ssl_endpoint']:
182 match
= re
.search(r
'([\d.]+|\[.+\])(:(\d+))?',
183 match
.group(2)) # type: ignore
185 port
= int(match
.group(3)) if \
186 match
.group(2) is not None else 443 if \
187 option_name
== 'ssl_endpoint' else \
189 ssl
= option_name
== 'ssl_endpoint'
191 if match
.group(1) == 'civetweb': # type: ignore
192 match
= re
.search(r
'port=(.*:)?(\d+)(s)?', config
)
194 port
= int(match
.group(2))
195 ssl
= match
.group(3) == 's'
197 raise LookupError('Failed to determine RGW port from "{}"'.format(config
))
200 class RgwClient(RestClient
):
201 _SYSTEM_USERID
= None
206 _user_instances
= {} # type: Dict[str, RgwClient]
207 _rgw_settings_snapshot
= None
210 def _load_settings():
211 # The API access key and secret key are mandatory for a minimal configuration.
212 if not (Settings
.RGW_API_ACCESS_KEY
and Settings
.RGW_API_SECRET_KEY
):
213 logger
.warning('No credentials found, please consult the '
214 'documentation about how to enable RGW for the '
216 raise NoCredentialsException()
218 if Options
.has_default_value('RGW_API_HOST') and \
219 Options
.has_default_value('RGW_API_PORT') and \
220 Options
.has_default_value('RGW_API_SCHEME'):
221 host
, port
, ssl
= _determine_rgw_addr()
223 host
= Settings
.RGW_API_HOST
224 port
= Settings
.RGW_API_PORT
225 ssl
= Settings
.RGW_API_SCHEME
== 'https'
227 RgwClient
._host
= host
228 RgwClient
._port
= port
230 RgwClient
._ADMIN
_PATH
= Settings
.RGW_API_ADMIN_RESOURCE
232 # Create an instance using the configured settings.
233 instance
= RgwClient(Settings
.RGW_API_USER_ID
,
234 Settings
.RGW_API_ACCESS_KEY
,
235 Settings
.RGW_API_SECRET_KEY
)
237 RgwClient
._SYSTEM
_USERID
= instance
.userid
239 # Append the instance to the internal map.
240 RgwClient
._user
_instances
[RgwClient
._SYSTEM
_USERID
] = instance
242 def _get_daemon_zone_info(self
): # type: () -> dict
243 return json_str_to_object(self
.proxy('GET', 'config?type=zone', None, None))
245 def _get_daemon_zonegroup_map(self
): # type: () -> List[dict]
246 zonegroups
= json_str_to_object(
247 self
.proxy('GET', 'config?type=zonegroup-map', None, None)
250 return [partial_dict(
252 ['api_name', 'zones']
253 ) for zonegroup
in zonegroups
['zonegroups']]
257 return (Settings
.RGW_API_HOST
,
258 Settings
.RGW_API_PORT
,
259 Settings
.RGW_API_ACCESS_KEY
,
260 Settings
.RGW_API_SECRET_KEY
,
261 Settings
.RGW_API_ADMIN_RESOURCE
,
262 Settings
.RGW_API_SCHEME
,
263 Settings
.RGW_API_USER_ID
,
264 Settings
.RGW_API_SSL_VERIFY
)
267 def instance(userid
):
268 # type: (Optional[str]) -> RgwClient
269 # Discard all cached instances if any rgw setting has changed
270 if RgwClient
._rgw
_settings
_snapshot
!= RgwClient
._rgw
_settings
():
271 RgwClient
._rgw
_settings
_snapshot
= RgwClient
._rgw
_settings
()
272 RgwClient
._user
_instances
.clear()
274 if not RgwClient
._user
_instances
:
275 RgwClient
._load
_settings
()
278 userid
= RgwClient
._SYSTEM
_USERID
280 if userid
not in RgwClient
._user
_instances
:
281 # Get the access and secret keys for the specified user.
282 keys
= RgwClient
.admin_instance().get_user_keys(userid
)
284 raise RequestException(
285 "User '{}' does not have any keys configured.".format(
288 # Create an instance and append it to the internal map.
289 RgwClient
._user
_instances
[userid
] = RgwClient(userid
, # type: ignore
293 return RgwClient
._user
_instances
[userid
] # type: ignore
296 def admin_instance():
297 return RgwClient
.instance(RgwClient
._SYSTEM
_USERID
)
299 def _reset_login(self
):
300 if self
.userid
!= RgwClient
._SYSTEM
_USERID
:
301 logger
.info("Fetching new keys for user: %s", self
.userid
)
302 keys
= RgwClient
.admin_instance().get_user_keys(self
.userid
)
303 self
.auth
= S3Auth(keys
['access_key'], keys
['secret_key'],
304 service_url
=self
.service_url
)
306 raise RequestException('Authentication failed for the "{}" user: wrong credentials'
307 .format(self
.userid
), status_code
=401)
309 def __init__(self
, # pylint: disable-msg=R0913
318 if not host
and not RgwClient
._host
:
319 RgwClient
._load
_settings
()
320 host
= host
if host
else RgwClient
._host
321 port
= port
if port
else RgwClient
._port
322 admin_path
= admin_path
if admin_path
else RgwClient
._ADMIN
_PATH
323 ssl
= ssl
if ssl
else RgwClient
._ssl
324 ssl_verify
= Settings
.RGW_API_SSL_VERIFY
326 self
.service_url
= build_url(host
=host
, port
=port
)
327 self
.admin_path
= admin_path
329 s3auth
= S3Auth(access_key
, secret_key
, service_url
=self
.service_url
)
330 super(RgwClient
, self
).__init
__(host
, port
, 'RGW', ssl
, s3auth
, ssl_verify
=ssl_verify
)
332 # If user ID is not set, then try to get it via the RGW Admin Ops API.
333 self
.userid
= userid
if userid
else self
._get
_user
_id
(self
.admin_path
) # type: str
335 logger
.info("Created new connection: user=%s, host=%s, port=%s, ssl=%d, sslverify=%d",
336 self
.userid
, host
, port
, ssl
, ssl_verify
)
338 @RestClient.api_get('/', resp_structure
='[0] > (ID & DisplayName)')
339 def is_service_online(self
, request
=None):
341 Consider the service as online if the response contains the
342 specified keys. Nothing more is checked here.
344 _
= request({'format': 'json'})
347 @RestClient.api_get('/{admin_path}/metadata/user?myself',
348 resp_structure
='data > user_id')
349 def _get_user_id(self
, admin_path
, request
=None):
350 # pylint: disable=unused-argument
352 Get the user ID of the user that is used to communicate with the
355 :return: The user ID of the user that is used to sign the
356 RGW Admin Ops API calls.
359 return response
['data']['user_id']
361 @RestClient.api_get('/{admin_path}/metadata/user', resp_structure
='[+]')
362 def _user_exists(self
, admin_path
, user_id
, request
=None):
363 # pylint: disable=unused-argument
366 return user_id
in response
367 return self
.userid
in response
369 def user_exists(self
, user_id
=None):
370 return self
._user
_exists
(self
.admin_path
, user_id
)
372 @RestClient.api_get('/{admin_path}/metadata/user?key={userid}',
373 resp_structure
='data > system')
374 def _is_system_user(self
, admin_path
, userid
, request
=None):
375 # pylint: disable=unused-argument
377 return strtobool(response
['data']['system'])
379 def is_system_user(self
):
380 return self
._is
_system
_user
(self
.admin_path
, self
.userid
)
383 '/{admin_path}/user',
384 resp_structure
='tenant & user_id & email & keys[*] > '
385 ' (user & access_key & secret_key)')
386 def _admin_get_user_keys(self
, admin_path
, userid
, request
=None):
387 # pylint: disable=unused-argument
388 colon_idx
= userid
.find(':')
389 user
= userid
if colon_idx
== -1 else userid
[:colon_idx
]
390 response
= request({'uid': user
})
391 for key
in response
['keys']:
392 if key
['user'] == userid
:
394 'access_key': key
['access_key'],
395 'secret_key': key
['secret_key']
399 def get_user_keys(self
, userid
):
400 return self
._admin
_get
_user
_keys
(self
.admin_path
, userid
)
402 @RestClient.api('/{admin_path}/{path}')
404 self
, # pylint: disable=too-many-arguments
411 # pylint: disable=unused-argument
412 return request(method
=method
, params
=params
, data
=data
,
415 def proxy(self
, method
, path
, params
, data
):
416 logger
.debug("proxying method=%s path=%s params=%s data=%s",
417 method
, path
, params
, data
)
418 return self
._proxy
_request
(self
.admin_path
, path
, method
,
421 @RestClient.api_get('/', resp_structure
='[1][*] > Name')
422 def get_buckets(self
, request
=None):
424 Get a list of names from all existing buckets of this user.
425 :return: Returns a list of bucket names.
427 response
= request({'format': 'json'})
428 return [bucket
['Name'] for bucket
in response
[1]]
430 @RestClient.api_get('/{bucket_name}')
431 def bucket_exists(self
, bucket_name
, userid
, request
=None):
433 Check if the specified bucket exists for this user.
434 :param bucket_name: The name of the bucket.
435 :return: Returns True if the bucket exists, otherwise False.
437 # pylint: disable=unused-argument
440 my_buckets
= self
.get_buckets()
441 if bucket_name
not in my_buckets
:
442 raise RequestException(
443 'Bucket "{}" belongs to other user'.format(bucket_name
),
446 except RequestException
as e
:
447 if e
.status_code
== 404:
452 @RestClient.api_put('/{bucket_name}')
453 def create_bucket(self
, bucket_name
, zonegroup
=None,
454 placement_target
=None, lock_enabled
=False,
456 logger
.info("Creating bucket: %s, zonegroup: %s, placement_target: %s",
457 bucket_name
, zonegroup
, placement_target
)
459 if zonegroup
and placement_target
:
460 create_bucket_configuration
= ET
.Element('CreateBucketConfiguration')
461 location_constraint
= ET
.SubElement(create_bucket_configuration
, 'LocationConstraint')
462 location_constraint
.text
= '{}:{}'.format(zonegroup
, placement_target
)
463 data
= ET
.tostring(create_bucket_configuration
, encoding
='unicode')
465 headers
= None # type: Optional[dict]
467 headers
= {'x-amz-bucket-object-lock-enabled': 'true'}
469 return request(data
=data
, headers
=headers
)
471 def get_placement_targets(self
): # type: () -> dict
472 zone
= self
._get
_daemon
_zone
_info
()
473 # A zone without realm id can only belong to default zonegroup.
474 zonegroup_name
= 'default'
476 zonegroup_map
= self
._get
_daemon
_zonegroup
_map
()
477 for zonegroup
in zonegroup_map
:
478 for realm_zone
in zonegroup
['zones']:
479 if realm_zone
['id'] == zone
['id']:
480 zonegroup_name
= zonegroup
['api_name']
483 placement_targets
= [] # type: List[Dict]
484 for placement_pool
in zone
['placement_pools']:
485 placement_targets
.append(
487 'name': placement_pool
['key'],
488 'data_pool': placement_pool
['val']['storage_classes']['STANDARD']['data_pool']
492 return {'zonegroup': zonegroup_name
, 'placement_targets': placement_targets
}
494 @RestClient.api_get('/{bucket_name}?versioning')
495 def get_bucket_versioning(self
, bucket_name
, request
=None):
497 Get bucket versioning.
498 :param str bucket_name: the name of the bucket.
499 :return: versioning info
502 # pylint: disable=unused-argument
504 if 'Status' not in result
:
505 result
['Status'] = 'Suspended'
506 if 'MfaDelete' not in result
:
507 result
['MfaDelete'] = 'Disabled'
510 @RestClient.api_put('/{bucket_name}?versioning')
511 def set_bucket_versioning(self
, bucket_name
, versioning_state
, mfa_delete
,
512 mfa_token_serial
, mfa_token_pin
, request
=None):
514 Set bucket versioning.
515 :param str bucket_name: the name of the bucket.
516 :param str versioning_state:
517 https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTVersioningStatus.html
518 :param str mfa_delete: MFA Delete state.
519 :param str mfa_token_serial:
520 https://docs.ceph.com/docs/master/radosgw/mfa/
521 :param str mfa_token_pin: value of a TOTP token at a certain time (auth code)
524 # pylint: disable=unused-argument
525 versioning_configuration
= ET
.Element('VersioningConfiguration')
526 status_element
= ET
.SubElement(versioning_configuration
, 'Status')
527 status_element
.text
= versioning_state
530 if mfa_delete
and mfa_token_serial
and mfa_token_pin
:
531 headers
['x-amz-mfa'] = '{} {}'.format(mfa_token_serial
, mfa_token_pin
)
532 mfa_delete_element
= ET
.SubElement(versioning_configuration
, 'MfaDelete')
533 mfa_delete_element
.text
= mfa_delete
535 data
= ET
.tostring(versioning_configuration
, encoding
='unicode')
538 request(data
=data
, headers
=headers
)
539 except RequestException
as error
:
541 if error
.status_code
== 403:
542 msg
= 'Bad MFA credentials: {}'.format(msg
)
543 # Avoid dashboard GUI redirections caused by status code (403, ...):
544 http_status_code
= 400 if 400 <= error
.status_code
< 500 else error
.status_code
545 raise DashboardException(msg
=msg
,
546 http_status_code
=http_status_code
,
549 @RestClient.api_get('/{bucket_name}?object-lock')
550 def get_bucket_locking(self
, bucket_name
, request
=None):
551 # type: (str, Optional[object]) -> dict
553 Gets the locking configuration for a bucket. The locking
554 configuration will be applied by default to every new object
555 placed in the specified bucket.
556 :param bucket_name: The name of the bucket.
557 :type bucket_name: str
558 :return: The locking configuration.
561 # pylint: disable=unused-argument
563 # Try to get the Object Lock configuration. If there is none,
564 # then return default values.
566 result
= request() # type: ignore
568 'lock_enabled': dict_get(result
, 'ObjectLockEnabled') == 'Enabled',
569 'lock_mode': dict_get(result
, 'Rule.DefaultRetention.Mode'),
570 'lock_retention_period_days': dict_get(result
, 'Rule.DefaultRetention.Days', 0),
571 'lock_retention_period_years': dict_get(result
, 'Rule.DefaultRetention.Years', 0)
573 except RequestException
as e
:
575 content
= json_str_to_object(e
.content
)
577 'Code') == 'ObjectLockConfigurationNotFoundError':
579 'lock_enabled': False,
580 'lock_mode': 'compliance',
581 'lock_retention_period_days': None,
582 'lock_retention_period_years': None
586 @RestClient.api_put('/{bucket_name}?object-lock')
587 def set_bucket_locking(self
,
590 retention_period_days
,
591 retention_period_years
,
593 # type: (str, str, int, int, Optional[object]) -> None
595 Places the locking configuration on the specified bucket. The
596 locking configuration will be applied by default to every new
597 object placed in the specified bucket.
598 :param bucket_name: The name of the bucket.
599 :type bucket_name: str
600 :param mode: The lock mode, e.g. `COMPLIANCE` or `GOVERNANCE`.
602 :param retention_period_days:
603 :type retention_period_days: int
604 :param retention_period_years:
605 :type retention_period_years: int
608 # pylint: disable=unused-argument
610 # Do some validations.
611 if retention_period_days
and retention_period_years
:
612 # https://docs.aws.amazon.com/AmazonS3/latest/API/archive-RESTBucketPUTObjectLockConfiguration.html
613 msg
= "Retention period requires either Days or Years. "\
614 "You can't specify both at the same time."
615 raise DashboardException(msg
=msg
, component
='rgw')
616 if not retention_period_days
and not retention_period_years
:
617 msg
= "Retention period requires either Days or Years. "\
618 "You must specify at least one."
619 raise DashboardException(msg
=msg
, component
='rgw')
621 # Generate the XML data like this:
622 # <ObjectLockConfiguration>
623 # <ObjectLockEnabled>string</ObjectLockEnabled>
626 # <Days>integer</Days>
627 # <Mode>string</Mode>
628 # <Years>integer</Years>
629 # </DefaultRetention>
631 # </ObjectLockConfiguration>
632 locking_configuration
= ET
.Element('ObjectLockConfiguration')
633 enabled_element
= ET
.SubElement(locking_configuration
,
635 enabled_element
.text
= 'Enabled' # Locking can't be disabled.
636 rule_element
= ET
.SubElement(locking_configuration
, 'Rule')
637 default_retention_element
= ET
.SubElement(rule_element
,
639 mode_element
= ET
.SubElement(default_retention_element
, 'Mode')
640 mode_element
.text
= mode
.upper()
641 if retention_period_days
:
642 days_element
= ET
.SubElement(default_retention_element
, 'Days')
643 days_element
.text
= str(retention_period_days
)
644 if retention_period_years
:
645 years_element
= ET
.SubElement(default_retention_element
, 'Years')
646 years_element
.text
= str(retention_period_years
)
648 data
= ET
.tostring(locking_configuration
, encoding
='unicode')
651 _
= request(data
=data
) # type: ignore
652 except RequestException
as e
:
653 raise DashboardException(msg
=str(e
), component
='rgw')