1 # -*- coding: utf-8 -*-
5 from collections
import Counter
6 from typing
import Dict
, List
, Optional
9 from mgr_util
import merge_dicts
12 from ..exceptions
import DashboardException
13 from ..plugins
.ttl_cache
import ttl_cache
, ttl_cache_invalidator
14 from ..security
import Scope
15 from ..services
._paginate
import ListPaginator
16 from ..services
.ceph_service
import CephService
17 from ..services
.exception
import handle_orchestrator_error
18 from ..services
.orchestrator
import OrchClient
, OrchFeature
19 from ..tools
import TaskManager
, merge_list_of_dicts_by_key
, str_to_bool
20 from . import APIDoc
, APIRouter
, BaseController
, Endpoint
, EndpointDoc
, \
21 ReadPermission
, RESTController
, Task
, UIRouter
, UpdatePermission
, \
23 from ._version
import APIVersion
24 from .orchestrator
import raise_if_no_orchestrator
27 "hostname": (str, "Hostname"),
29 "type": (str, "type of service"),
30 "id": (str, "Service Id"),
31 }], "Services related to the host"),
32 "service_instances": ([{
33 "type": (str, "type of service"),
34 "count": (int, "Number of instances of the service"),
35 }], "Service instances related to the host"),
36 "ceph_version": (str, "Ceph version"),
37 "addr": (str, "Host address"),
38 "labels": ([str], "Labels related to the host"),
39 "service_type": (str, ""),
42 "orchestrator": (bool, "")
48 "name": (str, "Hostname"),
49 "addr": (str, "Host address"),
51 "rejected_reasons": ([str], ""),
52 "available": (bool, "If the device can be provisioned to an OSD"),
53 "path": (str, "Device path"),
55 "removable": (str, ""),
60 "sas_address": (str, ""),
61 "sas_device_handle": (str, ""),
62 "support_discard": (str, ""),
63 "rotational": (str, ""),
64 "nr_requests": (str, ""),
65 "scheduler_mode": (str, ""),
70 "sectorsize": (int, ""),
72 "human_readable_size": (str, ""),
73 "holders": ([str], "")
77 "sectorsize": (str, ""),
79 "human_readable_size": (str, ""),
86 "cluster_name": (str, ""),
88 "osd_fsid": (str, ""),
89 "cluster_fsid": (str, ""),
90 "osdspec_affinity": (str, ""),
91 "block_uuid": (str, ""),
93 "human_readable_type": (str, "Device type. ssd or hdd"),
94 "device_id": (str, "Device's udev ID"),
96 "serialNum": (str, ""),
97 "transport": (str, ""),
98 "mediaType": (str, ""),
100 "linkSpeed": (str, ""),
103 "IDENTsupport": (str, ""),
104 "IDENTstatus": (str, ""),
105 "FAILsupport": (str, ""),
106 "FAILstatus": (str, ""),
108 "errors": ([str], "")
110 "osd_ids": ([int], "Device OSD IDs")
112 "labels": ([str], "Host labels")
116 def host_task(name
, metadata
, wait_for
=10.0):
117 return Task("host/{}".format(name
), metadata
, wait_for
)
120 def populate_service_instances(hostname
, services
):
121 orch
= OrchClient
.instance()
123 services
= (daemon
['daemon_type']
124 for daemon
in (d
.to_dict()
125 for d
in orch
.services
.list_daemons(hostname
=hostname
)))
127 services
= (daemon
['type'] for daemon
in services
)
128 return [{'type': k
, 'count': v
} for k
, v
in Counter(services
).items()]
131 @ttl_cache(60, label
='get_hosts')
132 def get_hosts(sources
=None):
134 Get hosts from various sources.
137 from_orchestrator
= True
139 _sources
= sources
.split(',')
140 from_ceph
= 'ceph' in _sources
141 from_orchestrator
= 'orchestrator' in _sources
143 if from_orchestrator
:
144 orch
= OrchClient
.instance()
155 }, host
.to_json()) for host
in orch
.hosts
.list()
168 'orchestrator': False
171 }) for server
in mgr
.list_servers()
176 def get_host(hostname
: str) -> Dict
:
178 Get a specific host from Ceph or Orchestrator (if available).
179 :param hostname: The name of the host to fetch.
180 :raises: cherrypy.HTTPError: If host not found.
182 for host
in get_hosts():
183 if host
['hostname'] == hostname
:
185 raise cherrypy
.HTTPError(404)
188 def get_device_osd_map():
189 """Get mappings from inventory devices to OSD IDs.
191 :return: Returns a dictionary containing mappings. Note one device might
192 shared between multiple OSDs.
206 for osd_id
, osd_metadata
in mgr
.get('osd_metadata').items():
207 hostname
= osd_metadata
.get('hostname')
208 devices
= osd_metadata
.get('devices')
209 if not hostname
or not devices
:
211 if hostname
not in result
:
212 result
[hostname
] = {}
213 # for OSD contains multiple devices, devices is in `sda,sdb`
214 for device
in devices
.split(','):
215 if device
not in result
[hostname
]:
216 result
[hostname
][device
] = [int(osd_id
)]
218 result
[hostname
][device
].append(int(osd_id
))
222 def get_inventories(hosts
: Optional
[List
[str]] = None,
223 refresh
: Optional
[bool] = None) -> List
[dict]:
224 """Get inventories from the Orchestrator and link devices with OSD IDs.
226 :param hosts: Hostnames to query.
227 :param refresh: Ask the Orchestrator to refresh the inventories. Note the this is an
228 asynchronous operation, the updated version of inventories need to
230 :return: Returns list of inventory.
234 if refresh
is not None:
235 do_refresh
= str_to_bool(refresh
)
236 orch
= OrchClient
.instance()
237 inventory_hosts
= [host
.to_json()
238 for host
in orch
.inventory
.list(hosts
=hosts
, refresh
=do_refresh
)]
239 device_osd_map
= get_device_osd_map()
240 for inventory_host
in inventory_hosts
:
241 host_osds
= device_osd_map
.get(inventory_host
['name'])
242 for device
in inventory_host
['devices']:
243 if host_osds
: # pragma: no cover
244 dev_name
= os
.path
.basename(device
['path'])
245 device
['osd_ids'] = sorted(host_osds
.get(dev_name
, []))
247 device
['osd_ids'] = []
248 return inventory_hosts
252 def add_host(hostname
: str, addr
: Optional
[str] = None,
253 labels
: Optional
[List
[str]] = None,
254 status
: Optional
[str] = None):
255 orch_client
= OrchClient
.instance()
257 host
.check_orchestrator_host_op(orch_client
, hostname
)
258 orch_client
.hosts
.add(hostname
, addr
, labels
)
259 if status
== 'maintenance':
260 orch_client
.hosts
.enter_maintenance(hostname
)
263 @APIRouter('/host', Scope
.HOSTS
)
264 @APIDoc("Get Host Details", "Host")
265 class Host(RESTController
):
266 @EndpointDoc("List Host Specifications",
268 'sources': (str, 'Host Sources'),
269 'facts': (bool, 'Host Facts')
271 responses
={200: LIST_HOST_SCHEMA
})
272 @RESTController.MethodMap(version
=APIVersion(1, 3))
273 def list(self
, sources
=None, facts
=False, offset
: int = 0,
274 limit
: int = 5, search
: str = '', sort
: str = ''):
275 hosts
= get_hosts(sources
)
276 params
= ['hostname']
277 paginator
= ListPaginator(int(offset
), int(limit
), sort
, search
, hosts
,
278 searchable_params
=params
, sortable_params
=params
,
279 default_sort
='+hostname')
280 # pylint: disable=unnecessary-comprehension
281 hosts
= [host
for host
in paginator
.list()]
282 orch
= OrchClient
.instance()
283 cherrypy
.response
.headers
['X-Total-Count'] = paginator
.get_count()
285 if 'services' not in host
:
286 host
['services'] = []
287 host
['service_instances'] = populate_service_instances(
288 host
['hostname'], host
['services'])
289 if str_to_bool(facts
):
291 if not orch
.get_missing_features(['get_facts']):
294 facts
= orch
.hosts
.get_facts(host
['hostname'])[0]
295 hosts_facts
.append(facts
)
296 return merge_list_of_dicts_by_key(hosts
, hosts_facts
, 'hostname')
298 raise DashboardException(
299 code
='invalid_orchestrator_backend', # pragma: no cover
300 msg
="Please enable the cephadm orchestrator backend "
301 "(try `ceph orch set backend cephadm`)",
302 component
='orchestrator',
303 http_status_code
=400)
305 raise DashboardException(code
='orchestrator_status_unavailable', # pragma: no cover
306 msg
="Please configure and enable the orchestrator if you "
307 "really want to gather facts from hosts",
308 component
='orchestrator',
309 http_status_code
=400)
312 @raise_if_no_orchestrator([OrchFeature
.HOST_LIST
, OrchFeature
.HOST_ADD
])
313 @handle_orchestrator_error('host')
314 @host_task('add', {'hostname': '{hostname}'})
317 'hostname': (str, 'Hostname'),
318 'addr': (str, 'Network Address'),
319 'labels': ([str], 'Host Labels'),
320 'status': (str, 'Host Status')
322 responses
={200: None, 204: None})
323 @RESTController.MethodMap(version
=APIVersion
.EXPERIMENTAL
)
324 def create(self
, hostname
: str,
325 addr
: Optional
[str] = None,
326 labels
: Optional
[List
[str]] = None,
327 status
: Optional
[str] = None): # pragma: no cover - requires realtime env
328 add_host(hostname
, addr
, labels
, status
)
330 @raise_if_no_orchestrator([OrchFeature
.HOST_LIST
, OrchFeature
.HOST_REMOVE
])
331 @handle_orchestrator_error('host')
332 @host_task('remove', {'hostname': '{hostname}'})
334 def delete(self
, hostname
): # pragma: no cover - requires realtime env
335 orch_client
= OrchClient
.instance()
336 self
.check_orchestrator_host_op(orch_client
, hostname
, False)
337 orch_client
.hosts
.remove(hostname
)
339 def check_orchestrator_host_op(self
, orch_client
, hostname
, add
=True): # pragma:no cover
340 """Check if we can adding or removing a host with orchestrator
342 :param orch_client: Orchestrator client
343 :param add: True for adding host operation, False for removing host
344 :raise DashboardException
346 host
= orch_client
.hosts
.get(hostname
)
348 raise DashboardException(
349 code
='orchestrator_add_existed_host',
350 msg
='{} is already in orchestrator'.format(hostname
),
351 component
='orchestrator')
352 if not add
and not host
:
353 raise DashboardException(
354 code
='orchestrator_remove_nonexistent_host',
355 msg
='Remove a non-existent host {} from orchestrator'.format(hostname
),
356 component
='orchestrator')
358 @RESTController.Resource('GET')
359 def devices(self
, hostname
):
361 return CephService
.get_devices_by_host(hostname
)
363 @RESTController.Resource('GET')
364 def smart(self
, hostname
):
365 # type: (str) -> dict
366 return CephService
.get_smart_data_by_host(hostname
)
368 @RESTController.Resource('GET')
369 @raise_if_no_orchestrator([OrchFeature
.DEVICE_LIST
])
370 @handle_orchestrator_error('host')
371 @EndpointDoc('Get inventory of a host',
373 'hostname': (str, 'Hostname'),
374 'refresh': (str, 'Trigger asynchronous refresh'),
376 responses
={200: INVENTORY_SCHEMA
})
377 def inventory(self
, hostname
, refresh
=None):
378 inventory
= get_inventories([hostname
], refresh
)
383 @RESTController.Resource('POST')
385 @raise_if_no_orchestrator([OrchFeature
.DEVICE_BLINK_LIGHT
])
386 @handle_orchestrator_error('host')
387 @host_task('identify_device', ['{hostname}', '{device}'], wait_for
=2.0)
388 def identify_device(self
, hostname
, device
, duration
):
389 # type: (str, str, int) -> None
391 Identify a device by switching on the device light for N seconds.
392 :param hostname: The hostname of the device to process.
393 :param device: The device identifier to process, e.g. ``/dev/dm-0`` or
394 ``ABC1234DEF567-1R1234_ABC8DE0Q``.
395 :param duration: The duration in seconds how long the LED should flash.
397 orch
= OrchClient
.instance()
398 TaskManager
.current_task().set_progress(0)
399 orch
.blink_device_light(hostname
, device
, 'ident', True)
400 for i
in range(int(duration
)):
401 percentage
= int(round(i
/ float(duration
) * 100))
402 TaskManager
.current_task().set_progress(percentage
)
404 orch
.blink_device_light(hostname
, device
, 'ident', False)
405 TaskManager
.current_task().set_progress(100)
407 @RESTController.Resource('GET')
408 @raise_if_no_orchestrator([OrchFeature
.DAEMON_LIST
])
409 def daemons(self
, hostname
: str) -> List
[dict]:
410 orch
= OrchClient
.instance()
411 daemons
= orch
.services
.list_daemons(hostname
=hostname
)
412 return [d
.to_dict() for d
in daemons
]
414 @handle_orchestrator_error('host')
415 @RESTController.MethodMap(version
=APIVersion(1, 2))
416 def get(self
, hostname
: str) -> Dict
:
418 Get the specified host.
419 :raises: cherrypy.HTTPError: If host not found.
421 host
= get_host(hostname
)
422 host
['service_instances'] = populate_service_instances(
423 host
['hostname'], host
['services'])
426 @ttl_cache_invalidator('get_hosts')
427 @raise_if_no_orchestrator([OrchFeature
.HOST_LABEL_ADD
,
428 OrchFeature
.HOST_LABEL_REMOVE
,
429 OrchFeature
.HOST_MAINTENANCE_ENTER
,
430 OrchFeature
.HOST_MAINTENANCE_EXIT
,
431 OrchFeature
.HOST_DRAIN
])
432 @handle_orchestrator_error('host')
435 'hostname': (str, 'Hostname'),
436 'update_labels': (bool, 'Update Labels'),
437 'labels': ([str], 'Host Labels'),
438 'maintenance': (bool, 'Enter/Exit Maintenance'),
439 'force': (bool, 'Force Enter Maintenance'),
440 'drain': (bool, 'Drain Host')
442 responses
={200: None, 204: None})
443 @RESTController.MethodMap(version
=APIVersion
.EXPERIMENTAL
)
444 def set(self
, hostname
: str, update_labels
: bool = False,
445 labels
: List
[str] = None, maintenance
: bool = False,
446 force
: bool = False, drain
: bool = False):
448 Update the specified host.
449 Note, this is only supported when Ceph Orchestrator is enabled.
450 :param hostname: The name of the host to be processed.
451 :param update_labels: To update the labels.
452 :param labels: List of labels.
453 :param maintenance: Enter/Exit maintenance mode.
454 :param force: Force enter maintenance mode.
455 :param drain: Drain host
457 orch
= OrchClient
.instance()
458 host
= get_host(hostname
)
461 status
= host
['status']
462 if status
!= 'maintenance':
463 orch
.hosts
.enter_maintenance(hostname
, force
)
465 if status
== 'maintenance':
466 orch
.hosts
.exit_maintenance(hostname
)
469 orch
.hosts
.drain(hostname
)
472 # only allow List[str] type for labels
473 if not isinstance(labels
, list):
474 raise DashboardException(
475 msg
='Expected list of labels. Please check API documentation.',
476 http_status_code
=400,
477 component
='orchestrator')
478 current_labels
= set(host
['labels'])
480 remove_labels
= list(current_labels
.difference(set(labels
)))
481 for label
in remove_labels
:
482 orch
.hosts
.remove_label(hostname
, label
)
484 add_labels
= list(set(labels
).difference(current_labels
))
485 for label
in add_labels
:
486 orch
.hosts
.add_label(hostname
, label
)
489 @UIRouter('/host', Scope
.HOSTS
)
490 class HostUi(BaseController
):
493 @handle_orchestrator_error('host')
494 def labels(self
) -> List
[str]:
497 Note, host labels are only supported when Ceph Orchestrator is enabled.
498 If Ceph Orchestrator is not enabled, an empty list is returned.
499 :return: A list of all host labels.
502 orch
= OrchClient
.instance()
504 for host
in orch
.hosts
.list():
505 labels
.extend(host
.labels
)
507 return list(set(labels
)) # Filter duplicate labels.
511 @raise_if_no_orchestrator([OrchFeature
.DEVICE_LIST
])
512 @handle_orchestrator_error('host')
513 def inventory(self
, refresh
=None):
514 return get_inventories(None, refresh
)