[ceph.git] / ceph / src / pybind / mgr / cephadm / services / monitoring.py

import logging
import os
from typing import List, Any, Tuple, Dict

from orchestrator import DaemonDescription
from cephadm.services.cephadmservice import CephadmService
from mgr_util import verify_tls, ServerConfigException, create_self_signed_cert

logger = logging.getLogger(__name__)

class GrafanaService(CephadmService):
    DEFAULT_SERVICE_PORT = 3000

    def create(self, daemon_id, host):
        # type: (str, str) -> str
        return self.mgr._create_daemon('grafana', daemon_id, host)

    def generate_config(self):
        # type: () -> Tuple[Dict[str, Any], List[str]]
        deps = []  # type: List[str]

        prom_services = []  # type: List[str]
        for dd in self.mgr.cache.get_daemons_by_service('prometheus'):
            prom_services.append(dd.hostname)
            deps.append(dd.name())
        grafana_data_sources = self.mgr.template.render(
            'services/grafana/ceph-dashboard.yml.j2', {'hosts': prom_services})

        cert = self.mgr.get_store('grafana_crt')
        pkey = self.mgr.get_store('grafana_key')
        if cert and pkey:
            try:
                verify_tls(cert, pkey)
            except ServerConfigException as e:
                logger.warning('Provided grafana TLS certificates invalid: %s', str(e))
                cert, pkey = None, None
        if not (cert and pkey):
            cert, pkey = create_self_signed_cert('Ceph', 'cephadm')
            self.mgr.set_store('grafana_crt', cert)
            self.mgr.set_store('grafana_key', pkey)
            self.mgr.check_mon_command({
                'prefix': 'dashboard set-grafana-api-ssl-verify',
                'value': 'false',
            })

        grafana_ini = self.mgr.template.render(
            'services/grafana/grafana.ini.j2', {'http_port': self.DEFAULT_SERVICE_PORT})

        config_file = {
            'files': {
                "grafana.ini": grafana_ini,
                'provisioning/datasources/ceph-dashboard.yml': grafana_data_sources,
                'certs/cert_file': '# generated by cephadm\n%s' % cert,
                'certs/cert_key': '# generated by cephadm\n%s' % pkey,
            }
        }
        return config_file, sorted(deps)

    def get_active_daemon(self, daemon_descrs: List[DaemonDescription]) -> DaemonDescription:
        # Use the least-created one as the active daemon
        return daemon_descrs[-1]

    def daemon_check_post(self, daemon_descrs: List[DaemonDescription]):
        # TODO: signed cert
        dd = self.get_active_daemon(daemon_descrs)
        service_url = 'https://{}:{}'.format(
            self._inventory_get_addr(dd.hostname), self.DEFAULT_SERVICE_PORT)
        self._set_service_url_on_dashboard(
            'Grafana',
            'dashboard get-grafana-api-url',
            'dashboard set-grafana-api-url',
            service_url
        )

class AlertmanagerService(CephadmService):
    DEFAULT_SERVICE_PORT = 9093

    def create(self, daemon_id, host) -> str:
        return self.mgr._create_daemon('alertmanager', daemon_id, host)

    def generate_config(self):
        # type: () -> Tuple[Dict[str, Any], List[str]]
        deps = [] # type: List[str]

        # dashboard(s)
        dashboard_urls = []
        mgr_map = self.mgr.get('mgr_map')
        port = None
        proto = None  # http: or https:
        url = mgr_map.get('services', {}).get('dashboard', None)
        if url:
            dashboard_urls.append(url)
            proto = url.split('/')[0]
            port = url.split('/')[2].split(':')[1]
        # scan all mgrs to generate deps and to get standbys too.
        # assume that they are all on the same port as the active mgr.
        for dd in self.mgr.cache.get_daemons_by_service('mgr'):
            # we consider mgr a dep even if the dashboard is disabled
            # in order to be consistent with _calc_daemon_deps().
            deps.append(dd.name())
            if not port:
                continue
            if dd.daemon_id == self.mgr.get_mgr_id():
                continue
            addr = self.mgr.inventory.get_addr(dd.hostname)
            dashboard_urls.append('%s//%s:%s/' % (proto, addr.split(':')[0],
                                                  port))

        context = {
            'dashboard_urls': dashboard_urls
        }
        yml = self.mgr.template.render('services/alertmanager/alertmanager.yml.j2', context)

        peers = []
        port = '9094'
        for dd in self.mgr.cache.get_daemons_by_service('alertmanager'):
            deps.append(dd.name())
            addr = self.mgr.inventory.get_addr(dd.hostname)
            peers.append(addr.split(':')[0] + ':' + port)
        return {
            "files": {
                "alertmanager.yml": yml
            },
            "peers": peers
        }, sorted(deps)

    def get_active_daemon(self, daemon_descrs: List[DaemonDescription]) -> DaemonDescription:
        # TODO: if there are multiple daemons, who is the active one?
        return daemon_descrs[0]

    def daemon_check_post(self, daemon_descrs: List[DaemonDescription]):
        dd = self.get_active_daemon(daemon_descrs)
        service_url = 'http://{}:{}'.format(self._inventory_get_addr(dd.hostname), self.DEFAULT_SERVICE_PORT)
        self._set_service_url_on_dashboard(
            'AlertManager',
            'dashboard get-alertmanager-api-host',
            'dashboard set-alertmanager-api-host',
            service_url
        )


class PrometheusService(CephadmService):
    DEFAULT_SERVICE_PORT = 9095

    def create(self, daemon_id, host) -> str:
        return self.mgr._create_daemon('prometheus', daemon_id, host)

    def generate_config(self):
        # type: () -> Tuple[Dict[str, Any], List[str]]
        deps = []  # type: List[str]

        # scrape mgrs
        mgr_scrape_list = []
        mgr_map = self.mgr.get('mgr_map')
        port = None
        t = mgr_map.get('services', {}).get('prometheus', None)
        if t:
            t = t.split('/')[2]
            mgr_scrape_list.append(t)
            port = '9283'
            if ':' in t:
                port = t.split(':')[1]
        # scan all mgrs to generate deps and to get standbys too.
        # assume that they are all on the same port as the active mgr.
        for dd in self.mgr.cache.get_daemons_by_service('mgr'):
            # we consider the mgr a dep even if the prometheus module is
            # disabled in order to be consistent with _calc_daemon_deps().
            deps.append(dd.name())
            if not port:
                continue
            if dd.daemon_id == self.mgr.get_mgr_id():
                continue
            addr = self.mgr.inventory.get_addr(dd.hostname)
            mgr_scrape_list.append(addr.split(':')[0] + ':' + port)

        # scrape node exporters
        nodes = []
        for dd in self.mgr.cache.get_daemons_by_service('node-exporter'):
            deps.append(dd.name())
            addr = self.mgr.inventory.get_addr(dd.hostname)
            nodes.append({
                'hostname': dd.hostname,
                'url': addr.split(':')[0] + ':9100'
            })

        # scrape alert managers
        alertmgr_targets = []
        for dd in self.mgr.cache.get_daemons_by_service('alertmanager'):
            deps.append(dd.name())
            addr = self.mgr.inventory.get_addr(dd.hostname)
            alertmgr_targets.append("'{}:9093'".format(addr.split(':')[0]))

        # generate the prometheus configuration
        context = {
            'alertmgr_targets': alertmgr_targets,
            'mgr_scrape_list': mgr_scrape_list,
            'nodes': nodes,
        }
        r = {
            'files': {
                'prometheus.yml':
                    self.mgr.template.render(
                        'services/prometheus/prometheus.yml.j2', context)
            }
        }

        # include alerts, if present in the container
        if os.path.exists(self.mgr.prometheus_alerts_path):
            with open(self.mgr.prometheus_alerts_path, 'r', encoding='utf-8') as f:
                alerts = f.read()
            r['files']['/etc/prometheus/alerting/ceph_alerts.yml'] = alerts

        return r, sorted(deps)

    def get_active_daemon(self, daemon_descrs: List[DaemonDescription]) -> DaemonDescription:
        # TODO: if there are multiple daemons, who is the active one?
        return daemon_descrs[0]

    def daemon_check_post(self, daemon_descrs: List[DaemonDescription]):
        dd = self.get_active_daemon(daemon_descrs)
        service_url = 'http://{}:{}'.format(
            self._inventory_get_addr(dd.hostname), self.DEFAULT_SERVICE_PORT)
        self._set_service_url_on_dashboard(
            'Prometheus',
            'dashboard get-prometheus-api-host',
            'dashboard set-prometheus-api-host',
            service_url
        )

class NodeExporterService(CephadmService):
    def create(self, daemon_id, host) -> str:
        return self.mgr._create_daemon('node-exporter', daemon_id, host)

    def generate_config(self) -> Tuple[Dict[str, Any], List[str]]:
        return {}, []
Commit	Line	Data
e306af50 TL	1	import logging
	2	import os
	3	from typing import List, Any, Tuple, Dict
	4
	5	from orchestrator import DaemonDescription
	6	from cephadm.services.cephadmservice import CephadmService
	7	from mgr_util import verify_tls, ServerConfigException, create_self_signed_cert
	8
	9	logger = logging.getLogger(__name__)
	10
	11	class GrafanaService(CephadmService):
	12	DEFAULT_SERVICE_PORT = 3000
	13
	14	def create(self, daemon_id, host):
	15	# type: (str, str) -> str
	16	return self.mgr._create_daemon('grafana', daemon_id, host)
	17
	18	def generate_config(self):
	19	# type: () -> Tuple[Dict[str, Any], List[str]]
	20	deps = [] # type: List[str]
	21
	22	prom_services = [] # type: List[str]
	23	for dd in self.mgr.cache.get_daemons_by_service('prometheus'):
	24	prom_services.append(dd.hostname)
	25	deps.append(dd.name())
	26	grafana_data_sources = self.mgr.template.render(
	27	'services/grafana/ceph-dashboard.yml.j2', {'hosts': prom_services})
	28
	29	cert = self.mgr.get_store('grafana_crt')
	30	pkey = self.mgr.get_store('grafana_key')
	31	if cert and pkey:
	32	try:
	33	verify_tls(cert, pkey)
	34	except ServerConfigException as e:
	35	logger.warning('Provided grafana TLS certificates invalid: %s', str(e))
	36	cert, pkey = None, None
	37	if not (cert and pkey):
	38	cert, pkey = create_self_signed_cert('Ceph', 'cephadm')
	39	self.mgr.set_store('grafana_crt', cert)
	40	self.mgr.set_store('grafana_key', pkey)
	41	self.mgr.check_mon_command({
	42	'prefix': 'dashboard set-grafana-api-ssl-verify',
	43	'value': 'false',
	44	})
	45
	46	grafana_ini = self.mgr.template.render(
	47	'services/grafana/grafana.ini.j2', {'http_port': self.DEFAULT_SERVICE_PORT})
	48
	49	config_file = {
	50	'files': {
	51	"grafana.ini": grafana_ini,
	52	'provisioning/datasources/ceph-dashboard.yml': grafana_data_sources,
	53	'certs/cert_file': '# generated by cephadm\n%s' % cert,
	54	'certs/cert_key': '# generated by cephadm\n%s' % pkey,
	55	}
	56	}
	57	return config_file, sorted(deps)
	58
	59	def get_active_daemon(self, daemon_descrs: List[DaemonDescription]) -> DaemonDescription:
	60	# Use the least-created one as the active daemon
	61	return daemon_descrs[-1]
	62
	63	def daemon_check_post(self, daemon_descrs: List[DaemonDescription]):
	64	# TODO: signed cert
65	dd = self.get_active_daemon(daemon_descrs)
66	service_url = 'https://{}:{}'.format(
67	self._inventory_get_addr(dd.hostname), self.DEFAULT_SERVICE_PORT)
68	self._set_service_url_on_dashboard(
69	'Grafana',
70	'dashboard get-grafana-api-url',
71	'dashboard set-grafana-api-url',
72	service_url
73	)
74
75	class AlertmanagerService(CephadmService):
76	DEFAULT_SERVICE_PORT = 9093
77
78	def create(self, daemon_id, host) -> str:
79	return self.mgr._create_daemon('alertmanager', daemon_id, host)
80
81	def generate_config(self):
82	# type: () -> Tuple[Dict[str, Any], List[str]]
83	deps = [] # type: List[str]
84
85	# dashboard(s)
86	dashboard_urls = []
87	mgr_map = self.mgr.get('mgr_map')
88	port = None
89	proto = None # http: or https:
90	url = mgr_map.get('services', {}).get('dashboard', None)
91	if url:
92	dashboard_urls.append(url)
93	proto = url.split('/')[0]
94	port = url.split('/')[2].split(':')[1]
95	# scan all mgrs to generate deps and to get standbys too.
96	# assume that they are all on the same port as the active mgr.
97	for dd in self.mgr.cache.get_daemons_by_service('mgr'):
98	# we consider mgr a dep even if the dashboard is disabled
99	# in order to be consistent with _calc_daemon_deps().
100	deps.append(dd.name())
101	if not port:
102	continue
103	if dd.daemon_id == self.mgr.get_mgr_id():
104	continue
105	addr = self.mgr.inventory.get_addr(dd.hostname)
106	dashboard_urls.append('%s//%s:%s/' % (proto, addr.split(':')[0],
107	port))
108
109	context = {
110	'dashboard_urls': dashboard_urls
111	}
112	yml = self.mgr.template.render('services/alertmanager/alertmanager.yml.j2', context)
113
114	peers = []
115	port = '9094'
116	for dd in self.mgr.cache.get_daemons_by_service('alertmanager'):
117	deps.append(dd.name())
118	addr = self.mgr.inventory.get_addr(dd.hostname)
119	peers.append(addr.split(':')[0] + ':' + port)
120	return {
121	"files": {
122	"alertmanager.yml": yml
123	},
124	"peers": peers
125	}, sorted(deps)
126
127	def get_active_daemon(self, daemon_descrs: List[DaemonDescription]) -> DaemonDescription:
128	# TODO: if there are multiple daemons, who is the active one?
129	return daemon_descrs[0]
130
131	def daemon_check_post(self, daemon_descrs: List[DaemonDescription]):
132	dd = self.get_active_daemon(daemon_descrs)
133	service_url = 'http://{}:{}'.format(self._inventory_get_addr(dd.hostname), self.DEFAULT_SERVICE_PORT)
134	self._set_service_url_on_dashboard(
135	'AlertManager',
136	'dashboard get-alertmanager-api-host',
137	'dashboard set-alertmanager-api-host',
138	service_url
139	)
140
141
142	class PrometheusService(CephadmService):
143	DEFAULT_SERVICE_PORT = 9095
144
145	def create(self, daemon_id, host) -> str:
146	return self.mgr._create_daemon('prometheus', daemon_id, host)
147
148	def generate_config(self):
149	# type: () -> Tuple[Dict[str, Any], List[str]]
150	deps = [] # type: List[str]
151
152	# scrape mgrs
153	mgr_scrape_list = []
154	mgr_map = self.mgr.get('mgr_map')
155	port = None
156	t = mgr_map.get('services', {}).get('prometheus', None)
157	if t:
158	t = t.split('/')[2]
159	mgr_scrape_list.append(t)
160	port = '9283'
161	if ':' in t:
162	port = t.split(':')[1]
163	# scan all mgrs to generate deps and to get standbys too.
164	# assume that they are all on the same port as the active mgr.
165	for dd in self.mgr.cache.get_daemons_by_service('mgr'):
166	# we consider the mgr a dep even if the prometheus module is
167	# disabled in order to be consistent with _calc_daemon_deps().
168	deps.append(dd.name())
169	if not port:
170	continue
171	if dd.daemon_id == self.mgr.get_mgr_id():
172	continue
173	addr = self.mgr.inventory.get_addr(dd.hostname)
174	mgr_scrape_list.append(addr.split(':')[0] + ':' + port)
175
176	# scrape node exporters
177	nodes = []
178	for dd in self.mgr.cache.get_daemons_by_service('node-exporter'):
179	deps.append(dd.name())
180	addr = self.mgr.inventory.get_addr(dd.hostname)
181	nodes.append({
182	'hostname': dd.hostname,
183	'url': addr.split(':')[0] + ':9100'
184	})
185
186	# scrape alert managers
187	alertmgr_targets = []
188	for dd in self.mgr.cache.get_daemons_by_service('alertmanager'):
189	deps.append(dd.name())
190	addr = self.mgr.inventory.get_addr(dd.hostname)
191	alertmgr_targets.append("'{}:9093'".format(addr.split(':')[0]))
192
193	# generate the prometheus configuration
194	context = {
195	'alertmgr_targets': alertmgr_targets,
196	'mgr_scrape_list': mgr_scrape_list,
197	'nodes': nodes,
198	}
199	r = {
200	'files': {
201	'prometheus.yml':
202	self.mgr.template.render(
203	'services/prometheus/prometheus.yml.j2', context)
204	}
205	}
206
207	# include alerts, if present in the container
208	if os.path.exists(self.mgr.prometheus_alerts_path):
209	with open(self.mgr.prometheus_alerts_path, 'r', encoding='utf-8') as f:
210	alerts = f.read()
211	r['files']['/etc/prometheus/alerting/ceph_alerts.yml'] = alerts
212
213	return r, sorted(deps)
214
215	def get_active_daemon(self, daemon_descrs: List[DaemonDescription]) -> DaemonDescription:
216	# TODO: if there are multiple daemons, who is the active one?
217	return daemon_descrs[0]
218
219	def daemon_check_post(self, daemon_descrs: List[DaemonDescription]):
220	dd = self.get_active_daemon(daemon_descrs)
221	service_url = 'http://{}:{}'.format(
222	self._inventory_get_addr(dd.hostname), self.DEFAULT_SERVICE_PORT)
223	self._set_service_url_on_dashboard(
224	'Prometheus',
225	'dashboard get-prometheus-api-host',
226	'dashboard set-prometheus-api-host',
227	service_url
228	)
229
230	class NodeExporterService(CephadmService):
231	def create(self, daemon_id, host) -> str:
232	return self.mgr._create_daemon('node-exporter', daemon_id, host)
233
234	def generate_config(self) -> Tuple[Dict[str, Any], List[str]]:
235	return {}, []