NodeExporterService, LokiService, PromtailService
from cephadm.module import CephadmOrchestrator
from ceph.deployment.service_spec import IscsiServiceSpec, MonitoringSpec, AlertManagerSpec, \
- ServiceSpec, RGWSpec, GrafanaSpec, SNMPGatewaySpec, IngressSpec, PlacementSpec
+ ServiceSpec, RGWSpec, GrafanaSpec, SNMPGatewaySpec, IngressSpec, PlacementSpec, \
+ PrometheusSpec, CephExporterSpec, NFSServiceSpec
from cephadm.tests.fixtures import with_host, with_service, _run_cephadm, async_side_effect
+from ceph.utils import datetime_now
+
from orchestrator import OrchestratorError
from orchestrator._interface import DaemonDescription
if prefix == 'set-cmd':
self.config = cmd_dict.get('value')
return 0, 'value set', ''
+ if prefix in ['auth get']:
+ return 0, '[foo]\nkeyring = asdf\n', ''
return -1, '', 'error'
def get_minimal_ceph_conf(self) -> str:
expected_call2 = call({'prefix': 'auth caps',
'entity': 'client.iscsi.a',
'caps': expected_caps})
+ expected_call3 = call({'prefix': 'auth get',
+ 'entity': 'client.iscsi.a'})
assert expected_call in self.mgr.mon_command.mock_calls
assert expected_call2 in self.mgr.mon_command.mock_calls
+ assert expected_call3 in self.mgr.mon_command.mock_calls
@patch('cephadm.utils.resolve_ip')
def test_iscsi_dashboard_config(self, mock_resolve_ip):
assert dashboard_expected_call in self.mgr.check_mon_command.mock_calls
+ @patch("cephadm.serve.CephadmServe._run_cephadm")
+ @patch("cephadm.module.CephadmOrchestrator.get_unique_name")
+ @patch("cephadm.services.iscsi.IscsiService.get_trusted_ips")
+ def test_iscsi_config(self, _get_trusted_ips, _get_name, _run_cephadm, cephadm_module: CephadmOrchestrator):
+
+ iscsi_daemon_id = 'testpool.test.qwert'
+ trusted_ips = '1.1.1.1,2.2.2.2'
+ api_port = 3456
+ api_user = 'test-user'
+ api_password = 'test-password'
+ pool = 'testpool'
+ _run_cephadm.side_effect = async_side_effect(('{}', '', 0))
+ _get_name.return_value = iscsi_daemon_id
+ _get_trusted_ips.return_value = trusted_ips
+
+ iscsi_gateway_conf = f"""# This file is generated by cephadm.
+[config]
+cluster_client_name = client.iscsi.{iscsi_daemon_id}
+pool = {pool}
+trusted_ip_list = {trusted_ips}
+minimum_gateways = 1
+api_port = {api_port}
+api_user = {api_user}
+api_password = {api_password}
+api_secure = False
+log_to_stderr = True
+log_to_stderr_prefix = debug
+log_to_file = False"""
+
+ with with_host(cephadm_module, 'test'):
+ with with_service(cephadm_module, IscsiServiceSpec(service_id=pool,
+ api_port=api_port,
+ api_user=api_user,
+ api_password=api_password,
+ pool=pool,
+ trusted_ip_list=trusted_ips)):
+ _run_cephadm.assert_called_with(
+ 'test',
+ f'iscsi.{iscsi_daemon_id}',
+ 'deploy',
+ [
+ '--name', f'iscsi.{iscsi_daemon_id}',
+ '--meta-json', f'{"{"}"service_name": "iscsi.{pool}", "ports": [{api_port}], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null, "extra_entrypoint_args": null{"}"}',
+ '--config-json', '-', '--tcp-ports', '3456'
+ ],
+ stdin=json.dumps({"config": "", "keyring": "[client.iscsi.testpool.test.qwert]\nkey = None\n", "files": {"iscsi-gateway.cfg": iscsi_gateway_conf}}),
+ image='')
+
class TestMonitoring:
def _get_config(self, url: str) -> str:
'deploy',
[
'--name', 'alertmanager.test',
- '--meta-json', '{"service_name": "alertmanager", "ports": [9093, 9094], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
+ '--meta-json', '{"service_name": "alertmanager", "ports": [9093, 9094], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null, "extra_entrypoint_args": null}',
'--config-json', '-', '--tcp-ports', '9093 9094'
],
stdin=json.dumps({"files": {"alertmanager.yml": y}, "peers": []}),
[
'--name', 'alertmanager.test',
'--meta-json',
- '{"service_name": "alertmanager", "ports": [9093, 9094], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
+ '{"service_name": "alertmanager", "ports": [9093, 9094], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null, "extra_entrypoint_args": null}',
'--config-json', '-', '--tcp-ports', '9093 9094'
],
stdin=json.dumps({"files": {"alertmanager.yml": y}, "peers": []}),
[
'--name', 'alertmanager.test',
'--meta-json',
- '{"service_name": "alertmanager", "ports": [9093, 9094], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
+ '{"service_name": "alertmanager", "ports": [9093, 9094], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null, "extra_entrypoint_args": null}',
'--config-json', '-', '--tcp-ports', '9093 9094'
],
stdin=json.dumps({"files": {"alertmanager.yml": y}, "peers": []}),
'deploy',
[
'--name', 'alertmanager.test',
- '--meta-json', '{"service_name": "alertmanager", "ports": [9093, 9094], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
+ '--meta-json', '{"service_name": "alertmanager", "ports": [9093, 9094], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null, "extra_entrypoint_args": null}',
'--config-json', '-', '--tcp-ports', '9093 9094'
],
stdin=json.dumps({"files": {"alertmanager.yml": y}, "peers": []}),
[
'--name', 'alertmanager.test',
'--meta-json',
- '{"service_name": "alertmanager", "ports": [9093, 9094], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
+ '{"service_name": "alertmanager", "ports": [9093, 9094], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null, "extra_entrypoint_args": null}',
'--config-json', '-', '--tcp-ports', '9093 9094'
],
stdin=json.dumps({"files": {"alertmanager.yml": y}, "peers": []}),
with with_host(cephadm_module, 'test'):
with with_service(cephadm_module, MonitoringSpec('node-exporter')) as _, \
- with_service(cephadm_module, MonitoringSpec('prometheus')) as _:
+ with_service(cephadm_module, CephExporterSpec('ceph-exporter')) as _, \
+ with_service(cephadm_module, PrometheusSpec('prometheus')) as _:
y = dedent("""
# This file is generated by cephadm.
labels:
instance: 'test'
+
+ - job_name: 'ceph-exporter'
+ honor_labels: true
+ static_configs:
+ - targets: ['[1::4]:9926']
+ labels:
+ instance: 'test'
""").lstrip()
_run_cephadm.assert_called_with(
[
'--name', 'prometheus.test',
'--meta-json',
- '{"service_name": "prometheus", "ports": [9095], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
+ ('{"service_name": "prometheus", "ports": [9095], "ip": null, "deployed_by": [], "rank": null, '
+ '"rank_generation": null, "extra_container_args": null, "extra_entrypoint_args": null}'),
'--config-json', '-',
'--tcp-ports', '9095'
],
stdin=json.dumps({"files": {"prometheus.yml": y,
- "/etc/prometheus/alerting/custom_alerts.yml": ""}}),
+ "/etc/prometheus/alerting/custom_alerts.yml": ""},
+ 'retention_time': '15d',
+ 'retention_size': '0'}),
image='')
@patch("cephadm.serve.CephadmServe._run_cephadm")
[
'--name', 'loki.test',
'--meta-json',
- '{"service_name": "loki", "ports": [3100], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
+ ('{"service_name": "loki", "ports": [3100], "ip": null, "deployed_by": [], "rank": null, '
+ '"rank_generation": null, "extra_container_args": null, "extra_entrypoint_args": null}'),
'--config-json', '-',
'--tcp-ports', '3100'
],
[
'--name', 'promtail.test',
'--meta-json',
- '{"service_name": "promtail", "ports": [9080], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
+ ('{"service_name": "promtail", "ports": [9080], "ip": null, "deployed_by": [], "rank": null, '
+ '"rank_generation": null, "extra_container_args": null, "extra_entrypoint_args": null}'),
'--config-json', '-',
'--tcp-ports', '9080'
],
cephadm_module.set_store("test/grafana_crt", "c")
cephadm_module.set_store("test/grafana_key", "k")
with with_service(
- cephadm_module, MonitoringSpec("prometheus")
+ cephadm_module, PrometheusSpec("prometheus")
) as _, with_service(cephadm_module, ServiceSpec("mgr")) as _, with_service(
cephadm_module, GrafanaSpec("grafana")
) as _:
[
'--name', 'grafana.test',
'--meta-json',
- '{"service_name": "grafana", "ports": [3000], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
+ ('{"service_name": "grafana", "ports": [3000], "ip": null, "deployed_by": [], "rank": null, '
+ '"rank_generation": null, "extra_container_args": null, "extra_entrypoint_args": null}'),
'--config-json', '-', '--tcp-ports', '3000'],
stdin=json.dumps({"files": files}),
image='')
_run_cephadm.assert_called_with(
'test', 'alertmanager.test', 'deploy', [
'--name', 'alertmanager.test',
- '--meta-json', '{"service_name": "alertmanager", "ports": [4200, 9094], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
+ '--meta-json', ('{"service_name": "alertmanager", "ports": [4200, 9094], "ip": null, "deployed_by": [], "rank": null, '
+ '"rank_generation": null, "extra_container_args": null, "extra_entrypoint_args": null}'),
'--config-json', '-',
'--tcp-ports', '4200 9094',
'--reconfig'
[
'--name', 'snmp-gateway.test',
'--meta-json',
- '{"service_name": "snmp-gateway", "ports": [9464], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
+ ('{"service_name": "snmp-gateway", "ports": [9464], "ip": null, "deployed_by": [], "rank": null, '
+ '"rank_generation": null, "extra_container_args": null, "extra_entrypoint_args": null}'),
'--config-json', '-',
'--tcp-ports', '9464'
],
[
'--name', 'snmp-gateway.test',
'--meta-json',
- '{"service_name": "snmp-gateway", "ports": [9465], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
+ ('{"service_name": "snmp-gateway", "ports": [9465], "ip": null, "deployed_by": [], "rank": null, '
+ '"rank_generation": null, "extra_container_args": null, "extra_entrypoint_args": null}'),
'--config-json', '-',
'--tcp-ports', '9465'
],
[
'--name', 'snmp-gateway.test',
'--meta-json',
- '{"service_name": "snmp-gateway", "ports": [9464], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
+ ('{"service_name": "snmp-gateway", "ports": [9464], "ip": null, "deployed_by": [], "rank": null, '
+ '"rank_generation": null, "extra_container_args": null, "extra_entrypoint_args": null}'),
'--config-json', '-',
'--tcp-ports', '9464'
],
[
'--name', 'snmp-gateway.test',
'--meta-json',
- '{"service_name": "snmp-gateway", "ports": [9464], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
+ ('{"service_name": "snmp-gateway", "ports": [9464], "ip": null, "deployed_by": [], "rank": null, '
+ '"rank_generation": null, "extra_container_args": null, "extra_entrypoint_args": null}'),
'--config-json', '-',
'--tcp-ports', '9464'
],
class TestIngressService:
+ @patch("cephadm.inventory.Inventory.get_addr")
+ @patch("cephadm.utils.resolve_ip")
+ @patch("cephadm.inventory.HostCache.get_daemons_by_service")
+ @patch("cephadm.serve.CephadmServe._run_cephadm")
+ def test_ingress_config_nfs_multiple_nfs_same_rank(self, _run_cephadm, _get_daemons_by_service, _resolve_ip, _get_addr, cephadm_module: CephadmOrchestrator):
+ _run_cephadm.side_effect = async_side_effect(('{}', '', 0))
+
+ def fake_resolve_ip(hostname: str) -> str:
+ if hostname == 'host1':
+ return '192.168.122.111'
+ elif hostname == 'host2':
+ return '192.168.122.222'
+ else:
+ return 'xxx.xxx.xxx.xxx'
+ _resolve_ip.side_effect = fake_resolve_ip
+
+ def fake_get_addr(hostname: str) -> str:
+ return hostname
+ _get_addr.side_effect = fake_get_addr
+
+ nfs_service = NFSServiceSpec(service_id="foo", placement=PlacementSpec(count=1, hosts=['host1', 'host2']),
+ port=12049)
+
+ ispec = IngressSpec(service_type='ingress',
+ service_id='nfs.foo',
+ backend_service='nfs.foo',
+ frontend_port=2049,
+ monitor_port=9049,
+ virtual_ip='192.168.122.100/24',
+ monitor_user='admin',
+ monitor_password='12345',
+ keepalived_password='12345')
+
+ cephadm_module.spec_store._specs = {
+ 'nfs.foo': nfs_service,
+ 'ingress.nfs.foo': ispec
+ }
+ cephadm_module.spec_store.spec_created = {
+ 'nfs.foo': datetime_now(),
+ 'ingress.nfs.foo': datetime_now()
+ }
+
+ # in both test cases we'll do here, we want only the ip
+ # for the host1 nfs daemon as we'll end up giving that
+ # one higher rank_generation but the same rank as the one
+ # on host2
+ haproxy_expected_conf = {
+ 'files':
+ {
+ 'haproxy.cfg':
+ '# This file is generated by cephadm.\n'
+ 'global\n'
+ ' log 127.0.0.1 local2\n'
+ ' chroot /var/lib/haproxy\n'
+ ' pidfile /var/lib/haproxy/haproxy.pid\n'
+ ' maxconn 8000\n'
+ ' daemon\n'
+ ' stats socket /var/lib/haproxy/stats\n\n'
+ 'defaults\n'
+ ' mode tcp\n'
+ ' log global\n'
+ ' timeout queue 1m\n'
+ ' timeout connect 10s\n'
+ ' timeout client 1m\n'
+ ' timeout server 1m\n'
+ ' timeout check 10s\n'
+ ' maxconn 8000\n\n'
+ 'frontend stats\n'
+ ' mode http\n'
+ ' bind 192.168.122.100:9049\n'
+ ' bind localhost:9049\n'
+ ' stats enable\n'
+ ' stats uri /stats\n'
+ ' stats refresh 10s\n'
+ ' stats auth admin:12345\n'
+ ' http-request use-service prometheus-exporter if { path /metrics }\n'
+ ' monitor-uri /health\n\n'
+ 'frontend frontend\n'
+ ' bind 192.168.122.100:2049\n'
+ ' default_backend backend\n\n'
+ 'backend backend\n'
+ ' mode tcp\n'
+ ' balance source\n'
+ ' hash-type consistent\n'
+ ' server nfs.foo.0 192.168.122.111:12049\n'
+ }
+ }
+
+ # verify we get the same cfg regardless of the order in which the nfs daemons are returned
+ # in this case both nfs are rank 0, so it should only take the one with rank_generation 1 a.k.a
+ # the one on host1
+ nfs_daemons = [
+ DaemonDescription(daemon_type='nfs', daemon_id='foo.0.1.host1.qwerty', hostname='host1', rank=0, rank_generation=1, ports=[12049]),
+ DaemonDescription(daemon_type='nfs', daemon_id='foo.0.0.host2.abcdef', hostname='host2', rank=0, rank_generation=0, ports=[12049])
+ ]
+ _get_daemons_by_service.return_value = nfs_daemons
+
+ haproxy_generated_conf = cephadm_module.cephadm_services['ingress'].haproxy_generate_config(
+ CephadmDaemonDeploySpec(host='host1', daemon_id='ingress', service_name=ispec.service_name()))
+
+ assert haproxy_generated_conf[0] == haproxy_expected_conf
+
+ # swapping order now, should still pick out the one with the higher rank_generation
+ # in this case both nfs are rank 0, so it should only take the one with rank_generation 1 a.k.a
+ # the one on host1
+ nfs_daemons = [
+ DaemonDescription(daemon_type='nfs', daemon_id='foo.0.0.host2.abcdef', hostname='host2', rank=0, rank_generation=0, ports=[12049]),
+ DaemonDescription(daemon_type='nfs', daemon_id='foo.0.1.host1.qwerty', hostname='host1', rank=0, rank_generation=1, ports=[12049])
+ ]
+ _get_daemons_by_service.return_value = nfs_daemons
+
+ haproxy_generated_conf = cephadm_module.cephadm_services['ingress'].haproxy_generate_config(
+ CephadmDaemonDeploySpec(host='host1', daemon_id='ingress', service_name=ispec.service_name()))
+
+ assert haproxy_generated_conf[0] == haproxy_expected_conf
+
@patch("cephadm.serve.CephadmServe._run_cephadm")
def test_ingress_config(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
'timeout connect 5s\n '
'timeout http-request 1s\n '
'timeout http-keep-alive 5s\n '
- 'timeout client 1s\n '
- 'timeout server 1s\n '
+ 'timeout client 30s\n '
+ 'timeout server 30s\n '
'timeout check 5s\n '
'maxconn 8000\n'
'\nfrontend stats\n '
assert haproxy_generated_conf[0] == haproxy_expected_conf
+ @patch("cephadm.serve.CephadmServe._run_cephadm")
+ def test_ingress_config_ssl_rgw(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
+ _run_cephadm.side_effect = async_side_effect(('{}', '', 0))
+
+ with with_host(cephadm_module, 'test'):
+ cephadm_module.cache.update_host_networks('test', {
+ '1.2.3.0/24': {
+ 'if0': ['1.2.3.4/32']
+ }
+ })
+
+ # the ingress backend
+ s = RGWSpec(service_id="foo", placement=PlacementSpec(count=1),
+ rgw_frontend_type='beast', rgw_frontend_port=443, ssl=True)
+
+ ispec = IngressSpec(service_type='ingress',
+ service_id='test',
+ backend_service='rgw.foo',
+ frontend_port=8089,
+ monitor_port=8999,
+ monitor_user='admin',
+ monitor_password='12345',
+ keepalived_password='12345',
+ virtual_interface_networks=['1.2.3.0/24'],
+ virtual_ip="1.2.3.4/32")
+ with with_service(cephadm_module, s) as _, with_service(cephadm_module, ispec) as _:
+ # generate the keepalived conf based on the specified spec
+ keepalived_generated_conf = cephadm_module.cephadm_services['ingress'].keepalived_generate_config(
+ CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name()))
+
+ keepalived_expected_conf = {
+ 'files':
+ {
+ 'keepalived.conf':
+ '# This file is generated by cephadm.\n'
+ 'vrrp_script check_backend {\n '
+ 'script "/usr/bin/curl http://localhost:8999/health"\n '
+ 'weight -20\n '
+ 'interval 2\n '
+ 'rise 2\n '
+ 'fall 2\n}\n\n'
+ 'vrrp_instance VI_0 {\n '
+ 'state MASTER\n '
+ 'priority 100\n '
+ 'interface if0\n '
+ 'virtual_router_id 50\n '
+ 'advert_int 1\n '
+ 'authentication {\n '
+ 'auth_type PASS\n '
+ 'auth_pass 12345\n '
+ '}\n '
+ 'unicast_src_ip 1::4\n '
+ 'unicast_peer {\n '
+ '}\n '
+ 'virtual_ipaddress {\n '
+ '1.2.3.4/32 dev if0\n '
+ '}\n '
+ 'track_script {\n '
+ 'check_backend\n }\n'
+ '}\n'
+ }
+ }
+
+ # check keepalived config
+ assert keepalived_generated_conf[0] == keepalived_expected_conf
+
+ # generate the haproxy conf based on the specified spec
+ haproxy_generated_conf = cephadm_module.cephadm_services['ingress'].haproxy_generate_config(
+ CephadmDaemonDeploySpec(host='test', daemon_id='ingress', service_name=ispec.service_name()))
+
+ haproxy_expected_conf = {
+ 'files':
+ {
+ 'haproxy.cfg':
+ '# This file is generated by cephadm.'
+ '\nglobal\n log '
+ '127.0.0.1 local2\n '
+ 'chroot /var/lib/haproxy\n '
+ 'pidfile /var/lib/haproxy/haproxy.pid\n '
+ 'maxconn 8000\n '
+ 'daemon\n '
+ 'stats socket /var/lib/haproxy/stats\n'
+ '\ndefaults\n '
+ 'mode http\n '
+ 'log global\n '
+ 'option httplog\n '
+ 'option dontlognull\n '
+ 'option http-server-close\n '
+ 'option forwardfor except 127.0.0.0/8\n '
+ 'option redispatch\n '
+ 'retries 3\n '
+ 'timeout queue 20s\n '
+ 'timeout connect 5s\n '
+ 'timeout http-request 1s\n '
+ 'timeout http-keep-alive 5s\n '
+ 'timeout client 30s\n '
+ 'timeout server 30s\n '
+ 'timeout check 5s\n '
+ 'maxconn 8000\n'
+ '\nfrontend stats\n '
+ 'mode http\n '
+ 'bind 1.2.3.4:8999\n '
+ 'bind localhost:8999\n '
+ 'stats enable\n '
+ 'stats uri /stats\n '
+ 'stats refresh 10s\n '
+ 'stats auth admin:12345\n '
+ 'http-request use-service prometheus-exporter if { path /metrics }\n '
+ 'monitor-uri /health\n'
+ '\nfrontend frontend\n '
+ 'bind 1.2.3.4:8089\n '
+ 'default_backend backend\n\n'
+ 'backend backend\n '
+ 'option forwardfor\n '
+ 'default-server ssl\n '
+ 'default-server verify none\n '
+ 'balance static-rr\n '
+ 'option httpchk HEAD / HTTP/1.0\n '
+ 'server '
+ + haproxy_generated_conf[1][0] + ' 1::4:443 check weight 100\n'
+ }
+ }
+
+ assert haproxy_generated_conf[0] == haproxy_expected_conf
+
@patch("cephadm.serve.CephadmServe._run_cephadm")
def test_ingress_config_multi_vips(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
'timeout connect 5s\n '
'timeout http-request 1s\n '
'timeout http-keep-alive 5s\n '
- 'timeout client 1s\n '
- 'timeout server 1s\n '
+ 'timeout client 30s\n '
+ 'timeout server 30s\n '
'timeout check 5s\n '
'maxconn 8000\n'
'\nfrontend stats\n '
projects / ceph.git / blobdiff