ceph/src/pybind/mgr/cephadm/tests/test_spec.py

   1 # Disable autopep8 for this file:
   2
   3 # fmt: off
   4
   5 import json
   6
   7 import pytest
   8
   9 from ceph.deployment.service_spec import ServiceSpec, NFSServiceSpec, RGWSpec, \
  10     IscsiServiceSpec, HostPlacementSpec, CustomContainerSpec
  11 from orchestrator import DaemonDescription, OrchestratorError
  12
  13
  14 @pytest.mark.parametrize(
  15     "spec_json",
  16     json.loads("""[
  17 {
  18   "placement": {
  19     "count": 1
  20   },
  21   "service_type": "alertmanager"
  22 },
  23 {
  24   "placement": {
  25     "host_pattern": "*"
  26   },
  27   "service_type": "crash"
  28 },
  29 {
  30   "placement": {
  31     "count": 1
  32   },
  33   "service_type": "grafana",
  34   "protocol": "https"
  35 },
  36 {
  37   "placement": {
  38     "count": 2
  39   },
  40   "service_type": "mgr"
  41 },
  42 {
  43   "placement": {
  44     "count": 5
  45   },
  46   "service_type": "mon"
  47 },
  48 {
  49   "placement": {
  50     "host_pattern": "*"
  51   },
  52   "service_type": "node-exporter"
  53 },
  54 {
  55   "placement": {
  56     "count": 1
  57   },
  58   "service_type": "prometheus"
  59 },
  60 {
  61   "placement": {
  62     "hosts": [
  63       {
  64         "hostname": "ceph-001",
  65         "network": "",
  66         "name": ""
  67       }
  68     ]
  69   },
  70   "service_type": "rgw",
  71   "service_id": "default-rgw-realm.eu-central-1.1",
  72   "rgw_realm": "default-rgw-realm",
  73   "rgw_zone": "eu-central-1"
  74 },
  75 {
  76   "service_type": "osd",
  77   "service_id": "osd_spec_default",
  78   "placement": {
  79     "host_pattern": "*"
  80   },
  81   "data_devices": {
  82     "model": "MC-55-44-XZ"
  83   },
  84   "db_devices": {
  85     "model": "SSD-123-foo"
  86   },
  87   "wal_devices": {
  88     "model": "NVME-QQQQ-987"
  89   }
  90 }
  91 ]
  92 """)
  93 )
  94 def test_spec_octopus(spec_json):
  95     # https://tracker.ceph.com/issues/44934
  96     # Those are real user data from early octopus.
  97     # Please do not modify those JSON values.
  98
  99     spec = ServiceSpec.from_json(spec_json)
 100
 101     # just some verification that we can sill read old octopus specs
 102     def convert_to_old_style_json(j):
 103         j_c = dict(j.copy())
 104         j_c.pop('service_name', None)
 105         if 'spec' in j_c:
 106             spec = j_c.pop('spec')
 107             j_c.update(spec)
 108         if 'placement' in j_c:
 109             if 'hosts' in j_c['placement']:
 110                 j_c['placement']['hosts'] = [
 111                     {
 112                         'hostname': HostPlacementSpec.parse(h).hostname,
 113                         'network': HostPlacementSpec.parse(h).network,
 114                         'name': HostPlacementSpec.parse(h).name
 115                     }
 116                     for h in j_c['placement']['hosts']
 117                 ]
 118         j_c.pop('objectstore', None)
 119         j_c.pop('filter_logic', None)
 120         j_c.pop('anonymous_access', None)
 121         return j_c
 122
 123     assert spec_json == convert_to_old_style_json(spec.to_json())
 124
 125
 126 @pytest.mark.parametrize(
 127     "dd_json",
 128     json.loads("""[
 129     {
 130         "hostname": "ceph-001",
 131         "container_id": "d94d7969094d",
 132         "container_image_id": "0881eb8f169f5556a292b4e2c01d683172b12830a62a9225a98a8e206bb734f0",
 133         "container_image_name": "docker.io/prom/alertmanager:latest",
 134         "daemon_id": "ceph-001",
 135         "daemon_type": "alertmanager",
 136         "version": "0.20.0",
 137         "status": 1,
 138         "status_desc": "running",
 139         "last_refresh": "2020-04-03T15:31:48.725856",
 140         "created": "2020-04-02T19:23:08.829543",
 141         "started": "2020-04-03T07:29:16.932838",
 142         "is_active": false
 143     },
 144     {
 145         "hostname": "ceph-001",
 146         "container_id": "c4b036202241",
 147         "container_image_id": "204a01f9b0b6710dd0c0af7f37ce7139c47ff0f0105d778d7104c69282dfbbf1",
 148         "container_image_name": "docker.io/ceph/ceph:v15",
 149         "daemon_id": "ceph-001",
 150         "daemon_type": "crash",
 151         "version": "15.2.0",
 152         "status": 1,
 153         "status_desc": "running",
 154         "last_refresh": "2020-04-03T15:31:48.725903",
 155         "created": "2020-04-02T19:23:11.390694",
 156         "started": "2020-04-03T07:29:16.910897",
 157         "is_active": false
 158     },
 159     {
 160         "hostname": "ceph-001",
 161         "container_id": "5b7b94b48f31",
 162         "container_image_id": "87a51ecf0b1c9a7b187b21c1b071425dafea0d765a96d5bc371c791169b3d7f4",
 163         "container_image_name": "docker.io/ceph/ceph-grafana:latest",
 164         "daemon_id": "ceph-001",
 165         "daemon_type": "grafana",
 166         "version": "6.6.2",
 167         "status": 1,
 168         "status_desc": "running",
 169         "last_refresh": "2020-04-03T15:31:48.725950",
 170         "created": "2020-04-02T19:23:52.025088",
 171         "started": "2020-04-03T07:29:16.847972",
 172         "is_active": false
 173     },
 174     {
 175         "hostname": "ceph-001",
 176         "container_id": "9ca007280456",
 177         "container_image_id": "204a01f9b0b6710dd0c0af7f37ce7139c47ff0f0105d778d7104c69282dfbbf1",
 178         "container_image_name": "docker.io/ceph/ceph:v15",
 179         "daemon_id": "ceph-001.gkjwqp",
 180         "daemon_type": "mgr",
 181         "version": "15.2.0",
 182         "status": 1,
 183         "status_desc": "running",
 184         "last_refresh": "2020-04-03T15:31:48.725807",
 185         "created": "2020-04-02T19:22:18.648584",
 186         "started": "2020-04-03T07:29:16.856153",
 187         "is_active": false
 188     },
 189     {
 190         "hostname": "ceph-001",
 191         "container_id": "3d1ba9a2b697",
 192         "container_image_id": "204a01f9b0b6710dd0c0af7f37ce7139c47ff0f0105d778d7104c69282dfbbf1",
 193         "container_image_name": "docker.io/ceph/ceph:v15",
 194         "daemon_id": "ceph-001",
 195         "daemon_type": "mon",
 196         "version": "15.2.0",
 197         "status": 1,
 198         "status_desc": "running",
 199         "last_refresh": "2020-04-03T15:31:48.725715",
 200         "created": "2020-04-02T19:22:13.863300",
 201         "started": "2020-04-03T07:29:17.206024",
 202         "is_active": false
 203     },
 204     {
 205         "hostname": "ceph-001",
 206         "container_id": "36d026c68ba1",
 207         "container_image_id": "e5a616e4b9cf68dfcad7782b78e118be4310022e874d52da85c55923fb615f87",
 208         "container_image_name": "docker.io/prom/node-exporter:latest",
 209         "daemon_id": "ceph-001",
 210         "daemon_type": "node-exporter",
 211         "version": "0.18.1",
 212         "status": 1,
 213         "status_desc": "running",
 214         "last_refresh": "2020-04-03T15:31:48.725996",
 215         "created": "2020-04-02T19:23:53.880197",
 216         "started": "2020-04-03T07:29:16.880044",
 217         "is_active": false
 218     },
 219     {
 220         "hostname": "ceph-001",
 221         "container_id": "faf76193cbfe",
 222         "container_image_id": "204a01f9b0b6710dd0c0af7f37ce7139c47ff0f0105d778d7104c69282dfbbf1",
 223         "container_image_name": "docker.io/ceph/ceph:v15",
 224         "daemon_id": "0",
 225         "daemon_type": "osd",
 226         "version": "15.2.0",
 227         "status": 1,
 228         "status_desc": "running",
 229         "last_refresh": "2020-04-03T15:31:48.726088",
 230         "created": "2020-04-02T20:35:02.991435",
 231         "started": "2020-04-03T07:29:19.373956",
 232         "is_active": false
 233     },
 234     {
 235         "hostname": "ceph-001",
 236         "container_id": "f82505bae0f1",
 237         "container_image_id": "204a01f9b0b6710dd0c0af7f37ce7139c47ff0f0105d778d7104c69282dfbbf1",
 238         "container_image_name": "docker.io/ceph/ceph:v15",
 239         "daemon_id": "1",
 240         "daemon_type": "osd",
 241         "version": "15.2.0",
 242         "status": 1,
 243         "status_desc": "running",
 244         "last_refresh": "2020-04-03T15:31:48.726134",
 245         "created": "2020-04-02T20:35:17.142272",
 246         "started": "2020-04-03T07:29:19.374002",
 247         "is_active": false
 248     },
 249     {
 250         "hostname": "ceph-001",
 251         "container_id": "2708d84cd484",
 252         "container_image_id": "358a0d2395fe711bb8258e8fb4b2d7865c0a9a6463969bcd1452ee8869ea6653",
 253         "container_image_name": "docker.io/prom/prometheus:latest",
 254         "daemon_id": "ceph-001",
 255         "daemon_type": "prometheus",
 256         "version": "2.17.1",
 257         "status": 1,
 258         "status_desc": "running",
 259         "last_refresh": "2020-04-03T15:31:48.726042",
 260         "created": "2020-04-02T19:24:10.281163",
 261         "started": "2020-04-03T07:29:16.926292",
 262         "is_active": false
 263     },
 264     {
 265         "hostname": "ceph-001",
 266         "daemon_id": "default-rgw-realm.eu-central-1.1.ceph-001.ytywjo",
 267         "daemon_type": "rgw",
 268         "status": 1,
 269         "status_desc": "starting",
 270         "is_active": false
 271     }
 272 ]""")
 273 )
 274 def test_dd_octopus(dd_json):
 275     # https://tracker.ceph.com/issues/44934
 276     # Those are real user data from early octopus.
 277     # Please do not modify those JSON values.
 278
 279     # Convert datetime properties to old style.
 280     # 2020-04-03T07:29:16.926292Z -> 2020-04-03T07:29:16.926292
 281     def convert_to_old_style_json(j):
 282         for k in ['last_refresh', 'created', 'started', 'last_deployed',
 283                   'last_configured']:
 284             if k in j:
 285                 j[k] = j[k].rstrip('Z')
 286         del j['daemon_name']
 287         return j
 288
 289     assert dd_json == convert_to_old_style_json(
 290         DaemonDescription.from_json(dd_json).to_json())
 291
 292
 293 @pytest.mark.parametrize("spec,dd,valid",
 294 [   # noqa: E128
 295     # https://tracker.ceph.com/issues/44934
 296     (
 297         RGWSpec(
 298             service_id="foo",
 299             rgw_realm="default-rgw-realm",
 300             rgw_zone="eu-central-1",
 301         ),
 302         DaemonDescription(
 303             daemon_type='rgw',
 304             daemon_id="foo.ceph-001.ytywjo",
 305             hostname="ceph-001",
 306         ),
 307         True
 308     ),
 309     (
 310         # no realm
 311         RGWSpec(
 312             service_id="foo.bar",
 313             rgw_zone="eu-central-1",
 314         ),
 315         DaemonDescription(
 316             daemon_type='rgw',
 317             daemon_id="foo.bar.ceph-001.ytywjo",
 318             hostname="ceph-001",
 319         ),
 320         True
 321     ),
 322     (
 323         # no realm or zone
 324         RGWSpec(
 325             service_id="bar",
 326         ),
 327         DaemonDescription(
 328             daemon_type='rgw',
 329             daemon_id="bar.host.domain.tld.ytywjo",
 330             hostname="host.domain.tld",
 331         ),
 332         True
 333     ),
 334     (
 335         # explicit naming
 336         RGWSpec(
 337             service_id="realm.zone",
 338         ),
 339         DaemonDescription(
 340             daemon_type='rgw',
 341             daemon_id="realm.zone.a",
 342             hostname="smithi028",
 343         ),
 344         True
 345     ),
 346     (
 347         # without host
 348         RGWSpec(
 349             service_type='rgw',
 350             service_id="foo",
 351         ),
 352         DaemonDescription(
 353             daemon_type='rgw',
 354             daemon_id="foo.hostname.ytywjo",
 355             hostname=None,
 356         ),
 357         False
 358     ),
 359     (
 360         # without host (2)
 361         RGWSpec(
 362             service_type='rgw',
 363             service_id="default-rgw-realm.eu-central-1.1",
 364         ),
 365         DaemonDescription(
 366             daemon_type='rgw',
 367             daemon_id="default-rgw-realm.eu-central-1.1.hostname.ytywjo",
 368             hostname=None,
 369         ),
 370         False
 371     ),
 372     (
 373         # service_id contains hostname
 374         # (sort of) https://tracker.ceph.com/issues/45294
 375         RGWSpec(
 376             service_id="default.rgw.realm.ceph.001",
 377         ),
 378         DaemonDescription(
 379             daemon_type='rgw',
 380             daemon_id="default.rgw.realm.ceph.001.ceph.001.ytywjo",
 381             hostname="ceph.001",
 382         ),
 383         True
 384     ),
 385
 386     # https://tracker.ceph.com/issues/45293
 387     (
 388         ServiceSpec(
 389             service_type='mds',
 390             service_id="a",
 391         ),
 392         DaemonDescription(
 393             daemon_type='mds',
 394             daemon_id="a.host1.abc123",
 395             hostname="host1",
 396         ),
 397         True
 398     ),
 399     (
 400         # '.' char in service_id
 401         ServiceSpec(
 402             service_type='mds',
 403             service_id="a.b.c",
 404         ),
 405         DaemonDescription(
 406             daemon_type='mds',
 407             daemon_id="a.b.c.host1.abc123",
 408             hostname="host1",
 409         ),
 410         True
 411     ),
 412
 413     # https://tracker.ceph.com/issues/45617
 414     (
 415         # daemon_id does not contain hostname
 416         ServiceSpec(
 417             service_type='mds',
 418             service_id="a",
 419         ),
 420         DaemonDescription(
 421             daemon_type='mds',
 422             daemon_id="a",
 423             hostname="host1",
 424         ),
 425         True
 426     ),
 427     (
 428         # daemon_id only contains hostname
 429         ServiceSpec(
 430             service_type='mds',
 431             service_id="host1",
 432         ),
 433         DaemonDescription(
 434             daemon_type='mds',
 435             daemon_id="host1",
 436             hostname="host1",
 437         ),
 438         True
 439     ),
 440
 441     # https://tracker.ceph.com/issues/45399
 442     (
 443         # daemon_id only contains hostname
 444         ServiceSpec(
 445             service_type='mds',
 446             service_id="a",
 447         ),
 448         DaemonDescription(
 449             daemon_type='mds',
 450             daemon_id="a.host1.abc123",
 451             hostname="host1.site",
 452         ),
 453         True
 454     ),
 455     (
 456         NFSServiceSpec(
 457             service_id="a",
 458         ),
 459         DaemonDescription(
 460             daemon_type='nfs',
 461             daemon_id="a.host1",
 462             hostname="host1.site",
 463         ),
 464         True
 465     ),
 466
 467     # https://tracker.ceph.com/issues/45293
 468     (
 469         NFSServiceSpec(
 470             service_id="a",
 471         ),
 472         DaemonDescription(
 473             daemon_type='nfs',
 474             daemon_id="a.host1",
 475             hostname="host1",
 476         ),
 477         True
 478     ),
 479     (
 480         # service_id contains a '.' char
 481         NFSServiceSpec(
 482             service_id="a.b.c",
 483         ),
 484         DaemonDescription(
 485             daemon_type='nfs',
 486             daemon_id="a.b.c.host1",
 487             hostname="host1",
 488         ),
 489         True
 490     ),
 491     (
 492         # trailing chars after hostname
 493         NFSServiceSpec(
 494             service_id="a.b.c",
 495         ),
 496         DaemonDescription(
 497             daemon_type='nfs',
 498             daemon_id="a.b.c.host1.abc123",
 499             hostname="host1",
 500         ),
 501         True
 502     ),
 503     (
 504         # chars after hostname without '.'
 505         NFSServiceSpec(
 506             service_id="a",
 507         ),
 508         DaemonDescription(
 509             daemon_type='nfs',
 510             daemon_id="a.host1abc123",
 511             hostname="host1",
 512         ),
 513         False
 514     ),
 515     (
 516         # chars before hostname without '.'
 517         NFSServiceSpec(
 518             service_id="a",
 519         ),
 520         DaemonDescription(
 521             daemon_type='nfs',
 522             daemon_id="ahost1.abc123",
 523             hostname="host1",
 524         ),
 525         False
 526     ),
 527
 528     # https://tracker.ceph.com/issues/45293
 529     (
 530         IscsiServiceSpec(
 531             service_type='iscsi',
 532             service_id="a",
 533         ),
 534         DaemonDescription(
 535             daemon_type='iscsi',
 536             daemon_id="a.host1.abc123",
 537             hostname="host1",
 538         ),
 539         True
 540     ),
 541     (
 542         # '.' char in service_id
 543         IscsiServiceSpec(
 544             service_type='iscsi',
 545             service_id="a.b.c",
 546         ),
 547         DaemonDescription(
 548             daemon_type='iscsi',
 549             daemon_id="a.b.c.host1.abc123",
 550             hostname="host1",
 551         ),
 552         True
 553     ),
 554     (
 555         # fixed daemon id for teuthology.
 556         IscsiServiceSpec(
 557             service_type='iscsi',
 558             service_id='iscsi',
 559         ),
 560         DaemonDescription(
 561             daemon_type='iscsi',
 562             daemon_id="iscsi.a",
 563             hostname="host1",
 564         ),
 565         True
 566     ),
 567
 568     (
 569         CustomContainerSpec(
 570             service_type='container',
 571             service_id='hello-world',
 572             image='docker.io/library/hello-world:latest',
 573         ),
 574         DaemonDescription(
 575             daemon_type='container',
 576             daemon_id='hello-world.mgr0',
 577             hostname='mgr0',
 578         ),
 579         True
 580     ),
 581
 582 ])
 583 def test_daemon_description_service_name(spec: ServiceSpec,
 584                                          dd: DaemonDescription,
 585                                          valid: bool):
 586     if valid:
 587         assert spec.service_name() == dd.service_name()
 588     else:
 589         with pytest.raises(OrchestratorError):
 590             dd.service_name()