ceph/src/pybind/mgr/cephadm/tests/test_spec.py

   1 # Disable autopep8 for this file:
   2
   3 # fmt: off
   4
   5 import json
   6
   7 import pytest
   8
   9 from ceph.deployment.service_spec import ServiceSpec, NFSServiceSpec, RGWSpec, \
  10     IscsiServiceSpec, HostPlacementSpec, CustomContainerSpec
  11 from orchestrator import DaemonDescription, OrchestratorError
  12
  13
  14 @pytest.mark.parametrize(
  15     "spec_json",
  16     json.loads("""[
  17 {
  18   "placement": {
  19     "count": 1
  20   },
  21   "service_type": "alertmanager"
  22 },
  23 {
  24   "placement": {
  25     "host_pattern": "*"
  26   },
  27   "service_type": "crash"
  28 },
  29 {
  30   "placement": {
  31     "count": 1
  32   },
  33   "service_type": "grafana"
  34 },
  35 {
  36   "placement": {
  37     "count": 2
  38   },
  39   "service_type": "mgr"
  40 },
  41 {
  42   "placement": {
  43     "count": 5
  44   },
  45   "service_type": "mon"
  46 },
  47 {
  48   "placement": {
  49     "host_pattern": "*"
  50   },
  51   "service_type": "node-exporter"
  52 },
  53 {
  54   "placement": {
  55     "count": 1
  56   },
  57   "service_type": "prometheus"
  58 },
  59 {
  60   "placement": {
  61     "hosts": [
  62       {
  63         "hostname": "ceph-001",
  64         "network": "",
  65         "name": ""
  66       }
  67     ]
  68   },
  69   "service_type": "rgw",
  70   "service_id": "default-rgw-realm.eu-central-1.1",
  71   "rgw_realm": "default-rgw-realm",
  72   "rgw_zone": "eu-central-1"
  73 },
  74 {
  75   "service_type": "osd",
  76   "service_id": "osd_spec_default",
  77   "placement": {
  78     "host_pattern": "*"
  79   },
  80   "data_devices": {
  81     "model": "MC-55-44-XZ"
  82   },
  83   "db_devices": {
  84     "model": "SSD-123-foo"
  85   },
  86   "wal_devices": {
  87     "model": "NVME-QQQQ-987"
  88   }
  89 }
  90 ]
  91 """)
  92 )
  93 def test_spec_octopus(spec_json):
  94     # https://tracker.ceph.com/issues/44934
  95     # Those are real user data from early octopus.
  96     # Please do not modify those JSON values.
  97
  98     spec = ServiceSpec.from_json(spec_json)
  99
 100     # just some verification that we can sill read old octopus specs
 101     def convert_to_old_style_json(j):
 102         j_c = dict(j.copy())
 103         j_c.pop('service_name', None)
 104         if 'spec' in j_c:
 105             spec = j_c.pop('spec')
 106             j_c.update(spec)
 107         if 'placement' in j_c:
 108             if 'hosts' in j_c['placement']:
 109                 j_c['placement']['hosts'] = [
 110                     {
 111                         'hostname': HostPlacementSpec.parse(h).hostname,
 112                         'network': HostPlacementSpec.parse(h).network,
 113                         'name': HostPlacementSpec.parse(h).name
 114                     }
 115                     for h in j_c['placement']['hosts']
 116                 ]
 117         j_c.pop('objectstore', None)
 118         j_c.pop('filter_logic', None)
 119         return j_c
 120
 121     assert spec_json == convert_to_old_style_json(spec.to_json())
 122
 123
 124 @pytest.mark.parametrize(
 125     "dd_json",
 126     json.loads("""[
 127     {
 128         "hostname": "ceph-001",
 129         "container_id": "d94d7969094d",
 130         "container_image_id": "0881eb8f169f5556a292b4e2c01d683172b12830a62a9225a98a8e206bb734f0",
 131         "container_image_name": "docker.io/prom/alertmanager:latest",
 132         "daemon_id": "ceph-001",
 133         "daemon_type": "alertmanager",
 134         "version": "0.20.0",
 135         "status": 1,
 136         "status_desc": "running",
 137         "last_refresh": "2020-04-03T15:31:48.725856",
 138         "created": "2020-04-02T19:23:08.829543",
 139         "started": "2020-04-03T07:29:16.932838",
 140         "is_active": false
 141     },
 142     {
 143         "hostname": "ceph-001",
 144         "container_id": "c4b036202241",
 145         "container_image_id": "204a01f9b0b6710dd0c0af7f37ce7139c47ff0f0105d778d7104c69282dfbbf1",
 146         "container_image_name": "docker.io/ceph/ceph:v15",
 147         "daemon_id": "ceph-001",
 148         "daemon_type": "crash",
 149         "version": "15.2.0",
 150         "status": 1,
 151         "status_desc": "running",
 152         "last_refresh": "2020-04-03T15:31:48.725903",
 153         "created": "2020-04-02T19:23:11.390694",
 154         "started": "2020-04-03T07:29:16.910897",
 155         "is_active": false
 156     },
 157     {
 158         "hostname": "ceph-001",
 159         "container_id": "5b7b94b48f31",
 160         "container_image_id": "87a51ecf0b1c9a7b187b21c1b071425dafea0d765a96d5bc371c791169b3d7f4",
 161         "container_image_name": "docker.io/ceph/ceph-grafana:latest",
 162         "daemon_id": "ceph-001",
 163         "daemon_type": "grafana",
 164         "version": "6.6.2",
 165         "status": 1,
 166         "status_desc": "running",
 167         "last_refresh": "2020-04-03T15:31:48.725950",
 168         "created": "2020-04-02T19:23:52.025088",
 169         "started": "2020-04-03T07:29:16.847972",
 170         "is_active": false
 171     },
 172     {
 173         "hostname": "ceph-001",
 174         "container_id": "9ca007280456",
 175         "container_image_id": "204a01f9b0b6710dd0c0af7f37ce7139c47ff0f0105d778d7104c69282dfbbf1",
 176         "container_image_name": "docker.io/ceph/ceph:v15",
 177         "daemon_id": "ceph-001.gkjwqp",
 178         "daemon_type": "mgr",
 179         "version": "15.2.0",
 180         "status": 1,
 181         "status_desc": "running",
 182         "last_refresh": "2020-04-03T15:31:48.725807",
 183         "created": "2020-04-02T19:22:18.648584",
 184         "started": "2020-04-03T07:29:16.856153",
 185         "is_active": false
 186     },
 187     {
 188         "hostname": "ceph-001",
 189         "container_id": "3d1ba9a2b697",
 190         "container_image_id": "204a01f9b0b6710dd0c0af7f37ce7139c47ff0f0105d778d7104c69282dfbbf1",
 191         "container_image_name": "docker.io/ceph/ceph:v15",
 192         "daemon_id": "ceph-001",
 193         "daemon_type": "mon",
 194         "version": "15.2.0",
 195         "status": 1,
 196         "status_desc": "running",
 197         "last_refresh": "2020-04-03T15:31:48.725715",
 198         "created": "2020-04-02T19:22:13.863300",
 199         "started": "2020-04-03T07:29:17.206024",
 200         "is_active": false
 201     },
 202     {
 203         "hostname": "ceph-001",
 204         "container_id": "36d026c68ba1",
 205         "container_image_id": "e5a616e4b9cf68dfcad7782b78e118be4310022e874d52da85c55923fb615f87",
 206         "container_image_name": "docker.io/prom/node-exporter:latest",
 207         "daemon_id": "ceph-001",
 208         "daemon_type": "node-exporter",
 209         "version": "0.18.1",
 210         "status": 1,
 211         "status_desc": "running",
 212         "last_refresh": "2020-04-03T15:31:48.725996",
 213         "created": "2020-04-02T19:23:53.880197",
 214         "started": "2020-04-03T07:29:16.880044",
 215         "is_active": false
 216     },
 217     {
 218         "hostname": "ceph-001",
 219         "container_id": "faf76193cbfe",
 220         "container_image_id": "204a01f9b0b6710dd0c0af7f37ce7139c47ff0f0105d778d7104c69282dfbbf1",
 221         "container_image_name": "docker.io/ceph/ceph:v15",
 222         "daemon_id": "0",
 223         "daemon_type": "osd",
 224         "version": "15.2.0",
 225         "status": 1,
 226         "status_desc": "running",
 227         "last_refresh": "2020-04-03T15:31:48.726088",
 228         "created": "2020-04-02T20:35:02.991435",
 229         "started": "2020-04-03T07:29:19.373956",
 230         "is_active": false
 231     },
 232     {
 233         "hostname": "ceph-001",
 234         "container_id": "f82505bae0f1",
 235         "container_image_id": "204a01f9b0b6710dd0c0af7f37ce7139c47ff0f0105d778d7104c69282dfbbf1",
 236         "container_image_name": "docker.io/ceph/ceph:v15",
 237         "daemon_id": "1",
 238         "daemon_type": "osd",
 239         "version": "15.2.0",
 240         "status": 1,
 241         "status_desc": "running",
 242         "last_refresh": "2020-04-03T15:31:48.726134",
 243         "created": "2020-04-02T20:35:17.142272",
 244         "started": "2020-04-03T07:29:19.374002",
 245         "is_active": false
 246     },
 247     {
 248         "hostname": "ceph-001",
 249         "container_id": "2708d84cd484",
 250         "container_image_id": "358a0d2395fe711bb8258e8fb4b2d7865c0a9a6463969bcd1452ee8869ea6653",
 251         "container_image_name": "docker.io/prom/prometheus:latest",
 252         "daemon_id": "ceph-001",
 253         "daemon_type": "prometheus",
 254         "version": "2.17.1",
 255         "status": 1,
 256         "status_desc": "running",
 257         "last_refresh": "2020-04-03T15:31:48.726042",
 258         "created": "2020-04-02T19:24:10.281163",
 259         "started": "2020-04-03T07:29:16.926292",
 260         "is_active": false
 261     },
 262     {
 263         "hostname": "ceph-001",
 264         "daemon_id": "default-rgw-realm.eu-central-1.1.ceph-001.ytywjo",
 265         "daemon_type": "rgw",
 266         "status": 1,
 267         "status_desc": "starting",
 268         "is_active": false
 269     }
 270 ]""")
 271 )
 272 def test_dd_octopus(dd_json):
 273     # https://tracker.ceph.com/issues/44934
 274     # Those are real user data from early octopus.
 275     # Please do not modify those JSON values.
 276
 277     # Convert datetime properties to old style.
 278     # 2020-04-03T07:29:16.926292Z -> 2020-04-03T07:29:16.926292
 279     def convert_to_old_style_json(j):
 280         for k in ['last_refresh', 'created', 'started', 'last_deployed',
 281                   'last_configured']:
 282             if k in j:
 283                 j[k] = j[k].rstrip('Z')
 284         del j['daemon_name']
 285         return j
 286
 287     assert dd_json == convert_to_old_style_json(
 288         DaemonDescription.from_json(dd_json).to_json())
 289
 290
 291 @pytest.mark.parametrize("spec,dd,valid",
 292 [   # noqa: E128
 293     # https://tracker.ceph.com/issues/44934
 294     (
 295         RGWSpec(
 296             service_id="foo",
 297             rgw_realm="default-rgw-realm",
 298             rgw_zone="eu-central-1",
 299         ),
 300         DaemonDescription(
 301             daemon_type='rgw',
 302             daemon_id="foo.ceph-001.ytywjo",
 303             hostname="ceph-001",
 304         ),
 305         True
 306     ),
 307     (
 308         # no realm
 309         RGWSpec(
 310             service_id="foo.bar",
 311             rgw_zone="eu-central-1",
 312         ),
 313         DaemonDescription(
 314             daemon_type='rgw',
 315             daemon_id="foo.bar.ceph-001.ytywjo",
 316             hostname="ceph-001",
 317         ),
 318         True
 319     ),
 320     (
 321         # no realm or zone
 322         RGWSpec(
 323             service_id="bar",
 324         ),
 325         DaemonDescription(
 326             daemon_type='rgw',
 327             daemon_id="bar.host.domain.tld.ytywjo",
 328             hostname="host.domain.tld",
 329         ),
 330         True
 331     ),
 332     (
 333         # explicit naming
 334         RGWSpec(
 335             service_id="realm.zone",
 336         ),
 337         DaemonDescription(
 338             daemon_type='rgw',
 339             daemon_id="realm.zone.a",
 340             hostname="smithi028",
 341         ),
 342         True
 343     ),
 344     (
 345         # without host
 346         RGWSpec(
 347             service_type='rgw',
 348             service_id="foo",
 349         ),
 350         DaemonDescription(
 351             daemon_type='rgw',
 352             daemon_id="foo.hostname.ytywjo",
 353             hostname=None,
 354         ),
 355         False
 356     ),
 357     (
 358         # without host (2)
 359         RGWSpec(
 360             service_type='rgw',
 361             service_id="default-rgw-realm.eu-central-1.1",
 362         ),
 363         DaemonDescription(
 364             daemon_type='rgw',
 365             daemon_id="default-rgw-realm.eu-central-1.1.hostname.ytywjo",
 366             hostname=None,
 367         ),
 368         False
 369     ),
 370     (
 371         # service_id contains hostname
 372         # (sort of) https://tracker.ceph.com/issues/45294
 373         RGWSpec(
 374             service_id="default.rgw.realm.ceph.001",
 375         ),
 376         DaemonDescription(
 377             daemon_type='rgw',
 378             daemon_id="default.rgw.realm.ceph.001.ceph.001.ytywjo",
 379             hostname="ceph.001",
 380         ),
 381         True
 382     ),
 383
 384     # https://tracker.ceph.com/issues/45293
 385     (
 386         ServiceSpec(
 387             service_type='mds',
 388             service_id="a",
 389         ),
 390         DaemonDescription(
 391             daemon_type='mds',
 392             daemon_id="a.host1.abc123",
 393             hostname="host1",
 394         ),
 395         True
 396     ),
 397     (
 398         # '.' char in service_id
 399         ServiceSpec(
 400             service_type='mds',
 401             service_id="a.b.c",
 402         ),
 403         DaemonDescription(
 404             daemon_type='mds',
 405             daemon_id="a.b.c.host1.abc123",
 406             hostname="host1",
 407         ),
 408         True
 409     ),
 410
 411     # https://tracker.ceph.com/issues/45617
 412     (
 413         # daemon_id does not contain hostname
 414         ServiceSpec(
 415             service_type='mds',
 416             service_id="a",
 417         ),
 418         DaemonDescription(
 419             daemon_type='mds',
 420             daemon_id="a",
 421             hostname="host1",
 422         ),
 423         True
 424     ),
 425     (
 426         # daemon_id only contains hostname
 427         ServiceSpec(
 428             service_type='mds',
 429             service_id="host1",
 430         ),
 431         DaemonDescription(
 432             daemon_type='mds',
 433             daemon_id="host1",
 434             hostname="host1",
 435         ),
 436         True
 437     ),
 438
 439     # https://tracker.ceph.com/issues/45399
 440     (
 441         # daemon_id only contains hostname
 442         ServiceSpec(
 443             service_type='mds',
 444             service_id="a",
 445         ),
 446         DaemonDescription(
 447             daemon_type='mds',
 448             daemon_id="a.host1.abc123",
 449             hostname="host1.site",
 450         ),
 451         True
 452     ),
 453     (
 454         NFSServiceSpec(
 455             service_id="a",
 456         ),
 457         DaemonDescription(
 458             daemon_type='nfs',
 459             daemon_id="a.host1",
 460             hostname="host1.site",
 461         ),
 462         True
 463     ),
 464
 465     # https://tracker.ceph.com/issues/45293
 466     (
 467         NFSServiceSpec(
 468             service_id="a",
 469         ),
 470         DaemonDescription(
 471             daemon_type='nfs',
 472             daemon_id="a.host1",
 473             hostname="host1",
 474         ),
 475         True
 476     ),
 477     (
 478         # service_id contains a '.' char
 479         NFSServiceSpec(
 480             service_id="a.b.c",
 481         ),
 482         DaemonDescription(
 483             daemon_type='nfs',
 484             daemon_id="a.b.c.host1",
 485             hostname="host1",
 486         ),
 487         True
 488     ),
 489     (
 490         # trailing chars after hostname
 491         NFSServiceSpec(
 492             service_id="a.b.c",
 493         ),
 494         DaemonDescription(
 495             daemon_type='nfs',
 496             daemon_id="a.b.c.host1.abc123",
 497             hostname="host1",
 498         ),
 499         True
 500     ),
 501     (
 502         # chars after hostname without '.'
 503         NFSServiceSpec(
 504             service_id="a",
 505         ),
 506         DaemonDescription(
 507             daemon_type='nfs',
 508             daemon_id="a.host1abc123",
 509             hostname="host1",
 510         ),
 511         False
 512     ),
 513     (
 514         # chars before hostname without '.'
 515         NFSServiceSpec(
 516             service_id="a",
 517         ),
 518         DaemonDescription(
 519             daemon_type='nfs',
 520             daemon_id="ahost1.abc123",
 521             hostname="host1",
 522         ),
 523         False
 524     ),
 525
 526     # https://tracker.ceph.com/issues/45293
 527     (
 528         IscsiServiceSpec(
 529             service_type='iscsi',
 530             service_id="a",
 531         ),
 532         DaemonDescription(
 533             daemon_type='iscsi',
 534             daemon_id="a.host1.abc123",
 535             hostname="host1",
 536         ),
 537         True
 538     ),
 539     (
 540         # '.' char in service_id
 541         IscsiServiceSpec(
 542             service_type='iscsi',
 543             service_id="a.b.c",
 544         ),
 545         DaemonDescription(
 546             daemon_type='iscsi',
 547             daemon_id="a.b.c.host1.abc123",
 548             hostname="host1",
 549         ),
 550         True
 551     ),
 552     (
 553         # fixed daemon id for teuthology.
 554         IscsiServiceSpec(
 555             service_type='iscsi',
 556             service_id='iscsi',
 557         ),
 558         DaemonDescription(
 559             daemon_type='iscsi',
 560             daemon_id="iscsi.a",
 561             hostname="host1",
 562         ),
 563         True
 564     ),
 565
 566     (
 567         CustomContainerSpec(
 568             service_type='container',
 569             service_id='hello-world',
 570             image='docker.io/library/hello-world:latest',
 571         ),
 572         DaemonDescription(
 573             daemon_type='container',
 574             daemon_id='hello-world.mgr0',
 575             hostname='mgr0',
 576         ),
 577         True
 578     ),
 579
 580 ])
 581 def test_daemon_description_service_name(spec: ServiceSpec,
 582                                          dd: DaemonDescription,
 583                                          valid: bool):
 584     if valid:
 585         assert spec.service_name() == dd.service_name()
 586     else:
 587         with pytest.raises(OrchestratorError):
 588             dd.service_name()